└── SubgEnumFramework ├── .gitignore ├── .settings ├── org.eclipse.ltk.core.refactoring.prefs └── org.eclipse.jdt.core.prefs ├── src ├── dbg │ └── hadoop │ │ ├── subgraphs │ │ ├── io │ │ │ ├── HVArrayPartitioner.java │ │ │ ├── IntegerPairPartitioner.java │ │ │ ├── DoubleIntegerPairPartitioner2.java │ │ │ ├── HyperVertexPartitioner.java │ │ │ ├── IntegerComparator.java │ │ │ ├── DoubleIntegerPairGroupComparator.java │ │ │ ├── HyperVertexSignGroupComparator.java │ │ │ ├── HyperVertexComparator.java │ │ │ ├── DoubleIntegerPairComparator.java │ │ │ ├── HyperVertexSignComparator.java │ │ │ ├── IntegerPairComparator.java │ │ │ ├── ColorVertexPartitioner.java │ │ │ ├── HVArrayComparator.java │ │ │ ├── HVArrayGroupComparator.java │ │ │ ├── HVArraySignComparator.java │ │ │ ├── HyperVertexSign.java │ │ │ └── DoubleIntegerPairWritable.java │ │ └── utils │ │ │ ├── bloomfilter │ │ │ ├── io │ │ │ │ ├── IAllocator.java │ │ │ │ ├── NativeAllocator.java │ │ │ │ ├── ISerializer.java │ │ │ │ └── RefCountedMemory.java │ │ │ ├── utils │ │ │ │ ├── IFilter.java │ │ │ │ ├── AlwaysPresentFilter.java │ │ │ │ ├── Murmur3BloomFilter.java │ │ │ │ └── BloomFilterSerializer.java │ │ │ └── obs │ │ │ │ └── IBitSet.java │ │ │ ├── HyperGraphTest.java │ │ │ ├── BinarySearch.java │ │ │ └── Config.java │ │ └── subgenum │ │ ├── maximalclique │ │ ├── MCliqueS4Mapper.java │ │ ├── MCliqueS2Mapper.java │ │ ├── MCliqueS3Mapper.java │ │ ├── MCliqueS3DegreeMapper.java │ │ ├── MCliqueS3Reducer.java │ │ ├── EdgeMapper.java │ │ ├── MCliqueS5Reducer.java │ │ ├── MCliqueS1Reducer.java │ │ ├── MCliqueS4Reducer.java │ │ ├── MCliqueS2Reducer.java │ │ ├── MCliqueS5Driver.java │ │ ├── MCliqueS4Driver.java │ │ ├── MCliqueS1Mapper.java │ │ ├── MCliqueS5Mapper.java │ │ └── MCliqueS3Driver.java │ │ ├── qtcounter │ │ ├── QtS3Mapper.java │ │ ├── QtS2Mapper.java │ │ ├── QtS3Reducer.java │ │ ├── EdgeMapper.java │ │ ├── QtS1Reducer.java │ │ ├── QtS2Reducer.java │ │ ├── QtS1Mapper.java │ │ ├── QtS3Driver.java │ │ └── QtS1Driver.java │ │ ├── frame │ │ ├── GeneralPatternCountIdentityMapper.java │ │ ├── GeneralPatternCountReducer.java │ │ └── GeneralPatternCountDriver.java │ │ ├── hypergraph │ │ ├── HyperGraphStageThreeMapper.java │ │ ├── HyperGraphStageSixCombiner.java │ │ ├── HyperGraphStageSixMapper.java │ │ ├── HyperGraphStageFourMapper2.java │ │ ├── HyperGraphStageFourMapper1.java │ │ ├── HyperGraphStageSixReducer.java │ │ ├── adjlist │ │ │ ├── GenAdjListMapper.java │ │ │ ├── GenAdjList.java │ │ │ └── GenAdjListDriver.java │ │ ├── triangle │ │ │ ├── EdgeMapper.java │ │ │ ├── TriangleReducer.java │ │ │ ├── TriangleMapper.java │ │ │ └── Triangle.java │ │ ├── HyperGraphStageFiveMapper2.java │ │ ├── HyperGraphStageTwoMapper.java │ │ ├── HyperGraphStageOneMapper.java │ │ ├── HyperGraphStageFiveMapper1.java │ │ ├── HyperGraphStageTwoReducer.java │ │ ├── HyperGraphStageFiveReducer.java │ │ ├── bloomfilter │ │ │ └── GenBloomFilter.java │ │ ├── HyperGraphStageOneReducer.java │ │ ├── HyperGraphStageFourReducer.java │ │ ├── HyperGraphStageThreeReducer.java │ │ ├── HyperGraphStageOneDriver.java │ │ ├── HyperGraphStageTwoDriver.java │ │ └── HyperGraphStageSixDriver.java │ │ └── prepare │ │ ├── GenDegreeReducer.java │ │ ├── UndirectGraphReducer.java │ │ ├── GenDegreeMapper2.java │ │ ├── PrepareDataS1Mapper2.java │ │ ├── PrepareDataDegreeMapper.java │ │ ├── PrepareDataS2Mapper.java │ │ ├── PrepareDataS1Mapper.java │ │ ├── GenDegreeMapper.java │ │ ├── PrepareDataS1Reducer.java │ │ ├── PrepareDataS2Reducer.java │ │ ├── UndirectGraphMapper.java │ │ ├── GenDegreeDriver.java │ │ ├── PrepareDataS2Driver.java │ │ └── PrepareDataS1Driver.java ├── gnu │ └── trove │ │ ├── strategy │ │ ├── IdentityHashingStrategy.java │ │ └── HashingStrategy.java │ │ ├── list │ │ ├── TLinkableAdapter.java │ │ └── TLinkable.java │ │ ├── impl │ │ └── package.html │ │ ├── iterator │ │ ├── TAdvancingIterator.java │ │ ├── TIntIterator.java │ │ ├── TLongIterator.java │ │ ├── TIterator.java │ │ ├── hash │ │ │ └── TObjectHashIterator.java │ │ └── TPrimitiveIterator.java │ │ ├── function │ │ ├── TIntFunction.java │ │ ├── TLongFunction.java │ │ └── TObjectFunction.java │ │ ├── procedure │ │ ├── TIntProcedure.java │ │ ├── TLongProcedure.java │ │ ├── TIntIntProcedure.java │ │ ├── TIntLongProcedure.java │ │ ├── TLongIntProcedure.java │ │ ├── TLongLongProcedure.java │ │ ├── TIntObjectProcedure.java │ │ ├── TObjectIntProcedure.java │ │ ├── TObjectProcedure.java │ │ ├── TLongObjectProcedure.java │ │ ├── TObjectLongProcedure.java │ │ ├── array │ │ │ └── ToObjectArrayProceedure.java │ │ └── TObjectObjectProcedure.java │ │ ├── Version.java │ │ ├── queue │ │ ├── TIntQueue.java │ │ └── TLongQueue.java │ │ └── stack │ │ ├── TIntStack.java │ │ └── TLongStack.java ├── core-site.xml └── hdfs-site.xml ├── .project ├── subgenum.conf ├── subgenum_pre.py └── subgenum_frame.py /SubgEnumFramework/.gitignore: -------------------------------------------------------------------------------- 1 | /bin/ 2 | -------------------------------------------------------------------------------- /SubgEnumFramework/.settings/org.eclipse.ltk.core.refactoring.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.ltk.core.refactoring.enable.project.refactoring.history=false 3 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/HVArrayPartitioner.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import org.apache.hadoop.io.Writable; 4 | import org.apache.hadoop.mapreduce.Partitioner; 5 | 6 | public class HVArrayPartitioner 7 | extends Partitioner{ 8 | 9 | @Override 10 | public int getPartition(HVArray key, Writable value, 11 | int numPartitions) { 12 | // TODO Auto-generated method stub 13 | return ((key.hashCode() & Integer.MAX_VALUE) % numPartitions); 14 | } 15 | } -------------------------------------------------------------------------------- /SubgEnumFramework/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | SubgEnumFramework 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | 15 | org.apache.hadoop.eclipse.Nature 16 | org.eclipse.jdt.core.javanature 17 | 18 | 19 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/maximalclique/MCliqueS4Mapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.maximalclique; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.mapreduce.Mapper; 7 | 8 | public class MCliqueS4Mapper 9 | extends Mapper { 10 | @Override 11 | public void map(LongWritable key, LongWritable value, Context context) 12 | throws IOException, InterruptedException{ 13 | context.write(value, key); 14 | } 15 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/IntegerPairPartitioner.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import org.apache.hadoop.io.Writable; 4 | import org.apache.hadoop.mapreduce.Partitioner; 5 | 6 | public class IntegerPairPartitioner 7 | extends Partitioner{ 8 | 9 | @Override 10 | public int getPartition(IntegerPairWritable key, Writable value, 11 | int numPartitions) { 12 | // TODO Auto-generated method stub 13 | return ((key.hashCode() & Integer.MAX_VALUE) % numPartitions); 14 | } 15 | 16 | 17 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/qtcounter/QtS3Mapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.qtcounter; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.io.NullWritable; 7 | import org.apache.hadoop.mapreduce.Mapper; 8 | 9 | public class QtS3Mapper 10 | extends Mapper { 11 | 12 | @Override 13 | public void map(LongWritable key, LongWritable value, Context context) 14 | throws IOException, InterruptedException{ 15 | context.write(NullWritable.get(), value); 16 | } 17 | 18 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/DoubleIntegerPairPartitioner2.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import org.apache.hadoop.io.IntWritable; 4 | import org.apache.hadoop.io.Writable; 5 | import org.apache.hadoop.mapreduce.Partitioner; 6 | 7 | public class DoubleIntegerPairPartitioner2 8 | extends Partitioner{ 9 | 10 | @Override 11 | public int getPartition(DoubleIntegerPairWritable key, Writable value, 12 | int numPartitions) { 13 | // TODO Auto-generated method stub 14 | return ((31 * key.getFirst() & Integer.MAX_VALUE) % numPartitions); 15 | } 16 | 17 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/frame/GeneralPatternCountIdentityMapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.frame; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.io.NullWritable; 7 | import org.apache.hadoop.mapreduce.Mapper; 8 | 9 | 10 | class GeneralPatternCountIdentityMapper extends 11 | Mapper { 12 | 13 | @Override 14 | public void map(NullWritable _key, LongWritable _value, Context context) 15 | throws IOException, InterruptedException { 16 | 17 | context.write(_key, _value); 18 | } 19 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/HyperVertexPartitioner.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import org.apache.hadoop.io.LongWritable; 4 | import org.apache.hadoop.io.Writable; 5 | import org.apache.hadoop.mapreduce.Partitioner; 6 | 7 | import dbg.hadoop.subgraphs.utils.HyperVertex; 8 | 9 | public class HyperVertexPartitioner 10 | extends Partitioner{ 11 | 12 | @Override 13 | public int getPartition(LongWritable key, Writable value, 14 | int numPartitions) { 15 | // TODO Auto-generated method stub 16 | return (HyperVertex.VertexID(key.get()) % numPartitions); 17 | } 18 | 19 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/maximalclique/MCliqueS2Mapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.maximalclique; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.mapreduce.Mapper; 7 | 8 | import dbg.hadoop.subgraphs.io.HVArray; 9 | 10 | public class MCliqueS2Mapper 11 | extends Mapper { 12 | @Override 13 | public void map(HVArray key, HVArray value, Context context) 14 | throws IOException, InterruptedException{ 15 | for(long u : value.toArrays()){ 16 | context.write(new LongWritable(u), key); 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageThreeMapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.mapreduce.Mapper; 6 | 7 | import dbg.hadoop.subgraphs.io.HVArray; 8 | import dbg.hadoop.subgraphs.io.HyperVertexSign; 9 | 10 | 11 | public class HyperGraphStageThreeMapper 12 | extends Mapper { 13 | 14 | @Override 15 | public void map(HyperVertexSign key, HVArray value, Context context) 16 | throws IOException, InterruptedException{ 17 | context.write(key, value); 18 | } 19 | } 20 | 21 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/qtcounter/QtS2Mapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.qtcounter; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.mapreduce.Mapper; 7 | 8 | import dbg.hadoop.subgraphs.io.HVArray; 9 | 10 | public class QtS2Mapper 11 | extends Mapper { 12 | 13 | @Override 14 | public void map(HVArray key, HVArray value, Context context) 15 | throws IOException, InterruptedException{ 16 | 17 | for(long u : value.toArrays()){ 18 | context.write(new LongWritable(u), key); 19 | } 20 | } 21 | 22 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/strategy/IdentityHashingStrategy.java: -------------------------------------------------------------------------------- 1 | package gnu.trove.strategy; 2 | 3 | /** 4 | * A {@link gnu.trove.strategy.HashingStrategy} that does identity comparisons 5 | * (==) and uses {@link System#identityHashCode(Object)} for hashCode generation. 6 | */ 7 | public class IdentityHashingStrategy implements HashingStrategy { 8 | static final long serialVersionUID = -5188534454583764904L; 9 | 10 | public int computeHashCode( K object ) { 11 | return System.identityHashCode( object ); 12 | } 13 | 14 | public boolean equals( K o1, K o2 ) { 15 | return o1 == o2; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/maximalclique/MCliqueS3Mapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.maximalclique; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.mapreduce.Mapper; 7 | 8 | import dbg.hadoop.subgraphs.io.HVArray; 9 | 10 | public class MCliqueS3Mapper 11 | extends Mapper { 12 | @Override 13 | public void map(LongWritable key, HVArray value, Context context) 14 | throws IOException, InterruptedException{ 15 | for(long v : value.toArrays()){ 16 | context.write(new LongWritable(v), key); 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/prepare/GenDegreeReducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.prepare; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.IntWritable; 6 | import org.apache.hadoop.mapreduce.Reducer; 7 | 8 | public class GenDegreeReducer extends Reducer { 9 | @Override 10 | public void reduce(IntWritable _key, Iterable values, 11 | Context context) throws IOException, InterruptedException{ 12 | int degree = 0; 13 | 14 | for(IntWritable v : values) { 15 | degree += v.get(); 16 | } 17 | context.write(_key, new IntWritable(degree)); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/IntegerComparator.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import org.apache.hadoop.io.IntWritable; 4 | import org.apache.hadoop.io.WritableComparator; 5 | 6 | public class IntegerComparator extends WritableComparator { 7 | 8 | protected IntegerComparator() { 9 | super(IntWritable.class); 10 | // TODO Auto-generated constructor stub 11 | } 12 | 13 | @Override 14 | public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2){ 15 | int cmp = 0; 16 | int i1 = readInt(b1, s1); 17 | int i2 = readInt(b2, s2); 18 | cmp = (i1 < i2) ? -1 : ((i1 == i2) ? 0: 1); 19 | return cmp; 20 | } 21 | 22 | } -------------------------------------------------------------------------------- /SubgEnumFramework/.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled 3 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 4 | org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve 5 | org.eclipse.jdt.core.compiler.compliance=1.6 6 | org.eclipse.jdt.core.compiler.debug.lineNumber=generate 7 | org.eclipse.jdt.core.compiler.debug.localVariable=generate 8 | org.eclipse.jdt.core.compiler.debug.sourceFile=generate 9 | org.eclipse.jdt.core.compiler.problem.assertIdentifier=error 10 | org.eclipse.jdt.core.compiler.problem.enumIdentifier=error 11 | org.eclipse.jdt.core.compiler.source=1.6 12 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/DoubleIntegerPairGroupComparator.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import org.apache.hadoop.io.WritableComparator; 4 | 5 | public class DoubleIntegerPairGroupComparator extends WritableComparator { 6 | 7 | protected DoubleIntegerPairGroupComparator() { 8 | super(DoubleIntegerPairWritable.class); 9 | // TODO Auto-generated constructor stub 10 | } 11 | 12 | @Override 13 | public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2){ 14 | int cmp = 0; 15 | int i1 = readInt(b1, s1); 16 | int i2 = readInt(b2, s2); 17 | cmp = (i1 < i2) ? -1 : ((i1 == i2) ? 0: 1); 18 | return cmp; 19 | } 20 | 21 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageSixCombiner.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.NullWritable; 6 | import org.apache.hadoop.mapreduce.Reducer; 7 | 8 | import dbg.hadoop.subgraphs.io.HVArray; 9 | 10 | /** 11 | * Merge duplicated results. 12 | * @author robeen 13 | * 14 | */ 15 | public class HyperGraphStageSixCombiner 16 | extends Reducer { 17 | 18 | @Override 19 | public void reduce(HVArray _key, Iterable values, 20 | Context context) throws IOException, InterruptedException { 21 | context.write(_key, NullWritable.get()); 22 | } 23 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/HyperVertexSignGroupComparator.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import org.apache.hadoop.io.WritableComparator; 4 | 5 | import dbg.hadoop.subgraphs.utils.Config; 6 | import dbg.hadoop.subgraphs.utils.HyperVertex; 7 | 8 | public class HyperVertexSignGroupComparator extends WritableComparator { 9 | 10 | protected HyperVertexSignGroupComparator() { 11 | super(HyperVertexSign.class); 12 | // TODO Auto-generated constructor stub 13 | } 14 | 15 | @Override 16 | public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2){ 17 | long v1 = readLong(b1, s1); 18 | long v2 = readLong(b2, s2); 19 | return HyperVertex.compare(v1, v2); 20 | } 21 | 22 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageSixMapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.io.NullWritable; 7 | import org.apache.hadoop.mapreduce.Mapper; 8 | 9 | import dbg.hadoop.subgraphs.io.HVArray; 10 | 11 | 12 | public class HyperGraphStageSixMapper 13 | extends Mapper { 14 | // The hypervertex set 15 | @Override 16 | public void map(LongWritable key, LongWritable value, Context context) 17 | throws IOException, InterruptedException{ 18 | context.write(new HVArray(key.get(), value.get()), NullWritable.get()); 19 | } 20 | } 21 | 22 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/HyperVertexComparator.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import org.apache.hadoop.io.LongWritable; 4 | import org.apache.hadoop.io.WritableComparator; 5 | 6 | import dbg.hadoop.subgraphs.utils.HyperVertex; 7 | 8 | public class HyperVertexComparator extends WritableComparator { 9 | 10 | protected HyperVertexComparator() { 11 | super(LongWritable.class); 12 | // TODO Auto-generated constructor stub 13 | } 14 | 15 | @Override 16 | public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2){ 17 | 18 | long cpr1 = readLong(b1, s1) & HyperVertex.COMP_MASK; 19 | long cpr2 = readLong(b2, s2) & HyperVertex.COMP_MASK; 20 | 21 | return (cpr1 == cpr2) ? 0 : ((cpr1 < cpr2) ? -1 : 1); 22 | } 23 | 24 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/maximalclique/MCliqueS3DegreeMapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.maximalclique; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.IntWritable; 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.mapreduce.Mapper; 8 | 9 | import dbg.hadoop.subgraphs.utils.HyperVertex; 10 | 11 | public class MCliqueS3DegreeMapper 12 | extends Mapper { 13 | //private static final LongWritable zero = new LongWritable(0); 14 | @Override 15 | public void map(IntWritable key, IntWritable value, Context context) 16 | throws IOException, InterruptedException{ 17 | LongWritable v = new LongWritable(HyperVertex.get(key.get(), value.get())); 18 | context.write(v, v); 19 | } 20 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/qtcounter/QtS3Reducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.qtcounter; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.io.NullWritable; 7 | import org.apache.hadoop.mapreduce.Reducer; 8 | 9 | 10 | public class QtS3Reducer 11 | extends Reducer { 12 | 13 | @Override 14 | public void reduce(NullWritable _key, Iterable values, 15 | Context context) throws IOException, InterruptedException { 16 | long counter = 0; 17 | for(LongWritable value : values){ 18 | counter += value.get(); 19 | } 20 | context.write(NullWritable.get(), new LongWritable(counter)); 21 | System.out.println("#cliques : " + counter); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/prepare/UndirectGraphReducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.prepare; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.IntWritable; 6 | import org.apache.hadoop.io.NullWritable; 7 | import org.apache.hadoop.io.Text; 8 | import org.apache.hadoop.mapreduce.Reducer; 9 | 10 | import dbg.hadoop.subgraphs.io.DoubleIntegerPairWritable; 11 | 12 | public class UndirectGraphReducer 13 | extends 14 | Reducer { 15 | @Override 16 | public void reduce(DoubleIntegerPairWritable _key, Iterable values, 17 | Context context) throws IOException, InterruptedException{ 18 | context.write(new IntWritable(_key.getFirst()), new IntWritable(_key.getSecond())); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/prepare/GenDegreeMapper2.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.prepare; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.IntWritable; 6 | import org.apache.hadoop.io.Text; 7 | import org.apache.hadoop.mapreduce.Mapper; 8 | 9 | /** 10 | * Get the statics of every node, specifically the degree 11 | * @author robeen 12 | * 13 | */ 14 | public class GenDegreeMapper2 extends Mapper { 15 | private final static IntWritable one = new IntWritable(1); 16 | @Override 17 | public void map(IntWritable key, IntWritable value, Context context) throws IOException, InterruptedException{ 18 | 19 | if(key.compareTo(value) != 0){ 20 | context.write(key, one); 21 | context.write(value, one); 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageFourMapper2.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.mapreduce.Mapper; 6 | 7 | import dbg.hadoop.subgraphs.io.HVArray; 8 | import dbg.hadoop.subgraphs.io.HyperVertexSign; 9 | 10 | /** 11 | * @author robeen 12 | * 13 | */ 14 | public class HyperGraphStageFourMapper2 15 | extends Mapper { 16 | // The hypervertex set 17 | @Override 18 | public void map(HyperVertexSign key, HVArray value, Context context) 19 | throws IOException, InterruptedException{ 20 | if(key.getSign() == 1){ 21 | context.write(new HyperVertexSign(key.getVertex(), -2), value); 22 | } 23 | else{ 24 | context.write(new HyperVertexSign(key.getVertex(), -3), value); 25 | } 26 | } 27 | } 28 | 29 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/DoubleIntegerPairComparator.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import org.apache.hadoop.io.WritableComparator; 4 | 5 | public class DoubleIntegerPairComparator extends WritableComparator { 6 | 7 | protected DoubleIntegerPairComparator() { 8 | super(DoubleIntegerPairWritable.class); 9 | // TODO Auto-generated constructor stub 10 | } 11 | 12 | @Override 13 | public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2){ 14 | int cmp = 0; 15 | int i1 = readInt(b1, s1); 16 | int i2 = readInt(b2, s2); 17 | cmp = (i1 < i2) ? -1 : ((i1 == i2) ? 0: 1); 18 | if(0 != cmp){ 19 | return cmp; 20 | } 21 | 22 | int j1 = readInt(b1, s1 + Integer.SIZE / Byte.SIZE); 23 | int j2 = readInt(b2, s2 + Integer.SIZE / Byte.SIZE); 24 | 25 | cmp = (j1 < j2) ? -1 : ((j1 == j2) ? 0: 1); 26 | return cmp; 27 | } 28 | 29 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/frame/GeneralPatternCountReducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.frame; 2 | 3 | import java.io.IOException; 4 | import org.apache.hadoop.io.LongWritable; 5 | import org.apache.hadoop.io.NullWritable; 6 | import org.apache.hadoop.mapreduce.Reducer; 7 | import org.apache.log4j.Logger; 8 | 9 | 10 | public class GeneralPatternCountReducer extends 11 | Reducer { 12 | private static Logger log = Logger.getLogger(GeneralPatternCountReducer.class); 13 | @Override 14 | public void reduce(NullWritable _key, Iterable values, 15 | Context context) throws IOException, InterruptedException { 16 | long sum = 0L; 17 | for(LongWritable val : values){ 18 | sum += val.get(); 19 | } 20 | log.info("# pattern graph: " + sum); 21 | context.write(NullWritable.get(), new LongWritable(sum)); 22 | } 23 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageFourMapper1.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.mapreduce.Mapper; 7 | 8 | import dbg.hadoop.subgraphs.io.HVArray; 9 | import dbg.hadoop.subgraphs.io.HyperVertexAdjList; 10 | import dbg.hadoop.subgraphs.io.HyperVertexSign; 11 | 12 | /** 13 | * @author robeen 14 | * 15 | */ 16 | public class HyperGraphStageFourMapper1 17 | extends 18 | Mapper { 19 | @Override 20 | public void map(LongWritable key, HyperVertexAdjList adjList, Context context) 21 | throws IOException, InterruptedException{ 22 | long largeThanThis[] = adjList.getLargeDegreeVertices(); 23 | context.write(new HyperVertexSign(key.get(), -1), new HVArray(largeThanThis)); 24 | } 25 | } 26 | 27 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/maximalclique/MCliqueS3Reducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.maximalclique; 2 | 3 | import java.io.IOException; 4 | import org.apache.hadoop.io.LongWritable; 5 | import org.apache.hadoop.mapreduce.Reducer; 6 | 7 | import dbg.hadoop.subgraphs.utils.HyperVertex; 8 | 9 | 10 | public class MCliqueS3Reducer 11 | extends Reducer { 12 | 13 | @Override 14 | public void reduce(LongWritable _key, Iterable values, 15 | Context context) throws IOException, InterruptedException { 16 | long largestClique = 0; 17 | // The vertex will belong to the largest clique 18 | for(LongWritable value : values){ 19 | if(HyperVertex.Size(largestClique) < HyperVertex.Size(value.get())){ 20 | largestClique = value.get(); 21 | } 22 | } 23 | context.write(_key, new LongWritable(largestClique)); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/prepare/PrepareDataS1Mapper2.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.prepare; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.IntWritable; 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.io.Text; 8 | import org.apache.hadoop.mapreduce.Mapper; 9 | 10 | import dbg.hadoop.subgraphs.io.DoubleIntegerPairWritable; 11 | 12 | 13 | /** 14 | * Get the statics of every node, specifically the degree 15 | * @author robeen 16 | * 17 | **/ 18 | 19 | public class PrepareDataS1Mapper2 20 | extends Mapper { 21 | 22 | @Override 23 | public void map(IntWritable key, IntWritable value, Context context) throws IOException, InterruptedException{ 24 | context.write(new DoubleIntegerPairWritable(key.get(), -1), 25 | new LongWritable(Long.valueOf(value.get()))); 26 | } 27 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageSixReducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.io.NullWritable; 7 | import org.apache.hadoop.mapreduce.Reducer; 8 | 9 | import dbg.hadoop.subgraphs.io.HVArray; 10 | 11 | /** 12 | * Merge duplicated results. 13 | * @author robeen 14 | * 15 | */ 16 | public class HyperGraphStageSixReducer 17 | extends Reducer { 18 | 19 | @Override 20 | public void reduce(HVArray _key, Iterable values, 21 | Context context) throws IOException, InterruptedException { 22 | context.write(new LongWritable(_key.getFirst()), new LongWritable(_key.getSecond())); 23 | //System.out.println(HyperVertex.toString(_key.getFirst()) + ";" + 24 | // HyperVertex.toString(_key.getSecond())); 25 | } 26 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/prepare/PrepareDataDegreeMapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.prepare; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.IntWritable; 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.mapreduce.Mapper; 8 | 9 | import dbg.hadoop.subgraphs.io.DoubleIntegerPairWritable; 10 | 11 | 12 | /** 13 | * Get the statics of every node, specifically the degree 14 | * @author robeen 15 | * 16 | **/ 17 | 18 | public class PrepareDataDegreeMapper 19 | extends Mapper { 20 | 21 | @Override 22 | public void map(IntWritable key, IntWritable value, Context context) throws IOException, InterruptedException{ 23 | //OutputCollector output, Reporter reporter) throws IOException { 24 | context.write(new DoubleIntegerPairWritable(key.get(), -2), new LongWritable(value.get())); 25 | } 26 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/prepare/PrepareDataS2Mapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.prepare; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.IntWritable; 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.io.Text; 8 | import org.apache.hadoop.mapreduce.Mapper; 9 | 10 | import dbg.hadoop.subgraphs.io.DoubleIntegerPairWritable; 11 | 12 | 13 | /** 14 | * Get the statics of every node, specifically the degree 15 | * @author robeen 16 | * 17 | **/ 18 | 19 | public class PrepareDataS2Mapper 20 | extends Mapper { 21 | 22 | @Override 23 | public void map(LongWritable key, IntWritable value, Context context) 24 | throws IOException, InterruptedException{ 25 | //OutputCollector output, Reporter reporter) throws IOException { 26 | context.write(new DoubleIntegerPairWritable(value.get(), -1), key); 27 | } 28 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/list/TLinkableAdapter.java: -------------------------------------------------------------------------------- 1 | package gnu.trove.list; 2 | 3 | /** 4 | * Simple adapter class implementing {@link TLinkable}, so you don't have to. Example: 5 | * 
 6 | 	private class MyObject extends TLinkableAdapter {
 7 | 		private final String value;
 8 | 
 9 | 		MyObject( String value ) {
10 | 			this.value = value;
11 | 		}
12 | 		
13 | 		public String getValue() {
14 | 			return value;
15 | 		}
16 | 	}
17 |  *
18 | */ 19 | public abstract class TLinkableAdapter implements TLinkable { 20 | private volatile T next; 21 | private volatile T prev; 22 | 23 | @Override 24 | public T getNext() { 25 | return next; 26 | } 27 | 28 | @Override 29 | public void setNext( T next ) { 30 | this.next = next; 31 | } 32 | 33 | @Override 34 | public T getPrevious() { 35 | return prev; 36 | } 37 | 38 | @Override 39 | public void setPrevious( T prev ) { 40 | this.prev = prev; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/adjlist/GenAdjListMapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph.adjlist; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.mapreduce.Mapper; 7 | 8 | import dbg.hadoop.subgraphs.utils.HyperVertex; 9 | 10 | /** 11 | * Get the statics of every node, specifically the degree 12 | * @author robeen 13 | * 14 | */ 15 | public class GenAdjListMapper 16 | extends Mapper { 17 | 18 | @Override 19 | public void map(LongWritable key, LongWritable value, Context context) throws IOException, InterruptedException{ 20 | //OutputCollector output, Reporter reporter) throws IOException { 21 | if(HyperVertex.Degree(key.get()) > 1){ 22 | context.write(key, value); 23 | } 24 | if(value.get() != -1 && key.get() != value.get()){ 25 | context.write(value, key); 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/maximalclique/EdgeMapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.maximalclique; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.mapreduce.Mapper; 7 | 8 | import dbg.hadoop.subgraphs.io.HVArraySign; 9 | import dbg.hadoop.subgraphs.utils.Config; 10 | 11 | /** 12 | * Get the statics of every node, specifically the degree 13 | * @author robeen 14 | * 15 | */ 16 | // public class InitMapper extends MapReduceBase implements 17 | public class EdgeMapper 18 | extends Mapper { 19 | 20 | @Override 21 | public void map(LongWritable key, LongWritable value, Context context) 22 | throws IOException, InterruptedException{ 23 | //OutputCollector output, Reporter reporter) throws IOException { 24 | context.write(new HVArraySign(key.get(), value.get(), Config.SMALLSIGN), new LongWritable()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/HyperVertexSignComparator.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import org.apache.hadoop.io.WritableComparator; 4 | 5 | import dbg.hadoop.subgraphs.utils.Config; 6 | import dbg.hadoop.subgraphs.utils.HyperVertex; 7 | 8 | public class HyperVertexSignComparator extends WritableComparator { 9 | 10 | protected HyperVertexSignComparator() { 11 | super(HyperVertexSign.class); 12 | // TODO Auto-generated constructor stub 13 | } 14 | 15 | @Override 16 | public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2){ 17 | int cmp = 0; 18 | long v1 = readLong(b1, s1); 19 | long v2 = readLong(b2, s2); 20 | cmp = HyperVertex.compare(v1, v2); 21 | if(0 != cmp){ 22 | return cmp; 23 | } 24 | 25 | int sign1 = readInt(b1, s1 + Config.NUMLONGBITS); 26 | int sign2 = readInt(b2, s2 + Config.NUMLONGBITS); 27 | 28 | cmp = (sign1 < sign2) ? -1 : ((sign1 == sign2) ? 0: 1); 29 | return cmp; 30 | } 31 | 32 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/maximalclique/MCliqueS5Reducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.maximalclique; 2 | 3 | import gnu.trove.set.hash.TLongHashSet; 4 | 5 | import java.io.IOException; 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.mapreduce.Reducer; 8 | 9 | import dbg.hadoop.subgraphs.utils.HyperVertex; 10 | 11 | 12 | public class MCliqueS5Reducer 13 | extends Reducer { 14 | 15 | @Override 16 | public void reduce(LongWritable _key, Iterable values, 17 | Context context) throws IOException, InterruptedException { 18 | TLongHashSet set = new TLongHashSet(); 19 | for(LongWritable v : values){ 20 | if(!set.contains(v.get())){ 21 | set.add(v.get()); 22 | context.write(_key, v); 23 | //System.out.println(HyperVertex.toString(_key.get()) + "\t" + HyperVertex.toString(v.get())); 24 | } 25 | } 26 | set.clear(); 27 | set = null; 28 | } 29 | 30 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | fs.defaultFS 22 | hdfs://localhost:9000 23 | 24 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/prepare/PrepareDataS1Mapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.prepare; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.io.Text; 7 | import org.apache.hadoop.mapreduce.Mapper; 8 | 9 | import dbg.hadoop.subgraphs.io.DoubleIntegerPairWritable; 10 | 11 | 12 | /** 13 | * Get the statics of every node, specifically the degree 14 | * @author robeen 15 | * 16 | **/ 17 | 18 | public class PrepareDataS1Mapper 19 | extends Mapper { 20 | 21 | @Override 22 | public void map(Text key, Text value, Context context) throws IOException, InterruptedException{ 23 | if(key.toString().startsWith("#") || key.toString().startsWith("%") || 24 | key.compareTo(value) == 0){ 25 | return; 26 | } 27 | context.write(new DoubleIntegerPairWritable(Integer.valueOf(key.toString()), -1), 28 | new LongWritable(Long.valueOf(value.toString()))); 29 | } 30 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/triangle/EdgeMapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph.triangle; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.mapreduce.Mapper; 7 | 8 | import dbg.hadoop.subgraphs.io.HVArray; 9 | import dbg.hadoop.subgraphs.io.HVArraySign; 10 | import dbg.hadoop.subgraphs.utils.Config; 11 | 12 | /** 13 | * Get the statics of every node, specifically the degree 14 | * @author robeen 15 | * 16 | */ 17 | // public class InitMapper extends MapReduceBase implements 18 | public class EdgeMapper 19 | extends Mapper { 20 | 21 | @Override 22 | public void map(LongWritable key, LongWritable value, Context context) throws IOException, InterruptedException{ 23 | //OutputCollector output, Reporter reporter) throws IOException { 24 | context.write(new HVArraySign(key.get(), value.get(), Config.SMALLSIGN), new HVArray()); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/qtcounter/EdgeMapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.qtcounter; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.mapreduce.Mapper; 7 | 8 | import dbg.hadoop.subgraphs.io.HVArray; 9 | import dbg.hadoop.subgraphs.io.HVArraySign; 10 | import dbg.hadoop.subgraphs.utils.Config; 11 | 12 | /** 13 | * Get the statics of every node, specifically the degree 14 | * @author robeen 15 | * 16 | */ 17 | // public class InitMapper extends MapReduceBase implements 18 | public class EdgeMapper 19 | extends Mapper { 20 | 21 | @Override 22 | public void map(LongWritable key, LongWritable value, Context context) 23 | throws IOException, InterruptedException{ 24 | //OutputCollector output, Reporter reporter) throws IOException { 25 | context.write(new HVArraySign(key.get(), value.get(), Config.SMALLSIGN), new LongWritable()); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageFiveMapper2.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.mapreduce.Mapper; 7 | 8 | import dbg.hadoop.subgraphs.io.HVArray; 9 | import dbg.hadoop.subgraphs.io.HyperVertexSign; 10 | 11 | /** 12 | * This mapper deals with the output from stage three.
13 | * Input: (hid: hyperVertexSet),
14 | * Output: foreach v in hyperVertexSet, output ((v, -2); hid) 15 | * @author robeen 16 | * 17 | */ 18 | public class HyperGraphStageFiveMapper2 19 | extends Mapper { 20 | // The hypervertex set 21 | @Override 22 | public void map(HyperVertexSign key, HVArray value, Context context) 23 | throws IOException, InterruptedException{ 24 | for(long u : value.toArrays()){ 25 | context.write(new HyperVertexSign(u, -2), new LongWritable(key.getVertex())); 26 | } 27 | } 28 | } 29 | 30 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/qtcounter/QtS1Reducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.qtcounter; 2 | 3 | import gnu.trove.list.array.TLongArrayList; 4 | 5 | import java.io.IOException; 6 | 7 | import org.apache.hadoop.io.LongWritable; 8 | import org.apache.hadoop.mapreduce.Reducer; 9 | 10 | import dbg.hadoop.subgraphs.io.HVArraySign; 11 | import dbg.hadoop.subgraphs.io.HVArray; 12 | import dbg.hadoop.subgraphs.utils.Config; 13 | 14 | 15 | public class QtS1Reducer 16 | extends Reducer { 17 | 18 | @Override 19 | public void reduce(HVArraySign _key, Iterable values, 20 | Context context) throws IOException, InterruptedException { 21 | if(_key.sign != Config.SMALLSIGN){ 22 | return; 23 | } 24 | TLongArrayList list = new TLongArrayList(); 25 | for(LongWritable v : values){ 26 | if(_key.sign != Config.SMALLSIGN){ 27 | list.add(v.get()); 28 | } 29 | } 30 | context.write(_key.vertexArray, new HVArray(list.toArray())); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/maximalclique/MCliqueS1Reducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.maximalclique; 2 | 3 | import gnu.trove.list.array.TLongArrayList; 4 | 5 | import java.io.IOException; 6 | 7 | import org.apache.hadoop.io.LongWritable; 8 | import org.apache.hadoop.mapreduce.Reducer; 9 | 10 | import dbg.hadoop.subgraphs.io.HVArray; 11 | import dbg.hadoop.subgraphs.io.HVArraySign; 12 | import dbg.hadoop.subgraphs.utils.Config; 13 | 14 | 15 | 16 | class MCliqueS1Reducer extends 17 | Reducer { 18 | 19 | @Override 20 | public void reduce(HVArraySign _key, Iterable values, 21 | Context context) throws IOException, InterruptedException { 22 | if (_key.sign != Config.SMALLSIGN) { 23 | return; 24 | } 25 | TLongArrayList list = new TLongArrayList(); 26 | for (LongWritable v : values) { 27 | if (_key.sign != Config.SMALLSIGN) { 28 | list.add(v.get()); 29 | } 30 | } 31 | context.write(_key.vertexArray, new HVArray(list.toArray())); 32 | } 33 | 34 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/IntegerPairComparator.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import org.apache.hadoop.io.WritableComparator; 4 | 5 | public class IntegerPairComparator extends WritableComparator { 6 | 7 | protected IntegerPairComparator() { 8 | super(IntegerPairWritable.class); 9 | // TODO Auto-generated constructor stub 10 | } 11 | 12 | @Override 13 | public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { 14 | int cmp = 0; 15 | int value1 = readInt(b1, s1); 16 | int value2 = readInt(b2, s2); // The first int is the sizes 17 | cmp = (value1 == value2)? 0 : ((value1 < value2)? -1 : 1); 18 | if(cmp != 0){ 19 | return cmp; 20 | } 21 | int size = value1; 22 | for(int i = 1; i <= size; ++i){ 23 | value1 = readInt(b1, s1 + i * Integer.SIZE / Byte.SIZE); 24 | value2 = readInt(b2, s2 + i * Integer.SIZE / Byte.SIZE); 25 | cmp = (value1 < value2) ? -1 : ((value1 == value2) ? 0: 1); 26 | if(0 != cmp){ 27 | return cmp; 28 | } 29 | } 30 | return cmp; 31 | } 32 | 33 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/prepare/GenDegreeMapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.prepare; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.IntWritable; 6 | import org.apache.hadoop.io.Text; 7 | import org.apache.hadoop.mapreduce.Mapper; 8 | 9 | /** 10 | * Get the statics of every node, specifically the degree 11 | * @author robeen 12 | * 13 | */ 14 | public class GenDegreeMapper extends Mapper { 15 | private final static IntWritable one = new IntWritable(1); 16 | @Override 17 | public void map(Text key, Text value, Context context) throws IOException, InterruptedException{ 18 | //OutputCollector output, Reporter reporter) throws IOException { 19 | if(key.toString().startsWith("#") || key.toString().startsWith("%")){ 20 | return; 21 | } 22 | if(key.compareTo(value) != 0){ 23 | context.write(new IntWritable(Integer.valueOf(key.toString())), one); 24 | context.write(new IntWritable(Integer.valueOf(value.toString())), one); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/prepare/PrepareDataS1Reducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.prepare; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.IntWritable; 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.mapreduce.Reducer; 8 | 9 | import dbg.hadoop.subgraphs.io.DoubleIntegerPairWritable; 10 | import dbg.hadoop.subgraphs.utils.HyperVertex; 11 | 12 | public class PrepareDataS1Reducer extends Reducer { 14 | @Override 15 | public void reduce(DoubleIntegerPairWritable _key, Iterable values, 16 | Context context) throws IOException, InterruptedException{ 17 | if(_key.getSecond() != -2){ 18 | return; 19 | } 20 | long vertex = 0; 21 | for(LongWritable v : values) { 22 | if(_key.getSecond() == -2){ 23 | // Encapsulate the vertex id + degree 24 | vertex = HyperVertex.get(_key.getFirst(), (int)v.get()); 25 | continue; 26 | } 27 | context.write(new LongWritable(vertex), new IntWritable((int)v.get())); 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/utils/bloomfilter/io/IAllocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package dbg.hadoop.subgraphs.utils.bloomfilter.io; 19 | 20 | public interface IAllocator 21 | { 22 | long allocate(long size); 23 | void free(long peer); 24 | } 25 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/prepare/PrepareDataS2Reducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.prepare; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.mapreduce.Reducer; 7 | 8 | import dbg.hadoop.subgraphs.io.DoubleIntegerPairWritable; 9 | import dbg.hadoop.subgraphs.utils.HyperVertex; 10 | 11 | public class PrepareDataS2Reducer extends Reducer { 13 | @Override 14 | public void reduce(DoubleIntegerPairWritable _key, Iterable values, 15 | Context context) throws IOException, InterruptedException{ 16 | if(_key.getSecond() != -2){ 17 | return; 18 | } 19 | long vertex = 0; 20 | for(LongWritable v : values) { 21 | if(_key.getSecond() == -2){ 22 | // Encapsulate the vertex id + degree 23 | vertex = HyperVertex.get(_key.getFirst(), (int)v.get()); 24 | continue; 25 | } 26 | if(HyperVertex.compare(vertex, v.get()) < 0){ 27 | context.write(new LongWritable(vertex), v); 28 | } 29 | else{ 30 | context.write(v, new LongWritable(vertex)); 31 | } 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/ColorVertexPartitioner.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import org.apache.hadoop.conf.Configurable; 4 | import org.apache.hadoop.conf.Configuration; 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.io.Writable; 7 | import org.apache.hadoop.mapreduce.Partitioner; 8 | 9 | import dbg.hadoop.subgraphs.utils.HyperVertex; 10 | 11 | public class ColorVertexPartitioner 12 | extends Partitioner 13 | implements Configurable{ 14 | private Configuration conf; 15 | 16 | @Override 17 | public int getPartition(LongWritable key, Writable value, 18 | int numPartitions) { 19 | // TODO Auto-generated method stub 20 | int thresh = conf.getInt("mapred.large.degree.threshold", Integer.MAX_VALUE); 21 | return HyperVertex.Degree(key.get()) < thresh ? HyperVertex.Color(key.get()) : 0; 22 | } 23 | 24 | @Override 25 | public Configuration getConf() { 26 | // TODO Auto-generated method stub 27 | return conf; 28 | } 29 | 30 | @Override 31 | public void setConf(Configuration _conf) { 32 | // TODO Auto-generated method stub 33 | this.conf = _conf; 34 | } 35 | 36 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/HVArrayComparator.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import org.apache.hadoop.io.WritableComparator; 4 | 5 | import dbg.hadoop.subgraphs.utils.Config; 6 | import dbg.hadoop.subgraphs.utils.HyperVertex; 7 | 8 | public class HVArrayComparator extends WritableComparator { 9 | 10 | protected HVArrayComparator() { 11 | super(HVArray.class); 12 | // TODO Auto-generated constructor stub 13 | } 14 | 15 | @Override 16 | public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2){ 17 | int cmp = 0; 18 | int len1 = readInt(b1, s1); 19 | int len2 = readInt(b2, s2); 20 | cmp = (len1 == len2)? 0 : ((len1 < len2)? -1 : 1); 21 | if(cmp != 0){ 22 | return cmp; 23 | } 24 | long cpr1 = 0L, cpr2 = 0L; 25 | for(int i = 0; i < len1; ++i){ 26 | cpr1 = readLong(b1, s1 + Config.NUMINTBITS + i * Config.NUMLONGBITS) & HyperVertex.COMP_MASK; 27 | cpr2 = readLong(b2, s2 + Config.NUMINTBITS + i * Config.NUMLONGBITS) & HyperVertex.COMP_MASK; 28 | cmp = (cpr1 < cpr2) ? -1 : ((cpr1 == cpr2) ? 0: 1); 29 | if(0 != cmp){ 30 | return cmp; 31 | } 32 | } 33 | return cmp; 34 | } 35 | 36 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageTwoMapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.mapreduce.Mapper; 7 | 8 | import dbg.hadoop.subgraphs.io.HVArray; 9 | import dbg.hadoop.subgraphs.io.HyperVertexAdjList; 10 | 11 | /** 12 | * @author robeen 13 | * 14 | */ 15 | public class HyperGraphStageTwoMapper 16 | extends 17 | Mapper { 18 | 19 | private static int thresh = 20; 20 | @Override 21 | public void map(LongWritable key, HyperVertexAdjList adjList, Context context) 22 | throws IOException, InterruptedException{ 23 | long largeThanThis[] = adjList.getLargeDegreeVertices(); 24 | long smallThanThis[] = adjList.getSmallDegreeVerticesGroup1(); 25 | 26 | if(largeThanThis.length + smallThanThis.length <= thresh){ 27 | context.write(new HVArray(smallThanThis, largeThanThis), key); 28 | } 29 | } 30 | 31 | @Override 32 | public void setup(Context context){ 33 | thresh = context.getConfiguration().getInt("mapred.hypergraph.threshold", 20); 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/qtcounter/QtS2Reducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.qtcounter; 2 | 3 | import gnu.trove.set.hash.TLongHashSet; 4 | 5 | import java.io.IOException; 6 | import java.util.Collection; 7 | import java.util.Iterator; 8 | 9 | import org.apache.hadoop.io.LongWritable; 10 | import org.apache.hadoop.mapreduce.Reducer; 11 | 12 | import dbg.hadoop.subgraphs.io.HVArray; 13 | import dbg.hadoop.subgraphs.utils.BronKerboschCliqueFinder; 14 | import dbg.hadoop.subgraphs.utils.Graph; 15 | 16 | 17 | public class QtS2Reducer 18 | extends Reducer { 19 | 20 | private static int cliqueSize = 4; 21 | @Override 22 | public void reduce(LongWritable _key, Iterable values, 23 | Context context) throws IOException, InterruptedException { 24 | Graph graph = new Graph(); 25 | for(HVArray vPair : values){ 26 | graph.addEdge(vPair.getFirst(), vPair.getSecond()); 27 | } 28 | long counter = graph.countCliquesOfSize(cliqueSize - 1); 29 | context.write(_key, new LongWritable(counter)); 30 | graph = null; 31 | } 32 | 33 | @Override 34 | public void setup(Context context){ 35 | cliqueSize = context.getConfiguration().getInt("mapred.clique.size", 4); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/utils/HyperGraphTest.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.utils; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.File; 5 | import java.io.FileReader; 6 | import java.io.IOException; 7 | 8 | 9 | public class HyperGraphTest { 10 | public static void main(String[] args) throws IOException{ 11 | if(args.length != 2){ 12 | System.out.println("HyperGraphTest "); 13 | System.exit(-1); 14 | } 15 | String graphFile = args[0]; 16 | String delim = args[1]; 17 | String line = ""; 18 | BufferedReader reader = new BufferedReader(new FileReader(new File(graphFile))); 19 | Graph g = new Graph(); 20 | while((line = reader.readLine()) != null){ 21 | if(line.startsWith("#")){ 22 | continue; 23 | } 24 | String[] s = line.split(delim); 25 | if(s.length != 2 || Long.valueOf(s[0]) == Long.valueOf(s[1])){ 26 | System.err.println("Format error: " + line); 27 | continue; 28 | } 29 | g.addEdge(Long.valueOf(s[0]), Long.valueOf(s[1])); 30 | } 31 | reader.close(); 32 | 33 | long begin = System.currentTimeMillis(); 34 | g.genHyperGraph(); 35 | long end = System.currentTimeMillis(); 36 | System.out.println("Time elapsed: " + (end - begin) + " ms"); 37 | } 38 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/utils/bloomfilter/utils/IFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package dbg.hadoop.subgraphs.utils.bloomfilter.utils; 19 | 20 | import java.io.Closeable; 21 | import java.nio.ByteBuffer; 22 | 23 | public interface IFilter extends Closeable 24 | { 25 | void add(ByteBuffer key); 26 | 27 | boolean isPresent(ByteBuffer key); 28 | 29 | void clear(); 30 | 31 | long serializedSize(); 32 | } 33 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/hdfs-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | dfs.replication 22 | 1 23 | 24 | 25 | dfs.namenode.name.dir 26 | file:///home/robeen/hdfs2/dfs/name 27 | 28 | 29 | 30 | dfs.datanode.data.dir 31 | file:///home/robeen/hdfs2/dfs/data 32 | 33 | 34 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/HVArrayGroupComparator.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import org.apache.hadoop.io.WritableComparator; 4 | 5 | import dbg.hadoop.subgraphs.utils.HyperVertex; 6 | import dbg.hadoop.subgraphs.utils.Config; 7 | 8 | /** 9 | * We simply use the hyperVertexArrayComparator by ingnoring the sign. 10 | * @author robeen 11 | * 12 | */ 13 | public class HVArrayGroupComparator extends WritableComparator { 14 | 15 | protected HVArrayGroupComparator() { 16 | super(HVArraySign.class); 17 | // TODO Auto-generated constructor stub 18 | } 19 | 20 | @Override 21 | public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2){ 22 | int cmp = 0; 23 | int len1 = readInt(b1, s1); 24 | int len2 = readInt(b2, s2); 25 | cmp = (len1 == len2)? 0 : ((len1 < len2)? -1 : 1); 26 | if(cmp != 0){ 27 | return cmp; 28 | } 29 | long cpr1 = 0L, cpr2 = 0L; 30 | for(int i = 0; i < len1; ++i){ 31 | cpr1 = readLong(b1, s1 + Config.NUMINTBITS + i * Config.NUMLONGBITS) & HyperVertex.COMP_MASK; 32 | cpr2 = readLong(b2, s2 + Config.NUMINTBITS + i * Config.NUMLONGBITS) & HyperVertex.COMP_MASK; 33 | cmp = (cpr1 < cpr2) ? -1 : ((cpr1 == cpr2) ? 0: 1); 34 | if(0 != cmp){ 35 | return cmp; 36 | } 37 | } 38 | return cmp; 39 | } 40 | 41 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageOneMapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.mapreduce.Mapper; 7 | 8 | import dbg.hadoop.subgraphs.io.HVArray; 9 | import dbg.hadoop.subgraphs.io.HyperVertexAdjList; 10 | 11 | /** 12 | 13 | * @author robeen 14 | * 15 | */ 16 | // public class InitMapper extends MapReduceBase implements 17 | public class HyperGraphStageOneMapper 18 | extends Mapper { 19 | 20 | private static int thresh = 20; 21 | @Override 22 | public void map(LongWritable key, HyperVertexAdjList adjList, Context context) 23 | throws IOException, InterruptedException{ 24 | long largeThanThis[] = adjList.getLargeDegreeVertices(); 25 | long smallThanThis[] = adjList.getSmallDegreeVerticesGroup1(); 26 | 27 | if(largeThanThis.length + smallThanThis.length > thresh){ 28 | context.write(new HVArray(key.get()), key); 29 | } 30 | else{ 31 | context.write(new HVArray(smallThanThis, key.get(), largeThanThis), key); 32 | } 33 | } 34 | 35 | @Override 36 | public void setup(Context context){ 37 | thresh = context.getConfiguration().getInt("mapred.hypergraph.threshold", 20); 38 | } 39 | } 40 | 41 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/prepare/UndirectGraphMapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.prepare; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.NullWritable; 6 | import org.apache.hadoop.io.Text; 7 | import org.apache.hadoop.mapreduce.Mapper; 8 | 9 | import dbg.hadoop.subgraphs.io.DoubleIntegerPairWritable; 10 | 11 | /** 12 | * Generate undrirect graph 13 | * @author robeen 14 | * 15 | */ 16 | public class UndirectGraphMapper extends Mapper { 17 | @Override 18 | public void map(Text _key, Text _value, Context context) throws IOException, InterruptedException{ 19 | //OutputCollector output, Reporter reporter) throws IOException { 20 | // degree of key is larger than value 21 | // We deprecate larger degree vertex 22 | if(_key.toString().startsWith("#") || _key.toString().startsWith("%") 23 | || _key.compareTo(_value) == 0){ 24 | return; 25 | } 26 | 27 | int keyInt = Integer.valueOf(_key.toString().trim()); 28 | int valueInt = Integer.valueOf(_value.toString().trim()); 29 | 30 | if(keyInt < valueInt){ 31 | context.write(new DoubleIntegerPairWritable(keyInt, valueInt), NullWritable.get()); 32 | } 33 | else{ 34 | context.write(new DoubleIntegerPairWritable(valueInt, keyInt), NullWritable.get()); 35 | } 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/impl/package.html: -------------------------------------------------------------------------------- 1 | 20 | 21 | This package (and its sub-packages) contain internal implementations used in Trove. These 22 | classes should not be accessed directly (treat them like com.sun 23 | classes. 24 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageFiveMapper1.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.mapreduce.Mapper; 7 | 8 | import dbg.hadoop.subgraphs.io.HVArray; 9 | import dbg.hadoop.subgraphs.io.HyperVertexSign; 10 | import dbg.hadoop.subgraphs.utils.HyperVertex; 11 | 12 | /** 13 | * This mapper deals with the output from stage four.
14 | * Input: (hid: true_neighbor), true neighbor stands for those vertices which are neighbors of
15 | * current hypervertex and does not belong to the hyperVertex.
16 | * Output: foreach v in true_neighbor, output ((v, -1); u) 17 | * @author robeen 18 | * 19 | */ 20 | public class HyperGraphStageFiveMapper1 21 | extends 22 | Mapper { 23 | // The hypervertex set 24 | @Override 25 | public void map(LongWritable key, HVArray value, Context context) 26 | throws IOException, InterruptedException{ 27 | if(value.size() == 0 && HyperVertex.isClique(key.get())){ 28 | long u = HyperVertex.get(HyperVertex.VertexID(key.get()), 29 | HyperVertex.Degree(key.get())); 30 | context.write(new HyperVertexSign(u, -1), key); 31 | } 32 | 33 | for(long u : value.toArrays()){ 34 | context.write(new HyperVertexSign(u, -1), key); 35 | } 36 | } 37 | } 38 | 39 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/utils/bloomfilter/utils/AlwaysPresentFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package dbg.hadoop.subgraphs.utils.bloomfilter.utils; 19 | 20 | import java.io.IOException; 21 | import java.nio.ByteBuffer; 22 | 23 | public class AlwaysPresentFilter implements IFilter 24 | { 25 | public boolean isPresent(ByteBuffer key) 26 | { 27 | return true; 28 | } 29 | 30 | public void add(ByteBuffer key) { } 31 | 32 | public void clear() { } 33 | 34 | public void close() throws IOException { } 35 | 36 | public long serializedSize() { return 0; } 37 | } 38 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageTwoReducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | 4 | import java.io.IOException; 5 | 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.mapreduce.Reducer; 8 | 9 | import dbg.hadoop.subgraphs.io.HVArray; 10 | import dbg.hadoop.subgraphs.io.HyperVertexSign; 11 | import dbg.hadoop.subgraphs.utils.Config; 12 | import dbg.hadoop.subgraphs.utils.HyperVertexHeap; 13 | 14 | public class HyperGraphStageTwoReducer extends Reducer 15 | { 16 | 17 | public void reduce(HVArray _key, Iterable values, 18 | Context context) throws IOException, InterruptedException { 19 | 20 | int size = 0; 21 | HyperVertexHeap vertexHeap = new HyperVertexHeap(Config.HEAPINITSIZE); 22 | for(LongWritable value: values){ 23 | ++size; 24 | vertexHeap.insert(value.get()); 25 | } 26 | vertexHeap.sort(); 27 | 28 | // Means we donot output the adjList, the vertex should belong to itself. 29 | // We donot need to check it anymore 30 | if(size == 1){ 31 | context.write(new HyperVertexSign(vertexHeap.getFirst(), 2), 32 | new HVArray(vertexHeap.getFirst())); 33 | } 34 | // We ouptut the _key.getFirst as the hypervertex id as it is the smallest vertex 35 | // in the set. 36 | else{ 37 | context.write(new HyperVertexSign(vertexHeap.getFirst(), 0), 38 | new HVArray(vertexHeap.toArrays())); 39 | } 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageFiveReducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.mapreduce.Reducer; 7 | 8 | import dbg.hadoop.subgraphs.utils.HyperVertex; 9 | import dbg.hadoop.subgraphs.io.HyperVertexSign; 10 | 11 | /** 12 | * Two different inputs will be received.
13 | * 1. ((u, -1); hid): The hyperVertex which is adjacent to u;
14 | * 2. ((u, -2); hid(u)): u's hyperverte id.
15 | * The second input are guaranteed to come first.
16 | * In this case, we know that hid and hid(u) are connected.
17 | * So we output (hid; hid(u)) or (hid(u); hid), the smaller hid comes first 18 | * @author robeen 19 | * 20 | */ 21 | public class HyperGraphStageFiveReducer 22 | extends Reducer { 23 | 24 | @Override 25 | public void reduce(HyperVertexSign _key, Iterable values, 26 | Context context) throws IOException, InterruptedException { 27 | boolean firstRecord = true; 28 | LongWritable curV = null; 29 | for(LongWritable v : values){ 30 | if(firstRecord){ 31 | curV = new LongWritable(v.get()); 32 | firstRecord = false; 33 | } 34 | else{ 35 | if(HyperVertex.compare(v.get(), curV.get()) < 0){ 36 | context.write(v, curV); 37 | } 38 | else{ 39 | context.write(curV, v); 40 | } 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/HVArraySignComparator.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import org.apache.hadoop.io.WritableComparator; 4 | 5 | import dbg.hadoop.subgraphs.utils.Config; 6 | import dbg.hadoop.subgraphs.utils.HyperVertex; 7 | 8 | /** 9 | * We simply use the hyperVertexArrayComparator by ingnoring the sign. 10 | * @author robeen 11 | * 12 | */ 13 | public class HVArraySignComparator extends WritableComparator { 14 | 15 | protected HVArraySignComparator() { 16 | super(HVArraySign.class); 17 | // TODO Auto-generated constructor stub 18 | } 19 | 20 | @Override 21 | public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2){ 22 | int cmp = 0; 23 | int len1 = readInt(b1, s1); 24 | int len2 = readInt(b2, s2); 25 | cmp = (len1 == len2)? 0 : ((len1 < len2)? -1 : 1); 26 | if(cmp != 0){ 27 | return cmp; 28 | } 29 | long cpr1 = 0L, cpr2 = 0L; 30 | for(int i = 0; i < len1; ++i){ 31 | cpr1 = readLong(b1, s1 + Config.NUMINTBITS + i * Config.NUMLONGBITS) & HyperVertex.COMP_MASK; 32 | cpr2 = readLong(b2, s2 + Config.NUMINTBITS + i * Config.NUMLONGBITS) & HyperVertex.COMP_MASK; 33 | cmp = (cpr1 < cpr2) ? -1 : ((cpr1 == cpr2) ? 0: 1); 34 | if(0 != cmp){ 35 | return cmp; 36 | } 37 | } 38 | int sign1 = readInt(b1, s1 + Config.NUMINTBITS + len1 * Config.NUMLONGBITS); 39 | int sign2 = readInt(b2, s2 + Config.NUMINTBITS + len1 * Config.NUMLONGBITS); 40 | cmp = (sign1 < sign2) ? -1 : ((sign1 == sign2) ? 0: 1); 41 | 42 | return cmp; 43 | } 44 | 45 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/iterator/TAdvancingIterator.java: -------------------------------------------------------------------------------- 1 | // //////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2009, Rob Eden All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | // //////////////////////////////////////////////////////////////////////////// 18 | package gnu.trove.iterator; 19 | 20 | /** 21 | * Common interface for iterators that operate via the "advance" method for moving the 22 | * cursor to the next element. 23 | */ 24 | public interface TAdvancingIterator extends TIterator { 25 | /** 26 | * Moves the iterator forward to the next entry. 27 | * 28 | * @throws java.util.NoSuchElementException if the iterator is already exhausted 29 | */ 30 | public void advance(); 31 | } 32 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/maximalclique/MCliqueS4Reducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.maximalclique; 2 | 3 | import java.io.IOException; 4 | import org.apache.hadoop.io.LongWritable; 5 | import org.apache.hadoop.io.Text; 6 | import org.apache.hadoop.mapreduce.Reducer; 7 | 8 | import dbg.hadoop.subgraphs.io.HVArray; 9 | import dbg.hadoop.subgraphs.utils.Config; 10 | import dbg.hadoop.subgraphs.utils.HyperVertex; 11 | import dbg.hadoop.subgraphs.utils.HyperVertexHeap; 12 | 13 | 14 | public class MCliqueS4Reducer 15 | extends Reducer { 16 | 17 | private static int cliqueSizeThresh = 20; 18 | @Override 19 | public void reduce(LongWritable _key, Iterable values, 20 | Context context) throws IOException, InterruptedException { 21 | int size = 0; 22 | // The vertex will belong to the largest clique 23 | HyperVertexHeap heap = new HyperVertexHeap(Config.HEAPINITSIZE); 24 | for(LongWritable value : values){ 25 | heap.insert(value.get()); 26 | ++size; 27 | } 28 | if(size >= cliqueSizeThresh){ 29 | heap.sort(); 30 | LongWritable outputKey = new LongWritable(HyperVertex.get 31 | (HyperVertex.VertexID(heap.getFirst()), true, 32 | size, HyperVertex.Degree(heap.getFirst()))); 33 | context.write(outputKey, new HVArray(heap.toArrays())); 34 | } 35 | heap.clear(); 36 | } 37 | 38 | @Override 39 | public void setup(Context context){ 40 | cliqueSizeThresh = context.getConfiguration().getInt("mapred.clique.size.threshold", 20); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/triangle/TriangleReducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph.triangle; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.NullWritable; 6 | import org.apache.hadoop.mapreduce.Reducer; 7 | 8 | import dbg.hadoop.subgraphs.io.HVArraySign; 9 | import dbg.hadoop.subgraphs.io.HVArray; 10 | import dbg.hadoop.subgraphs.utils.Config; 11 | 12 | 13 | /** 14 | * Suppose the triangle is (v1, v2, v3) where v1 < v2 < v3 as follows:
15 | * v1
16 | * / \
17 | * v2---v3
18 | * @author robeen 19 | * 20 | */ 21 | public class TriangleReducer 22 | extends Reducer { 23 | 24 | public void reduce(HVArraySign _key, Iterable values, 25 | Context context) throws IOException, InterruptedException { 26 | long v2 = _key.vertexArray.getFirst(); 27 | long v3 = _key.vertexArray.getSecond(); 28 | if(v2 == v3){ 29 | //for(HVArray value: values){ 30 | //System.err.println(new HVArray(value.getFirst(), v2, v3)); 31 | // context.write(NullWritable.get(), new HVArray(value.getFirst(), v2, v3)); 32 | //} 33 | return; 34 | } 35 | else{ 36 | if(_key.sign != Config.SMALLSIGN){ 37 | return; 38 | } 39 | for(HVArray value : values){ 40 | if(_key.sign == Config.SMALLSIGN){ 41 | continue; 42 | } 43 | else{ 44 | context.write(NullWritable.get(), new HVArray(value.getFirst(), v2, v3)); 45 | //System.err.println(new HVArray(value.getFirst(), v2, v3)); 46 | } 47 | } 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/utils/BinarySearch.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.utils; 2 | 3 | public class BinarySearch { 4 | public static int binarySearch(long target, long[] list) { 5 | int left = 0; 6 | int right = list.length - 1; 7 | while (left <= right) { 8 | int mid = left + ((right - left) >> 1); 9 | if (list[mid] == target) 10 | return mid; 11 | else if (list[mid] > target) 12 | right = mid - 1; 13 | else 14 | left = mid + 1; 15 | } 16 | return -1; 17 | } 18 | 19 | /** 20 | * Find target in the list, indiced from (include) and to (exclude) 21 | * @param target 22 | * @param list 23 | * @param from 24 | * @param to 25 | * @return 26 | */ 27 | public static int binarySearch(long target, long[] list, int from, int to) { 28 | int left = from; 29 | int right = to - 1; 30 | while (left <= right) { 31 | int mid = left + ((right - left) >> 1); 32 | if (list[mid] == target) 33 | return mid; 34 | else if (list[mid] > target) 35 | right = mid - 1; 36 | else 37 | left = mid + 1; 38 | } 39 | return -1; 40 | } 41 | 42 | public static int findLargeIndex(long target, long[] list) { 43 | int res = -1; 44 | int left = 0; 45 | int right = list.length - 1; 46 | while (left <= right) { 47 | int mid = left + ((right - left) >> 1); 48 | if (list[mid] == target) { 49 | res = mid + 1; 50 | break; 51 | } else if (list[mid] > target) { 52 | right = mid - 1; 53 | } else { 54 | left = mid + 1; 55 | } 56 | } 57 | if(res == -1){ 58 | res = left; 59 | } 60 | return res; 61 | } 62 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/function/TIntFunction.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.function; 20 | 21 | ////////////////////////////////////////////////// 22 | // THIS IS A GENERATED CLASS. DO NOT HAND EDIT! // 23 | ////////////////////////////////////////////////// 24 | 25 | 26 | /** 27 | * Interface for functions that accept and return one int primitive. 28 | */ 29 | public interface TIntFunction { 30 | /** 31 | * Execute this function with value 32 | * 33 | * @param value a int input 34 | * @return a int result 35 | */ 36 | public int execute( int value ); 37 | } 38 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/function/TLongFunction.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.function; 20 | 21 | ////////////////////////////////////////////////// 22 | // THIS IS A GENERATED CLASS. DO NOT HAND EDIT! // 23 | ////////////////////////////////////////////////// 24 | 25 | 26 | /** 27 | * Interface for functions that accept and return one long primitive. 28 | */ 29 | public interface TLongFunction { 30 | /** 31 | * Execute this function with value 32 | * 33 | * @param value a long input 34 | * @return a long result 35 | */ 36 | public long execute( long value ); 37 | } 38 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/iterator/TIntIterator.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.iterator; 20 | 21 | ////////////////////////////////////////////////// 22 | // THIS IS A GENERATED CLASS. DO NOT HAND EDIT! // 23 | ////////////////////////////////////////////////// 24 | 25 | 26 | /** 27 | * Iterator for int collections. 28 | */ 29 | public interface TIntIterator extends TIterator { 30 | /** 31 | * Advances the iterator to the next element in the underlying collection 32 | * and returns it. 33 | * 34 | * @return the next int in the collection 35 | * @exception NoSuchElementException if the iterator is already exhausted 36 | */ 37 | public int next(); 38 | } 39 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/bloomfilter/GenBloomFilter.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph.bloomfilter; 2 | 3 | import java.io.IOException; 4 | 5 | import dbg.hadoop.subgraphs.utils.BloomFilterOpr; 6 | import dbg.hadoop.subgraphs.utils.Config; 7 | import dbg.hadoop.subgraphs.utils.InputInfo; 8 | 9 | public class GenBloomFilter{ 10 | 11 | public static void main(String[] args) throws InstantiationException, 12 | IllegalAccessException, ClassNotFoundException, IOException{ 13 | run(new InputInfo(args)); 14 | } 15 | 16 | public static void run(InputInfo inputInfo) throws IOException, InstantiationException, 17 | IllegalAccessException, ClassNotFoundException{ 18 | 19 | String inputFilePath = inputInfo.inputFilePath; 20 | float falsePositive = inputInfo.falsePositive; 21 | long elemSize = inputInfo.elemSize; 22 | int bfType = inputInfo.bfType; 23 | boolean isHyper = inputInfo.isHyper; 24 | String workDir = inputInfo.workDir; 25 | 26 | if(inputFilePath.isEmpty()){ 27 | System.err.println("Input file not specified!"); 28 | System.exit(-1);; 29 | } 30 | 31 | String input = ""; 32 | if (bfType == Config.EDGE) { 33 | input = isHyper ? workDir + Config.hyperEdge : workDir 34 | + Config.preparedFileDir; 35 | } else { 36 | input = workDir + Config.distinctTwinTwigDir; 37 | } 38 | String output = isHyper ? workDir + Config.hyperGraphBloomFilterDir : workDir + Config.bloomFilterFileDir; 39 | 40 | BloomFilterOpr bloomFilterOpr = new BloomFilterOpr( 41 | input, output, 42 | falsePositive, elemSize, bfType, workDir); 43 | 44 | bloomFilterOpr.createBloomFilter(); 45 | bloomFilterOpr.writeBloomFilter(); 46 | } 47 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/iterator/TLongIterator.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.iterator; 20 | 21 | ////////////////////////////////////////////////// 22 | // THIS IS A GENERATED CLASS. DO NOT HAND EDIT! // 23 | ////////////////////////////////////////////////// 24 | 25 | 26 | /** 27 | * Iterator for long collections. 28 | */ 29 | public interface TLongIterator extends TIterator { 30 | /** 31 | * Advances the iterator to the next element in the underlying collection 32 | * and returns it. 33 | * 34 | * @return the next long in the collection 35 | * @exception NoSuchElementException if the iterator is already exhausted 36 | */ 37 | public long next(); 38 | } 39 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/iterator/TIterator.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2009, Rob Eden All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | package gnu.trove.iterator; 19 | 20 | /** 21 | * Common interface for all iterators used in Trove. 22 | */ 23 | public interface TIterator { 24 | /** 25 | * Returns true if the iterator can be advanced past its current location. 26 | * 27 | * @return a boolean value 28 | */ 29 | public boolean hasNext(); 30 | 31 | /** 32 | * Removes the last entry returned by the iterator. The result of invoking this method 33 | * more than once for a single entry is undefined and can leave the underlying data 34 | * structure in a confused state. 35 | */ 36 | public void remove(); 37 | } 38 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/function/TObjectFunction.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.function; 20 | 21 | /** 22 | * Interface for functions that accept and return one Object reference. 23 | *

24 | * Created: Mon Nov 5 22:19:36 2001 25 | * 26 | * @author Eric D. Friedman 27 | * @version $Id: TObjectFunction.java,v 1.1.2.1 2009/09/06 17:02:19 upholderoftruth Exp $ 28 | */ 29 | 30 | public interface TObjectFunction { 31 | 32 | /** 33 | * Execute this function with value 34 | * 35 | * @param value an Object input 36 | * @return an Object result 37 | */ 38 | public R execute( T value ); 39 | }// TObjectFunction 40 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/utils/bloomfilter/io/NativeAllocator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package dbg.hadoop.subgraphs.utils.bloomfilter.io; 19 | 20 | import java.lang.reflect.Field; 21 | 22 | import sun.misc.Unsafe; 23 | 24 | public class NativeAllocator implements IAllocator 25 | { 26 | static final Unsafe unsafe; 27 | static 28 | { 29 | try 30 | { 31 | Field field = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); 32 | field.setAccessible(true); 33 | unsafe = (sun.misc.Unsafe) field.get(null); 34 | } 35 | catch (Exception e) 36 | { 37 | throw new AssertionError(e); 38 | } 39 | } 40 | 41 | public long allocate(long size) 42 | { 43 | return unsafe.allocateMemory(size); 44 | } 45 | 46 | public void free(long peer) 47 | { 48 | unsafe.freeMemory(peer); 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /SubgEnumFramework/subgenum.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015 BloomReach, Inc. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | # Number of threads you want Briefly to run 18 | run_threads = 2 19 | 20 | 21 | # Hadoop settings. We are using the example JAR for word count. 22 | # Plesae install hadoop exetuables from: https://archive.apache.org/dist/hadoop/core/hadoop-1.0.3/hadoop-1.0.3-bin.tar.gz 23 | # or later with hadoop-examples.jar. Modify the path or filename to the jar files accordingly. 24 | hadoop_home = "~/hadoop" 25 | hadoop.runner = "local" 26 | hadoop.bin = "/usr/bin/hadoop" 27 | hadoop.jar = "SubgEnumFramework_cloud.jar" 28 | jar_file = "SubgEnumFramework_cloud.jar" 29 | timeout = 5 30 | num_reducers = 60 31 | is_undirected = false 32 | separator = default 33 | bf_falsepositive = 0.001 34 | bf_element_size = 10000 35 | bf_type = 0 36 | bf_enable = true 37 | clique_size = 4 38 | result_compression = true 39 | clique_size_thresh = 20 40 | count_pattern_once = false 41 | count_only=true 42 | max_size = 2000 43 | square_partition = false 44 | square_partition_thresh = 2000 45 | chordalsquare_partition = false 46 | chordalsquare_partition_thresh = 2000 47 | skip_chordalsquare = false 48 | skip_square = false 49 | left_deep=false 50 | nonoverlapping=false 51 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/procedure/TIntProcedure.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.procedure; 20 | 21 | 22 | ////////////////////////////////////////////////// 23 | // THIS IS A GENERATED CLASS. DO NOT HAND EDIT! // 24 | ////////////////////////////////////////////////// 25 | 26 | 27 | /** 28 | * Interface for procedures with one int parameter. 29 | */ 30 | public interface TIntProcedure { 31 | /** 32 | * Executes this procedure. A false return value indicates that 33 | * the application executing this procedure should not invoke this 34 | * procedure again. 35 | * 36 | * @param value a value of type int 37 | * @return true if additional invocations of the procedure are 38 | * allowed. 39 | */ 40 | public boolean execute( int value ); 41 | } 42 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/procedure/TLongProcedure.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.procedure; 20 | 21 | 22 | ////////////////////////////////////////////////// 23 | // THIS IS A GENERATED CLASS. DO NOT HAND EDIT! // 24 | ////////////////////////////////////////////////// 25 | 26 | 27 | /** 28 | * Interface for procedures with one long parameter. 29 | */ 30 | public interface TLongProcedure { 31 | /** 32 | * Executes this procedure. A false return value indicates that 33 | * the application executing this procedure should not invoke this 34 | * procedure again. 35 | * 36 | * @param value a value of type long 37 | * @return true if additional invocations of the procedure are 38 | * allowed. 39 | */ 40 | public boolean execute( long value ); 41 | } 42 | -------------------------------------------------------------------------------- /SubgEnumFramework/subgenum_pre.py: -------------------------------------------------------------------------------- 1 | from briefly import * 2 | from briefly.common import * 3 | import os 4 | 5 | objs = Pipeline("subgenum hadoop pipeline") 6 | prop = objs.prop 7 | 8 | @simple_hadoop_process 9 | def preprocess(self): 10 | self.config.hadoop.jar = 'SubgEnumFramework_cloud.jar' # path to your local jar 11 | self.config.defaults( 12 | main_class = 'dbg.hadoop.subgenum.prepare.PrepareData', # This is special for hadoop-examples.jar. Use full class name instead. 13 | args = ['mapred.input.file=${input}', \ 14 | 'mapred.reduce.tasks=${num_reducers}', \ 15 | 'jar.file.name=${jar_file}', \ 16 | 'graph.undirected=${is_undirected}', \ 17 | 'mapred.input.key.value.separator=${separator}', \ 18 | 'map.input.max.size=${max_size}', \ 19 | 'enable.bloom.filter=${bf_enable}', \ 20 | 'bloom.filter.element.size=${bf_element_size}', \ 21 | 'bloom.filter.false.positive.rate=${bf_falsepositive}', \ 22 | 'mapred.clique.size.threshold=${clique_size_thresh}'] 23 | ) 24 | 25 | if __name__ == '__main__': 26 | ''' 27 | This is the setting of several parameters for preparing the dataset 28 | data: should be replaced with the dataset name. For example, if you 29 | name you dataset `abc`, then you should create a folder named `abc` 30 | under the home directory of hdfs. 31 | 32 | ''' 33 | data = "[please_put_your_dataset_name]" 34 | target = objs | preprocess() 35 | prop['input'] = data + '/dd' 36 | prop['jar_file'] = '[put your jar file directory]' 37 | prop['separator'] = '[default[tab] | comma | space]' 38 | prop['bf_element_size'] = 'Number of edges in your graph data' 39 | objs.run(target) 40 | os.system("cp build/preprocess-* build/" + data + "-preprocess.log") 41 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/procedure/TIntIntProcedure.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.procedure; 20 | 21 | ////////////////////////////////////////////////// 22 | // THIS IS A GENERATED CLASS. DO NOT HAND EDIT! // 23 | ////////////////////////////////////////////////// 24 | 25 | 26 | /** 27 | * Interface for procedures that take two parameters of type int and int. 28 | */ 29 | public interface TIntIntProcedure { 30 | 31 | /** 32 | * Executes this procedure. A false return value indicates that 33 | * the application executing this procedure should not invoke this 34 | * procedure again. 35 | * 36 | * @param a a int value 37 | * @param b a int value 38 | * @return true if additional invocations of the procedure are 39 | * allowed. 40 | */ 41 | public boolean execute( int a, int b ); 42 | } 43 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/procedure/TIntLongProcedure.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.procedure; 20 | 21 | ////////////////////////////////////////////////// 22 | // THIS IS A GENERATED CLASS. DO NOT HAND EDIT! // 23 | ////////////////////////////////////////////////// 24 | 25 | 26 | /** 27 | * Interface for procedures that take two parameters of type int and long. 28 | */ 29 | public interface TIntLongProcedure { 30 | 31 | /** 32 | * Executes this procedure. A false return value indicates that 33 | * the application executing this procedure should not invoke this 34 | * procedure again. 35 | * 36 | * @param a a int value 37 | * @param b a long value 38 | * @return true if additional invocations of the procedure are 39 | * allowed. 40 | */ 41 | public boolean execute( int a, long b ); 42 | } 43 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/procedure/TLongIntProcedure.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.procedure; 20 | 21 | ////////////////////////////////////////////////// 22 | // THIS IS A GENERATED CLASS. DO NOT HAND EDIT! // 23 | ////////////////////////////////////////////////// 24 | 25 | 26 | /** 27 | * Interface for procedures that take two parameters of type long and int. 28 | */ 29 | public interface TLongIntProcedure { 30 | 31 | /** 32 | * Executes this procedure. A false return value indicates that 33 | * the application executing this procedure should not invoke this 34 | * procedure again. 35 | * 36 | * @param a a long value 37 | * @param b a int value 38 | * @return true if additional invocations of the procedure are 39 | * allowed. 40 | */ 41 | public boolean execute( long a, int b ); 42 | } 43 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/procedure/TLongLongProcedure.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.procedure; 20 | 21 | ////////////////////////////////////////////////// 22 | // THIS IS A GENERATED CLASS. DO NOT HAND EDIT! // 23 | ////////////////////////////////////////////////// 24 | 25 | 26 | /** 27 | * Interface for procedures that take two parameters of type long and long. 28 | */ 29 | public interface TLongLongProcedure { 30 | 31 | /** 32 | * Executes this procedure. A false return value indicates that 33 | * the application executing this procedure should not invoke this 34 | * procedure again. 35 | * 36 | * @param a a long value 37 | * @param b a long value 38 | * @return true if additional invocations of the procedure are 39 | * allowed. 40 | */ 41 | public boolean execute( long a, long b ); 42 | } 43 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/procedure/TIntObjectProcedure.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.procedure; 20 | 21 | ////////////////////////////////////////////////// 22 | // THIS IS A GENERATED CLASS. DO NOT HAND EDIT! // 23 | ////////////////////////////////////////////////// 24 | 25 | 26 | /** 27 | * Interface for procedures that take two parameters of type int and Object. 28 | */ 29 | public interface TIntObjectProcedure { 30 | 31 | /** 32 | * Executes this procedure. A false return value indicates that 33 | * the application executing this procedure should not invoke this 34 | * procedure again. 35 | * 36 | * @param a a int value 37 | * @param b an Object value 38 | * @return true if additional invocations of the procedure are 39 | * allowed. 40 | */ 41 | public boolean execute( int a, T b ); 42 | } 43 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/procedure/TObjectIntProcedure.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.procedure; 20 | 21 | ////////////////////////////////////////////////// 22 | // THIS IS A GENERATED CLASS. DO NOT HAND EDIT! // 23 | ////////////////////////////////////////////////// 24 | 25 | 26 | /** 27 | * Interface for procedures that take two parameters of type Object and int. 28 | */ 29 | public interface TObjectIntProcedure { 30 | 31 | /** 32 | * Executes this procedure. A false return value indicates that 33 | * the application executing this procedure should not invoke this 34 | * procedure again. 35 | * 36 | * @param a an Object value 37 | * @param b a int value 38 | * @return true if additional invocations of the procedure are 39 | * allowed. 40 | */ 41 | public boolean execute( K a, int b ); 42 | } 43 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/procedure/TObjectProcedure.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | 20 | package gnu.trove.procedure; 21 | 22 | /** 23 | * Interface for procedures with one Object parameter. 24 | * 25 | * Created: Mon Nov 5 21:45:49 2001 26 | * 27 | * @author Eric D. Friedman 28 | * @version $Id: TObjectProcedure.java,v 1.1.2.1 2009/09/02 21:52:33 upholderoftruth Exp $ 29 | */ 30 | 31 | public interface TObjectProcedure { 32 | /** 33 | * Executes this procedure. A false return value indicates that 34 | * the application executing this procedure should not invoke this 35 | * procedure again. 36 | * 37 | * @param object an Object value 38 | * @return true if additional invocations of the procedure are 39 | * allowed. 40 | */ 41 | public boolean execute(T object); 42 | }// TObjectProcedure 43 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/procedure/TLongObjectProcedure.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.procedure; 20 | 21 | ////////////////////////////////////////////////// 22 | // THIS IS A GENERATED CLASS. DO NOT HAND EDIT! // 23 | ////////////////////////////////////////////////// 24 | 25 | 26 | /** 27 | * Interface for procedures that take two parameters of type long and Object. 28 | */ 29 | public interface TLongObjectProcedure { 30 | 31 | /** 32 | * Executes this procedure. A false return value indicates that 33 | * the application executing this procedure should not invoke this 34 | * procedure again. 35 | * 36 | * @param a a long value 37 | * @param b an Object value 38 | * @return true if additional invocations of the procedure are 39 | * allowed. 40 | */ 41 | public boolean execute( long a, T b ); 42 | } 43 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/procedure/TObjectLongProcedure.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.procedure; 20 | 21 | ////////////////////////////////////////////////// 22 | // THIS IS A GENERATED CLASS. DO NOT HAND EDIT! // 23 | ////////////////////////////////////////////////// 24 | 25 | 26 | /** 27 | * Interface for procedures that take two parameters of type Object and long. 28 | */ 29 | public interface TObjectLongProcedure { 30 | 31 | /** 32 | * Executes this procedure. A false return value indicates that 33 | * the application executing this procedure should not invoke this 34 | * procedure again. 35 | * 36 | * @param a an Object value 37 | * @param b a long value 38 | * @return true if additional invocations of the procedure are 39 | * allowed. 40 | */ 41 | public boolean execute( K a, long b ); 42 | } 43 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/utils/bloomfilter/obs/IBitSet.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package dbg.hadoop.subgraphs.utils.bloomfilter.obs; 19 | 20 | import java.io.Closeable; 21 | import java.io.DataOutput; 22 | import java.io.IOException; 23 | 24 | import dbg.hadoop.subgraphs.utils.bloomfilter.db.TypeSizes; 25 | 26 | public interface IBitSet extends Closeable 27 | { 28 | public long capacity(); 29 | 30 | /** 31 | * Returns true or false for the specified bit index. The index should be 32 | * less than the capacity. 33 | */ 34 | public boolean get(long index); 35 | 36 | /** 37 | * Sets the bit at the specified index. The index should be less than the 38 | * capacity. 39 | */ 40 | public void set(long index); 41 | 42 | /** 43 | * clears the bit. The index should be less than the capacity. 44 | */ 45 | public void clear(long index); 46 | 47 | public void serialize(DataOutput out) throws IOException; 48 | 49 | public long serializedSize(TypeSizes type); 50 | 51 | public void clear(); 52 | } 53 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/utils/bloomfilter/io/ISerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package dbg.hadoop.subgraphs.utils.bloomfilter.io; 19 | 20 | import java.io.DataInput; 21 | import java.io.DataOutput; 22 | import java.io.IOException; 23 | 24 | import dbg.hadoop.subgraphs.utils.bloomfilter.db.TypeSizes; 25 | 26 | public interface ISerializer 27 | { 28 | /** 29 | * Serialize the specified type into the specified DataOutput instance. 30 | * @param t type that needs to be serialized 31 | * @param out DataOutput into which serialization needs to happen. 32 | * @throws java.io.IOException 33 | */ 34 | public void serialize(T t, DataOutput out) throws IOException; 35 | 36 | /** 37 | * Deserialize from the specified DataInput instance. 38 | * @param in DataInput from which deserialization needs to happen. 39 | * @throws IOException 40 | * @return the type that was deserialized 41 | */ 42 | public T deserialize(DataInput in) throws IOException; 43 | 44 | public long serializedSize(T t, TypeSizes type); 45 | } 46 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/maximalclique/MCliqueS2Reducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.maximalclique; 2 | 3 | import gnu.trove.set.hash.TLongHashSet; 4 | 5 | import java.io.IOException; 6 | import java.util.Collection; 7 | import java.util.Iterator; 8 | 9 | import org.apache.hadoop.io.LongWritable; 10 | import org.apache.hadoop.mapreduce.Reducer; 11 | 12 | import dbg.hadoop.subgraphs.io.HVArray; 13 | import dbg.hadoop.subgraphs.utils.Graph; 14 | import dbg.hadoop.subgraphs.utils.HyperVertex; 15 | import dbg.hadoop.subgraphs.utils.HyperVertexHeap; 16 | 17 | 18 | public class MCliqueS2Reducer 19 | extends Reducer { 20 | 21 | private static int cliqueSizeThresh = 20; 22 | @Override 23 | public void reduce(LongWritable _key, Iterable values, 24 | Context context) throws IOException, InterruptedException { 25 | Graph graph = new Graph(); 26 | for(HVArray vPair : values){ 27 | graph.addEdge(vPair.getFirst(), vPair.getSecond()); 28 | graph.addEdge(_key.get(), vPair.getFirst()); 29 | graph.addEdge(_key.get(), vPair.getSecond()); 30 | } 31 | 32 | Collection cliques = graph.findCliqueCover(cliqueSizeThresh); 33 | Iterator iter = cliques.iterator(); 34 | while(iter.hasNext()){ 35 | long[] curClique = iter.next().toArrays(); 36 | if(curClique.length < cliqueSizeThresh){ 37 | continue; 38 | } 39 | long representVertex = curClique[0]; 40 | context.write(new LongWritable(HyperVertex.get( 41 | HyperVertex.VertexID(representVertex), true, 42 | curClique.length, HyperVertex.Degree(representVertex))), 43 | new HVArray(curClique)); 44 | } 45 | 46 | graph = null; 47 | cliques.clear(); 48 | cliques = null; 49 | } 50 | 51 | @Override 52 | public void setup(Context context){ 53 | cliqueSizeThresh = context.getConfiguration().getInt("mapred.clique.size.threshold", 20); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/procedure/array/ToObjectArrayProceedure.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.procedure.array; 20 | 21 | import gnu.trove.procedure.TObjectProcedure; 22 | 23 | 24 | 25 | /** 26 | * A procedure which stores each value it receives into a target array. 27 | *

28 | * Created: Sat Jan 12 10:13:42 2002 29 | * 30 | * @author Eric D. Friedman 31 | * @version $Id: ToObjectArrayProceedure.java,v 1.1.2.1 2009/09/02 21:52:33 upholderoftruth Exp $ 32 | */ 33 | 34 | public final class ToObjectArrayProceedure implements TObjectProcedure { 35 | 36 | private final T[] target; 37 | private int pos = 0; 38 | 39 | 40 | public ToObjectArrayProceedure( final T[] target ) { 41 | this.target = target; 42 | } 43 | 44 | 45 | public final boolean execute( T value ) { 46 | target[pos++] = value; 47 | return true; 48 | } 49 | } // ToObjectArrayProcedure -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/procedure/TObjectObjectProcedure.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.procedure; 20 | 21 | /** 22 | * Interface for procedures that take two Object parameters. 23 | *

24 | * Created: Mon Nov 5 22:03:30 2001 25 | * 26 | * @author Eric D. Friedman 27 | * @version $Id: TObjectObjectProcedure.java,v 1.1.2.1 2009/09/06 17:02:20 upholderoftruth Exp $ 28 | */ 29 | 30 | public interface TObjectObjectProcedure { 31 | 32 | /** 33 | * Executes this procedure. A false return value indicates that 34 | * the application executing this procedure should not invoke this 35 | * procedure again. 36 | * 37 | * @param a an Object value 38 | * @param b an Object value 39 | * @return true if additional invocations of the procedure are 40 | * allowed. 41 | */ 42 | public boolean execute( K a, V b ); 43 | }// TObjectObjectProcedure 44 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageOneReducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.io.LongWritable; 6 | import org.apache.hadoop.mapreduce.Reducer; 7 | 8 | import dbg.hadoop.subgraphs.utils.Config; 9 | import dbg.hadoop.subgraphs.utils.HyperVertexHeap; 10 | import dbg.hadoop.subgraphs.io.HVArray; 11 | import dbg.hadoop.subgraphs.io.HyperVertexSign; 12 | 13 | public class HyperGraphStageOneReducer 14 | extends Reducer { 15 | 16 | public void reduce(HVArray _key, Iterable values, 17 | Context context) throws IOException, InterruptedException { 18 | 19 | int size = 0; 20 | HyperVertexHeap vertexHeap = new HyperVertexHeap(Config.HEAPINITSIZE); 21 | 22 | for(LongWritable value: values){ 23 | ++size; 24 | vertexHeap.insert(value.get()); 25 | } 26 | vertexHeap.sort(); 27 | // Means we donot output the adjList, the vertex should belong to itself. 28 | // We donot need to check it anymore 29 | if(_key.size() == 1){ 30 | context.write(new HyperVertexSign(_key.getFirst(), 0), _key); 31 | //System.out.println("output-key: " + new HyperVertexSign(_key.getFirst(), 0)); 32 | //System.out.println("output-value: " + _key); 33 | } 34 | // Means the output adjList only finds one key, it should belong to itself. 35 | // But we should check it in the second round. 36 | else if(size == 1){ 37 | context.write(new HyperVertexSign(vertexHeap.getFirst(), 2), 38 | new HVArray(vertexHeap.getFirst())); 39 | } 40 | // We ouptut the _key.getFirst as the hypervertex id as it is the smallest vertex 41 | // in the set. 42 | else{ 43 | context.write(new HyperVertexSign(vertexHeap.getFirst(), 1), 44 | new HVArray(vertexHeap.toArrays())); 45 | //System.out.println("output-key: " + new HyperVertexSign(vertexHeap.getFirst(), 1)); 46 | //System.out.println("output-value: " + new HVArray(vertexHeap.toArrays())); 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/utils/bloomfilter/utils/Murmur3BloomFilter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package dbg.hadoop.subgraphs.utils.bloomfilter.utils; 19 | 20 | import java.nio.ByteBuffer; 21 | 22 | import dbg.hadoop.subgraphs.utils.bloomfilter.db.TypeSizes; 23 | import dbg.hadoop.subgraphs.utils.bloomfilter.obs.IBitSet; 24 | 25 | public class Murmur3BloomFilter extends BloomFilter 26 | { 27 | public static final Murmur3BloomFilterSerializer serializer = new Murmur3BloomFilterSerializer(); 28 | 29 | public Murmur3BloomFilter(int hashes, IBitSet bs) 30 | { 31 | super(hashes, bs); 32 | } 33 | 34 | public long serializedSize() 35 | { 36 | return serializer.serializedSize(this, TypeSizes.NATIVE); 37 | } 38 | 39 | protected void hash(ByteBuffer b, int position, int remaining, long seed, long[] result) 40 | { 41 | MurmurHash.hash3_x64_128(b, b.position(), b.remaining(), seed, result); 42 | } 43 | 44 | public static class Murmur3BloomFilterSerializer extends BloomFilterSerializer 45 | { 46 | protected BloomFilter createFilter(int hashes, IBitSet bs) 47 | { 48 | return new Murmur3BloomFilter(hashes, bs); 49 | } 50 | } 51 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/utils/bloomfilter/io/RefCountedMemory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package dbg.hadoop.subgraphs.utils.bloomfilter.io; 19 | 20 | import java.util.concurrent.atomic.AtomicInteger; 21 | 22 | import dbg.hadoop.subgraphs.utils.bloomfilter.io.Memory; 23 | 24 | public class RefCountedMemory extends Memory 25 | { 26 | private final AtomicInteger references = new AtomicInteger(1); 27 | 28 | public RefCountedMemory(long size) 29 | { 30 | super(size); 31 | } 32 | 33 | /** 34 | * @return true if we succeed in referencing before the reference count reaches zero. 35 | * (A FreeableMemory object is created with a reference count of one.) 36 | */ 37 | public boolean reference() 38 | { 39 | while (true) 40 | { 41 | int n = references.get(); 42 | if (n <= 0) 43 | return false; 44 | if (references.compareAndSet(n, n + 1)) 45 | return true; 46 | } 47 | } 48 | 49 | /** decrement reference count. if count reaches zero, the object is freed. */ 50 | public void unreference() 51 | { 52 | if (references.decrementAndGet() == 0) 53 | free(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/triangle/TriangleMapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph.triangle; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.mapreduce.Mapper; 8 | 9 | import dbg.hadoop.subgraphs.io.HyperVertexAdjList; 10 | import dbg.hadoop.subgraphs.io.HVArraySign; 11 | import dbg.hadoop.subgraphs.io.HVArray; 12 | import dbg.hadoop.subgraphs.utils.BloomFilterOpr; 13 | import dbg.hadoop.subgraphs.utils.Config; 14 | import dbg.hadoop.subgraphs.utils.TwinTwigGenerator; 15 | 16 | public class TriangleMapper 17 | extends Mapper { 18 | 19 | private static BloomFilterOpr bloomfilterOpr = null; 20 | private static boolean enableBF; 21 | 22 | private TwinTwigGenerator ttwigGen = null; 23 | // The hypervertex set 24 | @Override 25 | public void map(LongWritable key, HyperVertexAdjList value, Context context) 26 | throws IOException, InterruptedException{ 27 | if(enableBF){ 28 | ttwigGen = new TwinTwigGenerator(key.get(), value, bloomfilterOpr.get()); 29 | } else{ 30 | ttwigGen = new TwinTwigGenerator(key.get(), value); 31 | } 32 | // keyMap = 3 = 011, v2, v3 as the key; 33 | ttwigGen.genTwinTwigOne(context, Config.LARGESIGN, (byte)3, (byte)0); 34 | } 35 | 36 | @Override 37 | public void setup(Context context) throws IOException{ 38 | Configuration conf = context.getConfiguration(); 39 | // We use bloomfilter as static. If it is already loaded, we will have 40 | // bloomfilterOpr != null, and we donot load it again in the case. 41 | enableBF = conf.getBoolean("enable.bloom.filter", true); 42 | if(enableBF && bloomfilterOpr == null){ 43 | bloomfilterOpr = new BloomFilterOpr 44 | (conf.getFloat("bloom.filter.false.positive.rate", (float) 0.001)); 45 | try { 46 | bloomfilterOpr.obtainBloomFilter(conf); 47 | } catch (ClassNotFoundException e) { 48 | e.printStackTrace(); 49 | } 50 | } 51 | } 52 | 53 | @Override 54 | public void cleanup(Context context){ 55 | ttwigGen = null; 56 | } 57 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/qtcounter/QtS1Mapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.qtcounter; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.io.LongWritable; 7 | import org.apache.hadoop.mapreduce.Mapper; 8 | import org.apache.log4j.Logger; 9 | 10 | import dbg.hadoop.subgraphs.io.HyperVertexAdjList; 11 | import dbg.hadoop.subgraphs.io.HVArraySign; 12 | import dbg.hadoop.subgraphs.utils.BloomFilterOpr; 13 | import dbg.hadoop.subgraphs.utils.Config; 14 | import dbg.hadoop.subgraphs.utils.HyperVertex; 15 | 16 | public class QtS1Mapper 17 | extends Mapper { 18 | 19 | private static BloomFilterOpr bloomfilterOpr = null; 20 | private static boolean enableBF; 21 | // The hypervertex set 22 | @Override 23 | public void map(LongWritable key, HyperVertexAdjList value, Context context) 24 | throws IOException, InterruptedException{ 25 | long largerThanCur[] = value.getLargeDegreeVertices(); 26 | int size = largerThanCur.length; 27 | long v1 = 0, v2 = 0; 28 | boolean output = true; 29 | for(int i = 0; i < size - 1; ++i){ 30 | for(int j = i + 1; j < size; ++j){ 31 | v1 = largerThanCur[i]; 32 | v2 = largerThanCur[j]; 33 | if(enableBF && !bloomfilterOpr.get().test(HyperVertex.VertexID(v1), 34 | HyperVertex.VertexID(v2))){ 35 | output = false; 36 | } 37 | if(output){ 38 | context.write(new HVArraySign(v1, v2, Config.LARGESIGN), key); 39 | } 40 | } 41 | } 42 | } 43 | 44 | @Override 45 | public void setup(Context context) throws IOException{ 46 | Configuration conf = context.getConfiguration(); 47 | // We use bloomfilter as static. If it is already loaded, we will have 48 | // bloomfilterOpr != null, and we donot load it again in the case. 49 | enableBF = conf.getBoolean("enable.bloom.filter", true); 50 | if(enableBF && bloomfilterOpr == null){ 51 | bloomfilterOpr = new BloomFilterOpr 52 | (conf.getFloat("bloom.filter.false.positive.rate", (float) 0.001)); 53 | try { 54 | bloomfilterOpr.obtainBloomFilter(conf); 55 | } catch (ClassNotFoundException e) { 56 | e.printStackTrace(); 57 | } 58 | } 59 | } 60 | } -------------------------------------------------------------------------------- /SubgEnumFramework/subgenum_frame.py: -------------------------------------------------------------------------------- 1 | from briefly import * 2 | from briefly.common import * 3 | 4 | objs = Pipeline("subgenum hadoop pipeline") 5 | prop = objs.prop 6 | 7 | @simple_hadoop_process 8 | def subgenum_frame(self): 9 | self.config.defaults( 10 | main_class = 'dbg.hadoop.subgenum.frame.MainEntry', # This is special for hadoop-examples.jar. Use full class name instead. 11 | args = ['enum.query=${query}', \ 12 | 'mapred.input.file=${input}', \ 13 | 'mapred.reduce.tasks=${num_reducers}', \ 14 | 'jar.file.name=${jar_file}', \ 15 | 'clique.number.vertices=${clique_size}', \ 16 | 'result.compression=${result_compression}', \ 17 | 'bloom.filter.false.positive.rate=${bf_falsepositive}', \ 18 | 'bloom.filter.type=${bf_type}', \ 19 | 'enable.bloom.filter=${bf_enable}', \ 20 | 'map.input.max.size=${max_size}', \ 21 | 'count.pattern.once=${count_pattern_once}', \ 22 | 'count.only=${count_only}', \ 23 | 'enum.clique.v2=${enum_clique_v2}', \ 24 | 'enum.house.square.partition=${square_partition}', \ 25 | 'enum.house.square.partition.thresh=${square_partition_thresh}', \ 26 | 'enum.solarsquare.chordalsquare.partition=${chordalsquare_partition}', \ 27 | 'enum.solarsquare.chordalsquare.partition.thresh=${chordalsquare_partition_thresh}', \ 28 | 'skip.chordalsquare=${skip_chordalsquare}', \ 29 | 'skip.square=${skip_square}', \ 30 | 'is.leftdeep=${left_deep}', \ 31 | 'is.nonoverlapping=${nonoverlapping}' 32 | ] 33 | ) 34 | 35 | data = '[please specify your dataset name]' 36 | prop['input'] = data + "/dd" 37 | prop['query'] = 'Please specify your query, supported are:' 38 | ''' 39 | square: q1 in the paper 40 | chordalsquare: q2 41 | clique: (q3 and q7) If you put clique in this part, you have to also suggest the number of nodes in the clique by 42 | putting prop['clique_size'] = 5, 6 .... Default is 4-clique 43 | house: q4 44 | tcsquare: q5 45 | near5clique: q6 46 | ''' 47 | prop['count_pattern_once'] = 'Please assign this to true if you want to get the sum results' 48 | 49 | target = objs | subgenum_frame() 50 | 51 | objs.run(target) 52 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/iterator/hash/TObjectHashIterator.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // Copyright (c) 2009, Rob Eden All Rights Reserved. 4 | // Copyright (c) 2009, Jeff Randall All Rights Reserved. 5 | // 6 | // This library is free software; you can redistribute it and/or 7 | // modify it under the terms of the GNU Lesser General Public 8 | // License as published by the Free Software Foundation; either 9 | // version 2.1 of the License, or (at your option) any later version. 10 | // 11 | // This library is distributed in the hope that it will be useful, 12 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | // GNU General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Lesser General Public 17 | // License along with this program; if not, write to the Free Software 18 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | /////////////////////////////////////////////////////////////////////////////// 20 | 21 | 22 | package gnu.trove.iterator.hash; 23 | 24 | import gnu.trove.impl.hash.TObjectHash; 25 | import gnu.trove.impl.hash.THashIterator; 26 | 27 | 28 | /** 29 | * Iterator for hashtables that use open addressing to resolve collisions. 30 | * 31 | * @author Eric D. Friedman 32 | * @author Rob Eden 33 | * @author Jeff Randall 34 | * @version $Id: TObjectHashIterator.java,v 1.1.2.4 2009/10/09 01:44:34 robeden Exp $ 35 | */ 36 | 37 | public class TObjectHashIterator extends THashIterator { 38 | 39 | protected final TObjectHash _objectHash; 40 | 41 | 42 | public TObjectHashIterator( TObjectHash hash ) { 43 | super( hash ); 44 | _objectHash = hash; 45 | } 46 | 47 | 48 | @SuppressWarnings("unchecked") 49 | protected E objectAtIndex( int index ) { 50 | Object obj = _objectHash._set[index]; 51 | if ( obj == TObjectHash.FREE || obj == TObjectHash.REMOVED ) { 52 | return null; 53 | } 54 | return (E) obj; 55 | } 56 | 57 | } // TObjectHashIterator 58 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/HyperVertexSign.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | 7 | import org.apache.hadoop.io.Writable; 8 | import org.apache.hadoop.io.WritableComparable; 9 | 10 | import dbg.hadoop.subgraphs.utils.HyperVertex; 11 | 12 | 13 | public class HyperVertexSign implements Writable, WritableComparable { 14 | private long vertex; 15 | private int sign; 16 | 17 | public HyperVertexSign(){ 18 | vertex = 0; 19 | sign = 0; 20 | } 21 | 22 | //public DoubleIntegerPairWritable(int[] integers){ 23 | // this.set(integers); 24 | //} 25 | 26 | public HyperVertexSign(long _vertex, int _sign){ 27 | this.set(_vertex, _sign); 28 | } 29 | 30 | public HyperVertexSign(HyperVertexSign values) { 31 | // TODO Auto-generated constructor stub 32 | this.set(values.getVertex(), values.getSign()); 33 | } 34 | 35 | public void set(long _vertex, int _sign){ 36 | this.vertex = _vertex; 37 | this.sign = _sign; 38 | } 39 | 40 | public long getVertex(){ 41 | return this.vertex; 42 | } 43 | 44 | public int getSign(){ 45 | return this.sign; 46 | } 47 | 48 | @Override 49 | public void readFields(DataInput in) throws IOException { 50 | // TODO Auto-generated method stub 51 | this.vertex = in.readLong(); 52 | this.sign = in.readInt(); 53 | } 54 | 55 | @Override 56 | public void write(DataOutput out) throws IOException { 57 | // TODO Auto-generated method stub 58 | out.writeLong(this.vertex); 59 | out.writeInt(this.sign); 60 | } 61 | 62 | @Override 63 | public int compareTo(Object obj) { 64 | // TODO Auto-generated method stub 65 | int cmp = 0; 66 | HyperVertexSign that = (HyperVertexSign)obj; 67 | if(this.vertex == that.getVertex() && this.sign == that.getSign()){ 68 | return cmp; 69 | } 70 | else if(this.vertex != that.getVertex()){ 71 | cmp = HyperVertex.compare(this.vertex, that.getVertex()); 72 | } 73 | else{ 74 | cmp = (this.sign < that.getSign()) ? -1 : 1; 75 | } 76 | 77 | return cmp; 78 | } 79 | 80 | @Override 81 | public String toString(){ 82 | return HyperVertex.toString(vertex) + "," + sign; 83 | } 84 | 85 | @Override 86 | public int hashCode(){ 87 | return (31 * HyperVertex.VertexID(vertex)); 88 | } 89 | 90 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/prepare/GenDegreeDriver.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.prepare; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.conf.Configured; 7 | import org.apache.hadoop.fs.Path; 8 | import org.apache.hadoop.io.IntWritable; 9 | import org.apache.hadoop.mapred.JobConf; 10 | import org.apache.hadoop.mapreduce.Job; 11 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 12 | import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; 13 | import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; 14 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 15 | import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; 16 | import org.apache.hadoop.util.Tool; 17 | 18 | public class GenDegreeDriver extends Configured implements Tool{ 19 | 20 | public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException { 21 | Configuration conf = new Configuration(); 22 | // The parameters: 23 | int numReducers = Integer.parseInt(args[2]); 24 | //conf.setBoolean("graph.undirected", Boolean.parseBoolean(args[4])); 25 | 26 | conf.setBoolean("mapreduce.map.output.compress", true); 27 | conf.set("mapreduce.map.output.compress.codec", "com.hadoop.compression.lzo.LzoCodec"); 28 | //conf.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.DefaultCodec"); 29 | 30 | Job job = new Job(conf, "Degree Generation"); 31 | ((JobConf)job.getConfiguration()).setJar(args[3]); 32 | 33 | job.setMapperClass(GenDegreeMapper2.class); 34 | job.setReducerClass(GenDegreeReducer.class); 35 | job.setCombinerClass(GenDegreeReducer.class); 36 | 37 | //job.setInputFormatClass(KeyValueTextInputFormat.class); 38 | job.setInputFormatClass(SequenceFileInputFormat.class); 39 | job.setOutputFormatClass(SequenceFileOutputFormat.class); 40 | 41 | job.setOutputKeyClass(IntWritable.class); 42 | job.setOutputValueClass(IntWritable.class); 43 | 44 | job.setNumReduceTasks(numReducers); 45 | 46 | FileInputFormat.setInputPaths(job, new Path(args[0])); 47 | FileOutputFormat.setOutputPath(job, new Path(args[1])); 48 | 49 | job.waitForCompletion(true); 50 | return 0; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/io/DoubleIntegerPairWritable.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.io; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | 7 | import org.apache.hadoop.io.Writable; 8 | import org.apache.hadoop.io.WritableComparable; 9 | 10 | 11 | public class DoubleIntegerPairWritable implements Writable, WritableComparable { 12 | private int integer1; 13 | private int integer2; 14 | 15 | public DoubleIntegerPairWritable(){ 16 | integer1 = 0; 17 | integer2 = 0; 18 | } 19 | 20 | //public DoubleIntegerPairWritable(int[] integers){ 21 | // this.set(integers); 22 | //} 23 | 24 | public DoubleIntegerPairWritable(int input1, int input2){ 25 | this.set(input1, input2); 26 | } 27 | 28 | public DoubleIntegerPairWritable(DoubleIntegerPairWritable values) { 29 | // TODO Auto-generated constructor stub 30 | this.set(values.getFirst(), values.getSecond()); 31 | } 32 | 33 | public void set(int input1, int input2){ 34 | this.integer1 = input1; 35 | this.integer2 = input2; 36 | } 37 | 38 | public int getFirst(){ 39 | return this.integer1; 40 | } 41 | 42 | public int getSecond(){ 43 | return this.integer2; 44 | } 45 | 46 | @Override 47 | public void readFields(DataInput in) throws IOException { 48 | // TODO Auto-generated method stub 49 | this.integer1 = in.readInt(); 50 | this.integer2 = in.readInt(); 51 | } 52 | 53 | @Override 54 | public void write(DataOutput out) throws IOException { 55 | // TODO Auto-generated method stub 56 | out.writeInt(this.integer1); 57 | out.writeInt(this.integer2); 58 | } 59 | 60 | @Override 61 | public int compareTo(Object obj) { 62 | // TODO Auto-generated method stub 63 | int cmp = 0; 64 | DoubleIntegerPairWritable dw = (DoubleIntegerPairWritable)obj; 65 | if(this.integer1 == dw.getFirst() & this.integer2 == dw.getSecond()){ 66 | return cmp; 67 | } 68 | else if(this.integer1 != dw.getFirst()){ 69 | cmp = (this.integer1 < dw.getFirst()) ? -1 : 1; 70 | } 71 | else{ 72 | cmp = (this.integer2 < dw.getSecond()) ? -1 : 1; 73 | } 74 | 75 | return cmp; 76 | } 77 | 78 | @Override 79 | public String toString(){ 80 | return integer1 + "," + integer2; 81 | } 82 | 83 | @Override 84 | public int hashCode(){ 85 | return (31 * integer1 + integer2); 86 | } 87 | 88 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/frame/GeneralPatternCountDriver.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.frame; 2 | 3 | import java.io.IOException; 4 | import java.net.URISyntaxException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.conf.Configured; 8 | import org.apache.hadoop.fs.Path; 9 | import org.apache.hadoop.io.LongWritable; 10 | import org.apache.hadoop.io.NullWritable; 11 | import org.apache.hadoop.mapred.JobConf; 12 | import org.apache.hadoop.mapreduce.Job; 13 | import org.apache.hadoop.mapreduce.Mapper; 14 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 15 | import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; 16 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 17 | import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 18 | import org.apache.hadoop.util.Tool; 19 | 20 | @SuppressWarnings("rawtypes") 21 | public class GeneralPatternCountDriver extends Configured implements Tool{ 22 | 23 | private Class mapperClass = null; 24 | 25 | public GeneralPatternCountDriver(Class _mapperClass) { 26 | mapperClass = _mapperClass; 27 | } 28 | 29 | public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException, URISyntaxException { 30 | assert(mapperClass != null); 31 | Configuration conf = getConf(); 32 | // The parameters: 33 | conf.setBoolean("mapred.compress.map.output", true); 34 | conf.set("mapred.map.output.compression.codec", "com.hadoop.compression.lzo.LzoCodec"); 35 | Job job = new Job(conf, "Frame Pattern Count"); 36 | ((JobConf)job.getConfiguration()).setJar(args[3]); 37 | 38 | job.setInputFormatClass(SequenceFileInputFormat.class); 39 | job.setOutputFormatClass(TextOutputFormat.class); 40 | 41 | job.setOutputKeyClass(NullWritable.class); 42 | job.setOutputValueClass(LongWritable.class); 43 | 44 | job.setMapperClass(this.mapperClass); 45 | job.setCombinerClass(GeneralPatternCountReducer.class); 46 | job.setReducerClass(GeneralPatternCountReducer.class); 47 | 48 | job.setNumReduceTasks(1); 49 | 50 | FileInputFormat.setInputPaths(job, new Path(args[0])); 51 | FileOutputFormat.setOutputPath(job, new Path(args[1])); 52 | 53 | job.waitForCompletion(true); 54 | return 0; 55 | } 56 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/utils/Config.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgraphs.utils; 2 | 3 | /** 4 | * Set several graph related parameters 5 | * @author robeen 6 | * 7 | */ 8 | public class Config{ 9 | public static final boolean undirected = true; 10 | //private static String bloomFilterFile = ""; 11 | // The home dir of the job. 12 | // Generally this is the .. directory of the input file 13 | public static final String homeDir = "."; 14 | // The degree file directory 15 | public static final String degreeFileDir = "degreeFile"; 16 | // The bloom filter file directory 17 | public static final String bloomFilterFileDir = "bloomFilter2"; 18 | public static final String undirectGraphDir = "undirected"; 19 | 20 | public static final String preparedFileDir = "prepared2"; 21 | public static final String adjListDir = "adjList2"; 22 | public static final String distinctTwinTwigDir = "distinct.twinTwig"; 23 | 24 | public static final String coloredVertexDir = "vertex.color"; 25 | public static final String coloredEdgeDir = "edge.color"; 26 | public static final String coloredAdjList = "adjList.color"; 27 | public static final String coloredPartitionDir = "partition.color"; 28 | 29 | public static final String hyperVertex = "hyper.vertex"; 30 | public static final String hyperEdge = "hyper.edge"; 31 | public static final String cliques = "hyper.cliques"; 32 | 33 | public static final String hyperGraphAdjList = "hyper.adjList"; 34 | public static final String hyperGraphBloomFilterDir = "hyper.bloomfilter"; 35 | 36 | public static final double overSizeRate = 0.1; 37 | 38 | public static final int NUMLONGBITS = Long.SIZE / Byte.SIZE; 39 | public static final int NUMINTBITS = Integer.SIZE / Byte.SIZE; 40 | 41 | public static final int SMALLSIGN = -2; 42 | public static final int LARGESIGN = -1; 43 | 44 | 45 | public static int TRIANGLE = 0; 46 | public static int SQUARE = 1; 47 | public static int FOURCLIQUE = 2; 48 | public static int FIVECLIQUE = 3; 49 | public static int ESQUARE = 4; 50 | 51 | public static int EDGE = 0; 52 | public static int TWINTWIG1 = 1; // X - W - Y (W < X < Y) 53 | public static int TWINTWIG2 = 2; // X - W - Y (X < W < Y) 54 | public static int TWINTWIG3 = 3; // X - W - Y (X < Y < W) 55 | public static int BF_TRIANGLE = 4; 56 | 57 | public static int HEAPINITSIZE = 20; 58 | 59 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageFourReducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import gnu.trove.set.hash.TLongHashSet; 4 | 5 | import java.io.IOException; 6 | import java.util.Arrays; 7 | 8 | import org.apache.hadoop.io.LongWritable; 9 | import org.apache.hadoop.mapreduce.Reducer; 10 | 11 | import dbg.hadoop.subgraphs.io.HVArray; 12 | import dbg.hadoop.subgraphs.io.HyperVertexSign; 13 | 14 | /** 15 | * Two different inputs will be received.
16 | * 1. ((u, -1); adj(u)): The adj list of u;
17 | * 2. ((u, -2); hyper(u)): The hyper vertex represented by u;
18 | * We guarantee that second input comes before the first one via secondarysort.
19 | * The second input only exits when u is the smallest vertex in the vertex set.
20 | * And if this is the case, we output (u; adj(u) \ hyper(u)); 21 | * @author robeen 22 | * 23 | */ 24 | public class HyperGraphStageFourReducer 25 | extends Reducer { 26 | 27 | public void reduce(HyperVertexSign _key, Iterable values, 28 | Context context) throws IOException, InterruptedException { 29 | // We do not receive the second 30 | boolean firstRecord = true; 31 | boolean isClique = (_key.getSign() == -2); 32 | TLongHashSet set = new TLongHashSet(); 33 | //HashMap map = new HashMap(); 34 | //System.out.println("input-key: " + _key); 35 | if(_key.getSign() == -1){ 36 | return; 37 | } 38 | long outputKey = 0; 39 | for(HVArray value: values){ 40 | if(firstRecord){ 41 | outputKey = _key.getVertex(); 42 | if(isClique){ 43 | for(long u : value.toArrays()){ 44 | if(u != _key.getVertex()){ 45 | //map.put(u, 0); 46 | set.add(u); 47 | } 48 | } 49 | } 50 | firstRecord = false; 51 | } 52 | else{ 53 | if(isClique){ 54 | int size = 0; 55 | long[] array = new long[value.size()]; 56 | for(long u : value.toArrays()){ 57 | if(!set.contains(u)){ 58 | array[size++] = u; 59 | } 60 | } 61 | context.write(new LongWritable(outputKey), 62 | new HVArray(Arrays.copyOf(array, size))); 63 | } 64 | else{ 65 | context.write(new LongWritable(outputKey), value); 66 | } 67 | } 68 | } 69 | set.clear(); 70 | set = null; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageThreeReducer.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.mapreduce.Reducer; 6 | 7 | import dbg.hadoop.subgraphs.io.HVArray; 8 | import dbg.hadoop.subgraphs.io.HyperVertexSign; 9 | import dbg.hadoop.subgraphs.utils.HyperVertex; 10 | 11 | 12 | /*** 13 | * This reducer deals with the inputs from both stage one and two.
14 | * There are two cases:
15 | * 16 | * CASE 1: The vertex has been assigned to a clique, or its neighbor
17 | * size is too large, not assign then. We directly output the key/value in this case.
18 | *
19 | * CASE 2: The vertex has been assigned to a vertexSet, or not assign
20 | * If assigned to a vertexSet, two records will be received, they are:
21 | * ((u, 0); set) and ((u, 2); {u}). By using secondary sort, it is guaranteed
22 | * that the first record comes before the second one. We output ((u, 0); set) then.
23 | * If assigned to itself, only ((u, 2); {u}) will be received, and we output ((u; 0); {u}). 24 | * 25 | * @author robeen 26 | * 27 | */ 28 | public class HyperGraphStageThreeReducer 29 | extends Reducer { 30 | 31 | public void reduce(HyperVertexSign _key, Iterable values, 32 | Context context) throws IOException, InterruptedException { 33 | // CASE 1 34 | if(_key.getSign() == 1 || _key.getSign() == 0){ 35 | for(HVArray value: values){ 36 | long hyperVertex = HyperVertex.get(_key.getVertex(), value.size(), (_key.getSign() == 1)); 37 | context.write(new HyperVertexSign(hyperVertex, _key.getSign()), value); 38 | return; 39 | } 40 | } 41 | // CASE 2 42 | else{ 43 | int size = 0; 44 | for(HVArray value: values){ 45 | if(value.size() > 1 && value.getFirst() != _key.getVertex()){ 46 | long hyperVertex = HyperVertex.get(_key.getVertex(), value.size(), (_key.getSign() == 1)); 47 | context.write(new HyperVertexSign(hyperVertex, _key.getSign()), value); 48 | return; 49 | } 50 | else{ 51 | ++size; 52 | } 53 | } 54 | if(size == 2){ 55 | long hyperVertex = HyperVertex.get(_key.getVertex(), 1, false); 56 | context.write(new HyperVertexSign(hyperVertex, 0), 57 | new HVArray(_key.getVertex())); 58 | } 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/qtcounter/QtS3Driver.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.qtcounter; 2 | 3 | import java.io.IOException; 4 | import java.net.URISyntaxException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.conf.Configured; 8 | import org.apache.hadoop.fs.Path; 9 | import org.apache.hadoop.io.LongWritable; 10 | import org.apache.hadoop.io.NullWritable; 11 | import org.apache.hadoop.mapred.JobConf; 12 | import org.apache.hadoop.mapreduce.Job; 13 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 14 | import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; 15 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 16 | import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 17 | //import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 18 | import org.apache.hadoop.util.Tool; 19 | 20 | 21 | public class QtS3Driver extends Configured implements Tool{ 22 | 23 | public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException, URISyntaxException { 24 | Configuration conf = getConf(); 25 | // The parameters: 26 | int numReducers = Integer.parseInt(args[2]); 27 | 28 | conf.setBoolean("mapred.compress.map.output", true); 29 | conf.set("mapred.map.output.compression.codec", "com.hadoop.compression.lzo.LzoCodec"); 30 | 31 | Job job = new Job(conf, "QtCounter Stage Three"); 32 | ((JobConf)job.getConfiguration()).setJar(args[3]); 33 | //JobConf job = new JobConf(getConf(), this.getClass()); 34 | 35 | job.setMapperClass(QtS3Mapper.class); 36 | job.setCombinerClass(QtS3Reducer.class); 37 | job.setReducerClass(QtS3Reducer.class); 38 | 39 | job.setOutputKeyClass(NullWritable.class); 40 | job.setOutputValueClass(LongWritable.class); 41 | 42 | job.setInputFormatClass(SequenceFileInputFormat.class); 43 | job.setOutputFormatClass(TextOutputFormat.class); 44 | //SequenceFileOutputFormat.setOutputCompressionType 45 | // (job, CompressionType.BLOCK); 46 | //SequenceFileOutputFormat.setOutputCompressorClass(job, LzoCodec.class); 47 | 48 | job.setNumReduceTasks(numReducers); 49 | 50 | FileInputFormat.setInputPaths(job, new Path(args[0])); 51 | FileOutputFormat.setOutputPath(job, new Path(args[1])); 52 | 53 | job.waitForCompletion(true); 54 | return 0; 55 | } 56 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/list/TLinkable.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.list; 20 | 21 | import java.io.Serializable; 22 | 23 | 24 | 25 | /** 26 | * Interface for Objects which can be inserted into a TLinkedList. 27 | * 28 | * @author Eric D. Friedman 29 | * @version $Id: TLinkable.java,v 1.1.2.2 2009/09/04 12:32:33 upholderoftruth Exp $ 30 | * @see gnu.trove.list.linked.TLinkedList 31 | */ 32 | 33 | public interface TLinkable extends Serializable { 34 | static final long serialVersionUID = 997545054865482562L; 35 | 36 | /** 37 | * Returns the linked list node after this one. 38 | * 39 | * @return a TLinkable value 40 | */ 41 | public T getNext(); 42 | 43 | 44 | /** 45 | * Returns the linked list node before this one. 46 | * 47 | * @return a TLinkable value 48 | */ 49 | public T getPrevious(); 50 | 51 | 52 | /** 53 | * Sets the linked list node after this one. 54 | * 55 | * @param linkable a TLinkable value 56 | */ 57 | public void setNext( T linkable ); 58 | 59 | 60 | /** 61 | * Sets the linked list node before this one. 62 | * 63 | * @param linkable a TLinkable value 64 | */ 65 | public void setPrevious( T linkable ); 66 | }// TLinkable 67 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/adjlist/GenAdjList.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph.adjlist; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.fs.Path; 5 | import org.apache.hadoop.util.ToolRunner; 6 | 7 | import dbg.hadoop.subgraphs.utils.Config; 8 | import dbg.hadoop.subgraphs.utils.Utility; 9 | import dbg.hadoop.subgraphs.utils.InputInfo; 10 | 11 | public class GenAdjList{ 12 | 13 | public static void main(String[] args) throws Exception { 14 | run(new InputInfo(args)); 15 | } 16 | 17 | public static void run(InputInfo inputInfo) throws Exception{ 18 | String numReducers = inputInfo.numReducers; 19 | String inputFile = inputInfo.inputFilePath; 20 | String jarFile = inputInfo.jarFile; 21 | int maxSize = inputInfo.maxSize; 22 | boolean isHyper = inputInfo.isHyper; 23 | String workDir = inputInfo.workDir; 24 | 25 | if(inputFile.isEmpty()){ 26 | System.err.println("Input file not specified!"); 27 | return; 28 | } 29 | 30 | if (workDir.toLowerCase().contains("hdfs")) { 31 | int pos = workDir.substring("hdfs://".length()).indexOf("/") 32 | + "hdfs://".length(); 33 | Utility.setDefaultFS(workDir.substring(0, pos)); 34 | } else { 35 | Utility.setDefaultFS(""); 36 | } 37 | 38 | Configuration conf = new Configuration(); 39 | conf.setInt("map.input.max.size", maxSize); 40 | 41 | String outputDir = workDir + Config.hyperGraphAdjList + "." + maxSize; 42 | String[] opts = { workDir + Config.hyperEdge, outputDir, numReducers, jarFile }; 43 | 44 | if (!isHyper) { 45 | outputDir = workDir + Config.adjListDir + "." + maxSize; 46 | opts[0] = workDir + Config.preparedFileDir; 47 | opts[1] = outputDir; 48 | } 49 | 50 | if(Utility.getFS().isDirectory(new Path(outputDir))) 51 | Utility.getFS().delete(new Path(outputDir)); 52 | if(Utility.getFS().isDirectory(new Path(outputDir + ".tmp"))) 53 | Utility.getFS().delete(new Path(outputDir + ".tmp")); 54 | 55 | if (maxSize == 0) { 56 | ToolRunner.run(conf, new GenAdjListDriver(), opts); 57 | } 58 | else{ 59 | opts[1] = outputDir + ".tmp"; 60 | ToolRunner.run(conf, new GenAdjListDriver(), opts); 61 | opts[0] = outputDir + ".tmp"; 62 | opts[1] = outputDir; 63 | ToolRunner.run(conf, new GenAdjListRandomShuffleDriver(), opts); 64 | 65 | Utility.getFS().delete(new Path(outputDir + ".tmp")); 66 | } 67 | } 68 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/strategy/HashingStrategy.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2002, Eric D. Friedman All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.strategy; 20 | 21 | import java.io.Serializable; 22 | 23 | 24 | /** 25 | * Interface to support pluggable hashing strategies in maps and sets. 26 | * Implementers can use this interface to make the trove hashing 27 | * algorithms use object values, values provided by the java runtime, 28 | * or a custom strategy when computing hashcodes. 29 | * 30 | * @author Eric Friedman 31 | * @author Rob Eden 32 | */ 33 | 34 | public interface HashingStrategy extends Serializable { 35 | static final long serialVersionUID = 5674097166776615540L; 36 | 37 | /** 38 | * Computes a hash code for the specified object. Implementers 39 | * can use the object's own hashCode method, the Java 40 | * runtime's identityHashCode, or a custom scheme. 41 | * 42 | * @param object for which the hashcode is to be computed 43 | * @return the hashCode 44 | */ 45 | int computeHashCode( T object ); 46 | 47 | /** 48 | * Compares o1 and o2 for equality. Strategy implementers may use 49 | * the objects' own equals() methods, compare object references, 50 | * or implement some custom scheme. 51 | * 52 | * @param o1 an Object value 53 | * @param o2 an Object value 54 | * @return true if the objects are equal according to this strategy. 55 | */ 56 | boolean equals( T o1, T o2 ); 57 | } 58 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/triangle/Triangle.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph.triangle; 2 | 3 | import java.net.URI; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.filecache.DistributedCache; 7 | import org.apache.hadoop.fs.Path; 8 | import org.apache.hadoop.util.ToolRunner; 9 | 10 | import dbg.hadoop.subgraphs.utils.Config; 11 | import dbg.hadoop.subgraphs.utils.Utility; 12 | import dbg.hadoop.subgraphs.utils.InputInfo; 13 | 14 | public class Triangle{ 15 | 16 | public static void main(String[] args) throws Exception{ 17 | run(new InputInfo(args)); 18 | } 19 | 20 | public static void run(InputInfo inputInfo) throws Exception{ 21 | String inputFilePath = inputInfo.inputFilePath; 22 | float falsePositive = inputInfo.falsePositive; 23 | boolean enableBF = true; 24 | boolean isHyper = inputInfo.isHyper; 25 | int maxSize = inputInfo.maxSize; 26 | 27 | String jarFile = inputInfo.jarFile; 28 | String numReducers = inputInfo.numReducers; 29 | String workDir = inputInfo.workDir; 30 | 31 | if(inputFilePath.isEmpty()){ 32 | System.err.println("Input file not specified!"); 33 | System.exit(-1);; 34 | } 35 | 36 | if (workDir.toLowerCase().contains("hdfs")) { 37 | int pos = workDir.substring("hdfs://".length()).indexOf("/") 38 | + "hdfs://".length(); 39 | Utility.setDefaultFS(workDir.substring(0, pos)); 40 | } else { 41 | Utility.setDefaultFS(""); 42 | } 43 | 44 | String output = workDir + "triangle.res"; 45 | 46 | if(Utility.getFS().isDirectory(new Path(output))){ 47 | Utility.getFS().delete(new Path(output)); 48 | } 49 | 50 | Configuration conf = new Configuration(); 51 | conf.setBoolean("enable.bloom.filter", enableBF); 52 | conf.setFloat("bloom.filter.false.positive.rate", (float)falsePositive); 53 | 54 | if(enableBF){ 55 | String bloomFilterFileName = "bloomFilter." + Config.EDGE + "." + falsePositive; 56 | DistributedCache.addCacheFile(new URI(new Path(workDir).toUri() 57 | .toString() + "/" + Config.bloomFilterFileDir + "/" + bloomFilterFileName), conf); 58 | } 59 | 60 | String adjListDir = isHyper ? workDir + Config.hyperGraphAdjList + "." + maxSize : 61 | workDir + Config.adjListDir + "." + maxSize; 62 | 63 | // The parameters: 64 | String opts[] = { workDir + Config.preparedFileDir, adjListDir, 65 | output, numReducers, jarFile}; 66 | 67 | ToolRunner.run(conf, new TriangleDriver(), opts); 68 | } 69 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/Version.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // Copyright (c) 2009, Rob Eden All Rights Reserved. 4 | // Copyright (c) 2009, Jeff Randall All Rights Reserved. 5 | // 6 | // This library is free software; you can redistribute it and/or 7 | // modify it under the terms of the GNU Lesser General Public 8 | // License as published by the Free Software Foundation; either 9 | // version 2.1 of the License, or (at your option) any later version. 10 | // 11 | // This library is distributed in the hope that it will be useful, 12 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | // GNU General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Lesser General Public 17 | // License along with this program; if not, write to the Free Software 18 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | /////////////////////////////////////////////////////////////////////////////// 20 | 21 | package gnu.trove; 22 | 23 | /** 24 | * Simple class meant as a possible main class (via manifest) to report the 25 | * implementation version of the trove4j jar. 26 | *

27 | * This may be useful to ask feedback WITH build version information 28 | *

29 | * The Main-Class entry in the manifest.mf should be set during the build as well 30 | * as the Implementation-Version manifest attribute should be set as well. 31 | *

32 | * Created by IntelliJ IDEA. 33 | * User: Johan Parent 34 | * Date: 3/03/11 35 | * Time: 22:10 36 | */ 37 | public class Version { 38 | public static void main(String[] args) { 39 | System.out.println(getVersion()); 40 | } 41 | 42 | /** 43 | * Returns the implementation version of trove4j. Intended for applications 44 | * wanting to return the version of trove4j they are using 45 | *

46 | * NOTE: this method will only return a useful version when working 47 | * with a trove4j jar as it requires a manifest file 48 | * 49 | * @return 50 | */ 51 | public static String getVersion() { 52 | String version = Version.class.getPackage().getImplementationVersion(); 53 | // 54 | if (version != null) { 55 | return "trove4j version " + version; 56 | } 57 | 58 | return "Sorry no Implementation-Version manifest attribute available"; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/iterator/TPrimitiveIterator.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // Copyright (c) 2009, Rob Eden All Rights Reserved. 4 | // Copyright (c) 2009, Jeff Randall All Rights Reserved. 5 | // 6 | // This library is free software; you can redistribute it and/or 7 | // modify it under the terms of the GNU Lesser General Public 8 | // License as published by the Free Software Foundation; either 9 | // version 2.1 of the License, or (at your option) any later version. 10 | // 11 | // This library is distributed in the hope that it will be useful, 12 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | // GNU General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Lesser General Public 17 | // License along with this program; if not, write to the Free Software 18 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | /////////////////////////////////////////////////////////////////////////////// 20 | 21 | package gnu.trove.iterator; 22 | 23 | /** 24 | * Implements all iterator functions for the hashed object set. 25 | * Subclasses may override objectAtIndex to vary the object 26 | * returned by calls to next() (e.g. for values, and Map.Entry 27 | * objects). 28 | *

29 | *

Note that iteration is fastest if you forego the calls to 30 | * hasNext in favor of checking the size of the structure 31 | * yourself and then call next() that many times: 32 | *

33 | * 

34 |  * Iterator i = collection.iterator();
35 |  * for (int size = collection.size(); size-- > 0;) {
36 |  *   Object o = i.next();
37 |  * }
38 |  *
39 | *

40 | *

You may, of course, use the hasNext(), next() idiom too if 41 | * you aren't in a performance critical spot.

42 | */ 43 | public interface TPrimitiveIterator extends TIterator { 44 | /** 45 | * Returns true if the iterator can be advanced past its current 46 | * location. 47 | * 48 | * @return a boolean value 49 | */ 50 | public boolean hasNext(); 51 | 52 | 53 | /** 54 | * Removes the last entry returned by the iterator. 55 | * Invoking this method more than once for a single entry 56 | * will leave the underlying data structure in a confused 57 | * state. 58 | */ 59 | public void remove(); 60 | 61 | } // TPrimitiveIterator 62 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/maximalclique/MCliqueS5Driver.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.maximalclique; 2 | 3 | import java.io.IOException; 4 | import java.net.URISyntaxException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.conf.Configured; 8 | import org.apache.hadoop.fs.Path; 9 | import org.apache.hadoop.io.LongWritable; 10 | import org.apache.hadoop.io.SequenceFile.CompressionType; 11 | import org.apache.hadoop.mapred.JobConf; 12 | import org.apache.hadoop.mapreduce.Job; 13 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 14 | import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; 15 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 16 | import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; 17 | //import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 18 | import org.apache.hadoop.util.Tool; 19 | 20 | import com.hadoop.compression.lzo.LzoCodec; 21 | 22 | import dbg.hadoop.subgraphs.io.HyperVertexComparator; 23 | 24 | public class MCliqueS5Driver extends Configured implements Tool{ 25 | 26 | public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException, URISyntaxException { 27 | Configuration conf = getConf(); 28 | // The parameters: 29 | int numReducers = Integer.parseInt(args[2]); 30 | 31 | conf.setBoolean("mapreduce.map.output.compress", true); 32 | conf.set("mapreduce.map.output.compress.codec", "com.hadoop.compression.lzo.LzoCodec"); 33 | 34 | Job job = new Job(conf, "Maximal Clique Stage Five"); 35 | ((JobConf)job.getConfiguration()).setJar(args[3]); 36 | //JobConf job = new JobConf(getConf(), this.getClass()); 37 | 38 | job.setMapperClass(MCliqueS5Mapper.class); 39 | job.setCombinerClass(MCliqueS5Reducer.class); 40 | job.setReducerClass(MCliqueS5Reducer.class); 41 | 42 | job.setOutputKeyClass(LongWritable.class); 43 | job.setOutputValueClass(LongWritable.class); 44 | 45 | job.setSortComparatorClass(HyperVertexComparator.class); 46 | 47 | job.setInputFormatClass(SequenceFileInputFormat.class); 48 | job.setOutputFormatClass(SequenceFileOutputFormat.class); 49 | SequenceFileOutputFormat.setOutputCompressionType 50 | (job, CompressionType.BLOCK); 51 | SequenceFileOutputFormat.setOutputCompressorClass(job, LzoCodec.class); 52 | 53 | job.setNumReduceTasks(numReducers); 54 | 55 | FileInputFormat.setInputPaths(job, new Path(args[0])); 56 | FileOutputFormat.setOutputPath(job, new Path(args[1])); 57 | 58 | job.waitForCompletion(true); 59 | return 0; 60 | } 61 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/queue/TIntQueue.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2009, Rob Eden All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.queue; 20 | 21 | 22 | import gnu.trove.TIntCollection; 23 | 24 | import java.io.Serializable; 25 | 26 | /** 27 | * Interface for Trove queue implementations. 28 | * 29 | * @see java.util.Queue 30 | */ 31 | public interface TIntQueue extends TIntCollection { 32 | /** 33 | * Retrieves and removes the head of this queue. This method differs from 34 | * {@link #poll} only in that it throws an exception if this queue is empty. 35 | */ 36 | public int element(); 37 | 38 | 39 | /** 40 | * Inserts the specified element into this queue if it is possible to do so 41 | * immediately without violating capacity restrictions. When using a 42 | * capacity-restricted queue, this method is generally preferable to 43 | * {@link #add}, which can fail to insert an element only by throwing an exception. 44 | * 45 | * @param e The element to add. 46 | * 47 | * @return true if the element was added to this queue, else false 48 | */ 49 | public boolean offer( int e ); 50 | 51 | 52 | /** 53 | * Retrieves, but does not remove, the head of this queue, or returns 54 | * {@link #getNoEntryValue} if this queue is empty. 55 | * 56 | * @return the head of this queue, or {@link #getNoEntryValue} if this queue is empty 57 | */ 58 | public int peek(); 59 | 60 | 61 | /** 62 | * Retrieves and removes the head of this queue, or returns {@link #getNoEntryValue} 63 | * if this queue is empty. 64 | * 65 | * @return the head of this queue, or {@link #getNoEntryValue} if this queue is empty 66 | */ 67 | public int poll(); 68 | } 69 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/queue/TLongQueue.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2009, Rob Eden All Rights Reserved. 3 | // 4 | // This library is free software; you can redistribute it and/or 5 | // modify it under the terms of the GNU Lesser General Public 6 | // License as published by the Free Software Foundation; either 7 | // version 2.1 of the License, or (at your option) any later version. 8 | // 9 | // This library is distributed in the hope that it will be useful, 10 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | // GNU General Public License for more details. 13 | // 14 | // You should have received a copy of the GNU Lesser General Public 15 | // License along with this program; if not, write to the Free Software 16 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 17 | /////////////////////////////////////////////////////////////////////////////// 18 | 19 | package gnu.trove.queue; 20 | 21 | 22 | import gnu.trove.TLongCollection; 23 | 24 | import java.io.Serializable; 25 | 26 | /** 27 | * Interface for Trove queue implementations. 28 | * 29 | * @see java.util.Queue 30 | */ 31 | public interface TLongQueue extends TLongCollection { 32 | /** 33 | * Retrieves and removes the head of this queue. This method differs from 34 | * {@link #poll} only in that it throws an exception if this queue is empty. 35 | */ 36 | public long element(); 37 | 38 | 39 | /** 40 | * Inserts the specified element into this queue if it is possible to do so 41 | * immediately without violating capacity restrictions. When using a 42 | * capacity-restricted queue, this method is generally preferable to 43 | * {@link #add}, which can fail to insert an element only by throwing an exception. 44 | * 45 | * @param e The element to add. 46 | * 47 | * @return true if the element was added to this queue, else false 48 | */ 49 | public boolean offer( long e ); 50 | 51 | 52 | /** 53 | * Retrieves, but does not remove, the head of this queue, or returns 54 | * {@link #getNoEntryValue} if this queue is empty. 55 | * 56 | * @return the head of this queue, or {@link #getNoEntryValue} if this queue is empty 57 | */ 58 | public long peek(); 59 | 60 | 61 | /** 62 | * Retrieves and removes the head of this queue, or returns {@link #getNoEntryValue} 63 | * if this queue is empty. 64 | * 65 | * @return the head of this queue, or {@link #getNoEntryValue} if this queue is empty 66 | */ 67 | public long poll(); 68 | } 69 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/adjlist/GenAdjListDriver.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph.adjlist; 2 | 3 | import java.io.IOException; 4 | import java.net.URISyntaxException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.conf.Configured; 8 | import org.apache.hadoop.fs.Path; 9 | import org.apache.hadoop.io.LongWritable; 10 | import org.apache.hadoop.io.SequenceFile.CompressionType; 11 | import org.apache.hadoop.mapred.JobConf; 12 | import org.apache.hadoop.mapreduce.Job; 13 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 14 | import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; 15 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 16 | import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; 17 | //import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 18 | import org.apache.hadoop.util.Tool; 19 | 20 | import com.hadoop.compression.lzo.LzoCodec; 21 | 22 | import dbg.hadoop.subgraphs.io.HyperVertexAdjList; 23 | import dbg.hadoop.subgraphs.io.HyperVertexComparator; 24 | 25 | public class GenAdjListDriver extends Configured implements Tool{ 26 | 27 | public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException, URISyntaxException { 28 | Configuration conf = getConf(); 29 | // The parameters: 30 | int numReducers = Integer.parseInt(args[2]); 31 | 32 | conf.setBoolean("mapreduce.map.output.compress", true); 33 | conf.set("mapreduce.map.output.compress.codec", "com.hadoop.compression.lzo.LzoCodec"); 34 | 35 | Job job = new Job(conf, "HyperGraphGenAdjList"); 36 | ((JobConf)job.getConfiguration()).setJar(args[3]); 37 | 38 | job.setMapperClass(GenAdjListMapper.class); 39 | job.setReducerClass(GenAdjListReducer.class); 40 | 41 | job.setSortComparatorClass(HyperVertexComparator.class); 42 | 43 | job.setMapOutputKeyClass(LongWritable.class); 44 | job.setMapOutputValueClass(LongWritable.class); 45 | job.setOutputKeyClass(LongWritable.class); 46 | job.setOutputValueClass(HyperVertexAdjList.class); 47 | 48 | job.setInputFormatClass(SequenceFileInputFormat.class); 49 | 50 | job.setOutputFormatClass(SequenceFileOutputFormat.class); 51 | SequenceFileOutputFormat.setOutputCompressionType 52 | (job, CompressionType.BLOCK); 53 | SequenceFileOutputFormat.setOutputCompressorClass(job, LzoCodec.class); 54 | 55 | job.setNumReduceTasks(numReducers); 56 | 57 | FileInputFormat.setInputPaths(job, new Path(args[0])); 58 | FileOutputFormat.setOutputPath(job, new Path(args[1])); 59 | 60 | job.waitForCompletion(true); 61 | return 0; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageOneDriver.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import java.io.IOException; 4 | import java.net.URISyntaxException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.conf.Configured; 8 | import org.apache.hadoop.fs.Path; 9 | import org.apache.hadoop.io.LongWritable; 10 | import org.apache.hadoop.io.SequenceFile.CompressionType; 11 | import org.apache.hadoop.mapred.JobConf; 12 | import org.apache.hadoop.mapreduce.Job; 13 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 14 | import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; 15 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 16 | import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; 17 | //import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 18 | import org.apache.hadoop.util.Tool; 19 | 20 | import com.hadoop.compression.lzo.LzoCodec; 21 | 22 | import dbg.hadoop.subgraphs.io.HVArray; 23 | import dbg.hadoop.subgraphs.io.HVArrayComparator; 24 | import dbg.hadoop.subgraphs.io.HyperVertexSign; 25 | 26 | public class HyperGraphStageOneDriver extends Configured implements Tool{ 27 | 28 | public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException, URISyntaxException { 29 | Configuration conf = getConf(); 30 | // The parameters: 31 | int numReducers = Integer.parseInt(args[2]); 32 | 33 | conf.setBoolean("mapred.compress.map.output", true); 34 | conf.set("mapred.map.output.compression.codec", "com.hadoop.compression.lzo.LzoCodec"); 35 | 36 | Job job = new Job(conf, "HyperGraphStageOne"); 37 | ((JobConf)job.getConfiguration()).setJar(args[3]); 38 | //JobConf job = new JobConf(getConf(), this.getClass()); 39 | 40 | job.setMapperClass(HyperGraphStageOneMapper.class); 41 | job.setReducerClass(HyperGraphStageOneReducer.class); 42 | 43 | job.setMapOutputKeyClass(HVArray.class); 44 | job.setMapOutputValueClass(LongWritable.class); 45 | job.setOutputKeyClass(HyperVertexSign.class); 46 | job.setOutputValueClass(HVArray.class); 47 | 48 | job.setSortComparatorClass(HVArrayComparator.class); 49 | 50 | job.setInputFormatClass(SequenceFileInputFormat.class); 51 | //if(enableOutputCompress){ 52 | job.setOutputFormatClass(SequenceFileOutputFormat.class); 53 | SequenceFileOutputFormat.setOutputCompressionType 54 | (job, CompressionType.BLOCK); 55 | SequenceFileOutputFormat.setOutputCompressorClass(job, LzoCodec.class); 56 | job.setNumReduceTasks(numReducers); 57 | 58 | FileInputFormat.setInputPaths(job, new Path(args[0])); 59 | FileOutputFormat.setOutputPath(job, new Path(args[1])); 60 | 61 | job.waitForCompletion(true); 62 | return 0; 63 | } 64 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageTwoDriver.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import java.io.IOException; 4 | import java.net.URISyntaxException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.conf.Configured; 8 | import org.apache.hadoop.fs.Path; 9 | import org.apache.hadoop.io.LongWritable; 10 | import org.apache.hadoop.io.SequenceFile.CompressionType; 11 | import org.apache.hadoop.mapred.JobConf; 12 | import org.apache.hadoop.mapreduce.Job; 13 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 14 | import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; 15 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 16 | import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; 17 | //import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 18 | import org.apache.hadoop.util.Tool; 19 | 20 | import com.hadoop.compression.lzo.LzoCodec; 21 | 22 | import dbg.hadoop.subgraphs.io.HVArray; 23 | import dbg.hadoop.subgraphs.io.HVArrayComparator; 24 | import dbg.hadoop.subgraphs.io.HyperVertexSign; 25 | 26 | public class HyperGraphStageTwoDriver extends Configured implements Tool{ 27 | 28 | public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException, URISyntaxException { 29 | Configuration conf = getConf(); 30 | // The parameters: 31 | int numReducers = Integer.parseInt(args[2]); 32 | 33 | conf.setBoolean("mapred.compress.map.output", true); 34 | conf.set("mapred.map.output.compression.codec", "com.hadoop.compression.lzo.LzoCodec"); 35 | 36 | Job job = new Job(conf, "HyperGraphStageTwo"); 37 | ((JobConf)job.getConfiguration()).setJar(args[3]); 38 | //JobConf job = new JobConf(getConf(), this.getClass()); 39 | 40 | job.setMapperClass(HyperGraphStageTwoMapper.class); 41 | job.setReducerClass(HyperGraphStageTwoReducer.class); 42 | 43 | job.setMapOutputKeyClass(HVArray.class); 44 | job.setMapOutputValueClass(LongWritable.class); 45 | job.setOutputKeyClass(HyperVertexSign.class); 46 | job.setOutputValueClass(HVArray.class); 47 | 48 | job.setSortComparatorClass(HVArrayComparator.class); 49 | 50 | job.setInputFormatClass(SequenceFileInputFormat.class); 51 | //if(enableOutputCompress){ 52 | job.setOutputFormatClass(SequenceFileOutputFormat.class); 53 | SequenceFileOutputFormat.setOutputCompressionType 54 | (job, CompressionType.BLOCK); 55 | SequenceFileOutputFormat.setOutputCompressorClass(job, LzoCodec.class); 56 | job.setNumReduceTasks(numReducers); 57 | 58 | FileInputFormat.setInputPaths(job, new Path(args[0])); 59 | FileOutputFormat.setOutputPath(job, new Path(args[1])); 60 | 61 | job.waitForCompletion(true); 62 | return 0; 63 | } 64 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/hypergraph/HyperGraphStageSixDriver.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.hypergraph; 2 | 3 | import java.io.IOException; 4 | import java.net.URISyntaxException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.conf.Configured; 8 | import org.apache.hadoop.fs.Path; 9 | import org.apache.hadoop.io.LongWritable; 10 | import org.apache.hadoop.io.NullWritable; 11 | import org.apache.hadoop.io.SequenceFile.CompressionType; 12 | import org.apache.hadoop.mapred.JobConf; 13 | import org.apache.hadoop.mapreduce.Job; 14 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 15 | import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; 16 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 17 | import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; 18 | //import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 19 | import org.apache.hadoop.util.Tool; 20 | 21 | import com.hadoop.compression.lzo.LzoCodec; 22 | 23 | import dbg.hadoop.subgraphs.io.HVArray; 24 | import dbg.hadoop.subgraphs.io.HVArrayComparator; 25 | 26 | public class HyperGraphStageSixDriver extends Configured implements Tool{ 27 | 28 | public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException, URISyntaxException { 29 | Configuration conf = getConf(); 30 | // The parameters: 31 | int numReducers = Integer.parseInt(args[2]); 32 | 33 | conf.setBoolean("mapred.compress.map.output", true); 34 | conf.set("mapred.map.output.compression.codec", "com.hadoop.compression.lzo.LzoCodec"); 35 | 36 | Job job = new Job(conf, "HyperGraphStageSix"); 37 | ((JobConf)job.getConfiguration()).setJar(args[3]); 38 | //JobConf job = new JobConf(getConf(), this.getClass()); 39 | 40 | job.setMapperClass(HyperGraphStageSixMapper.class); 41 | job.setCombinerClass(HyperGraphStageSixCombiner.class); 42 | job.setReducerClass(HyperGraphStageSixReducer.class); 43 | 44 | job.setMapOutputKeyClass(HVArray.class); 45 | job.setMapOutputValueClass(NullWritable.class); 46 | job.setOutputKeyClass(LongWritable.class); 47 | job.setOutputValueClass(LongWritable.class); 48 | 49 | job.setSortComparatorClass(HVArrayComparator.class); 50 | 51 | job.setInputFormatClass(SequenceFileInputFormat.class); 52 | //if(enableOutputCompress){ 53 | job.setOutputFormatClass(SequenceFileOutputFormat.class); 54 | SequenceFileOutputFormat.setOutputCompressionType 55 | (job, CompressionType.BLOCK); 56 | SequenceFileOutputFormat.setOutputCompressorClass(job, LzoCodec.class); 57 | job.setNumReduceTasks(numReducers); 58 | 59 | FileInputFormat.setInputPaths(job, new Path(args[0])); 60 | FileOutputFormat.setOutputPath(job, new Path(args[1])); 61 | 62 | job.waitForCompletion(true); 63 | return 0; 64 | } 65 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgraphs/utils/bloomfilter/utils/BloomFilterSerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package dbg.hadoop.subgraphs.utils.bloomfilter.utils; 19 | 20 | import java.io.DataInput; 21 | import java.io.DataOutput; 22 | import java.io.IOException; 23 | 24 | import dbg.hadoop.subgraphs.utils.bloomfilter.db.TypeSizes; 25 | import dbg.hadoop.subgraphs.utils.bloomfilter.io.ISerializer; 26 | import dbg.hadoop.subgraphs.utils.bloomfilter.obs.IBitSet; 27 | import dbg.hadoop.subgraphs.utils.bloomfilter.obs.OffHeapBitSet; 28 | //import dbg.hadoop.subgraphs.utils.bloomfilter.OffHeapBitSet; 29 | import dbg.hadoop.subgraphs.utils.bloomfilter.obs.OpenBitSet; 30 | 31 | abstract class BloomFilterSerializer implements ISerializer 32 | { 33 | public void serialize(BloomFilter bf, DataOutput out) throws IOException 34 | { 35 | out.writeInt(bf.hashCount); 36 | bf.bitset.serialize(out); 37 | } 38 | 39 | public BloomFilter deserialize(DataInput in) throws IOException 40 | { 41 | return deserialize(in, false); 42 | } 43 | 44 | public BloomFilter deserialize(DataInput in, boolean offheap) throws IOException 45 | { 46 | int hashes = in.readInt(); 47 | IBitSet bs = offheap ? OffHeapBitSet.deserialize(in) : OpenBitSet.deserialize(in); 48 | //IBitSet bs = OpenBitSet.deserialize(in); 49 | return createFilter(hashes, bs); 50 | } 51 | 52 | protected abstract BloomFilter createFilter(int hashes, IBitSet bs); 53 | 54 | /** 55 | * Calculates a serialized size of the given Bloom Filter 56 | * @see BloomFilterSerializer#serialize(BloomFilter, DataOutput) 57 | * 58 | * @param bf Bloom filter to calculate serialized size 59 | * 60 | * @return serialized size of the given bloom filter 61 | */ 62 | public long serializedSize(BloomFilter bf, TypeSizes typeSizes) 63 | { 64 | int size = typeSizes.sizeof(bf.hashCount); // hash count 65 | size += bf.bitset.serializedSize(typeSizes); 66 | return size; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/maximalclique/MCliqueS4Driver.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.maximalclique; 2 | 3 | import java.io.IOException; 4 | import java.net.URISyntaxException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.conf.Configured; 8 | import org.apache.hadoop.fs.Path; 9 | import org.apache.hadoop.io.LongWritable; 10 | import org.apache.hadoop.io.NullWritable; 11 | import org.apache.hadoop.io.SequenceFile.CompressionType; 12 | import org.apache.hadoop.io.Text; 13 | import org.apache.hadoop.mapred.JobConf; 14 | import org.apache.hadoop.mapreduce.Job; 15 | import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; 16 | import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; 17 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 18 | import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; 19 | import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 20 | //import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 21 | import org.apache.hadoop.util.Tool; 22 | 23 | import com.hadoop.compression.lzo.LzoCodec; 24 | 25 | import dbg.hadoop.subgraphs.io.HVArray; 26 | import dbg.hadoop.subgraphs.io.HyperVertexComparator; 27 | 28 | public class MCliqueS4Driver extends Configured implements Tool{ 29 | 30 | public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException, URISyntaxException { 31 | Configuration conf = getConf(); 32 | // The parameters: 33 | int numReducers = Integer.parseInt(args[2]); 34 | 35 | conf.setBoolean("mapreduce.map.output.compress", true); 36 | conf.set("mapreduce.map.output.compress.codec", "com.hadoop.compression.lzo.LzoCodec"); 37 | 38 | Job job = new Job(conf, "Maximal Clique Stage Four"); 39 | ((JobConf)job.getConfiguration()).setJar(args[3]); 40 | //JobConf job = new JobConf(getConf(), this.getClass()); 41 | 42 | job.setMapperClass(MCliqueS4Mapper.class); 43 | job.setReducerClass(MCliqueS4Reducer.class); 44 | 45 | job.setMapOutputKeyClass(LongWritable.class); 46 | job.setMapOutputValueClass(LongWritable.class); 47 | //job.setOutputKeyClass(LongWritable.class); 48 | job.setOutputKeyClass(LongWritable.class); 49 | job.setOutputValueClass(HVArray.class); 50 | 51 | job.setSortComparatorClass(HyperVertexComparator.class); 52 | 53 | job.setInputFormatClass(SequenceFileInputFormat.class); 54 | //job.setOutputFormatClass(TextOutputFormat.class); 55 | job.setOutputFormatClass(SequenceFileOutputFormat.class); 56 | SequenceFileOutputFormat.setOutputCompressionType 57 | (job, CompressionType.BLOCK); 58 | SequenceFileOutputFormat.setOutputCompressorClass(job, LzoCodec.class); 59 | 60 | job.setNumReduceTasks(numReducers); 61 | 62 | FileInputFormat.setInputPaths(job, new Path(args[0])); 63 | FileOutputFormat.setOutputPath(job, new Path(args[1])); 64 | 65 | job.waitForCompletion(true); 66 | return 0; 67 | } 68 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/maximalclique/MCliqueS1Mapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.maximalclique; 2 | 3 | import gnu.trove.list.array.TLongArrayList; 4 | 5 | import java.io.IOException; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | import org.apache.hadoop.io.LongWritable; 9 | import org.apache.hadoop.mapreduce.Mapper; 10 | 11 | import dbg.hadoop.subgraphs.io.HVArraySign; 12 | import dbg.hadoop.subgraphs.io.HyperVertexAdjList; 13 | import dbg.hadoop.subgraphs.utils.BloomFilterOpr; 14 | import dbg.hadoop.subgraphs.utils.Config; 15 | import dbg.hadoop.subgraphs.utils.HyperVertex; 16 | 17 | 18 | class MCliqueS1Mapper extends 19 | Mapper { 20 | 21 | private static boolean enableBF; 22 | private static BloomFilterOpr bloomfilterOpr = null; 23 | private static int cliqueSizeThresh = 20; 24 | private static TLongArrayList list = null; 25 | 26 | // The hypervertex set 27 | @Override 28 | public void map(LongWritable key, HyperVertexAdjList value, Context context) 29 | throws IOException, InterruptedException { 30 | 31 | if(HyperVertex.Degree(key.get()) < cliqueSizeThresh) { 32 | return; 33 | } 34 | 35 | list.clear(); 36 | long largerThanCur[] = value.getLargeDegreeVertices(); 37 | for(int i = 0; i < largerThanCur.length; ++i) { 38 | if(HyperVertex.Degree(largerThanCur[i]) >= cliqueSizeThresh){ 39 | list.add(largerThanCur[i]); 40 | } 41 | } 42 | long v1 = 0, v2 = 0; 43 | int size = largerThanCur.length; 44 | boolean output = true; 45 | 46 | for (int i = 0; i < size - 1; ++i) { 47 | v1 = largerThanCur[i]; 48 | if(HyperVertex.Degree(v1) < cliqueSizeThresh) { 49 | continue; 50 | } 51 | for (int j = i + 1; j < size; ++j) { 52 | v2 = largerThanCur[j]; 53 | if(HyperVertex.Degree(v2) < cliqueSizeThresh) { 54 | continue; 55 | } 56 | if(enableBF) 57 | output = bloomfilterOpr.get().test(HyperVertex.VertexID(v1), 58 | HyperVertex.VertexID(v2)); 59 | if (output) { 60 | context.write(new HVArraySign(v1, v2, Config.LARGESIGN), 61 | key); 62 | } 63 | } 64 | } 65 | } 66 | 67 | @Override 68 | public void setup(Context context) throws IOException { 69 | Configuration conf = context.getConfiguration(); 70 | list = new TLongArrayList(); 71 | cliqueSizeThresh = conf.getInt("mapred.clique.size.threshold", 20); 72 | enableBF = conf.getBoolean("enable.bloom.filter", true); 73 | // We use bloomfilter as static. If it is already loaded, we will have 74 | // bloomfilterOpr != null, and we donot load it again in the case. 75 | 76 | if (enableBF && bloomfilterOpr == null) { 77 | bloomfilterOpr = new BloomFilterOpr(conf.getFloat( 78 | "bloom.filter.false.positive.rate", (float) 0.001)); 79 | try { 80 | bloomfilterOpr.obtainBloomFilter(conf); 81 | assert(bloomfilterOpr.get() != null); 82 | } catch (ClassNotFoundException e) { 83 | e.printStackTrace(); 84 | } 85 | } 86 | } 87 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/maximalclique/MCliqueS5Mapper.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.maximalclique; 2 | 3 | import gnu.trove.set.hash.TLongHashSet; 4 | 5 | import java.io.IOException; 6 | 7 | import org.apache.hadoop.io.LongWritable; 8 | import org.apache.hadoop.mapreduce.Mapper; 9 | 10 | import dbg.hadoop.subgraphs.io.HyperVertexAdjList; 11 | import dbg.hadoop.subgraphs.utils.HyperVertex; 12 | import dbg.hadoop.subgraphs.utils.Utility; 13 | 14 | /** 15 | * @author robeen 16 | * 17 | */ 18 | public class MCliqueS5Mapper 19 | extends Mapper { 20 | 21 | private static TLongHashSet outputSet; 22 | @Override 23 | public void map(LongWritable key, HyperVertexAdjList adjList, Context context) 24 | throws IOException, InterruptedException{ 25 | // If current vertex belong to one clique, 26 | // replace it with the clique representative vertex 27 | outputSet.clear(); 28 | 29 | long cur = Utility.cliqueMap.contains(key.get()) ? Utility.cliqueMap 30 | .get(key.get()) : key.get(); 31 | for(long v : adjList.getLargeDegreeVertices()){ 32 | long newV = Utility.cliqueMap.contains(v) ? Utility.cliqueMap.get(v) : v; 33 | writeOutput(cur, newV, context); 34 | } 35 | 36 | for(long v : adjList.getSmallDegreeVerticesGroup1()){ 37 | long newV = Utility.cliqueMap.contains(v) ? Utility.cliqueMap.get(v) : v; 38 | writeOutput(cur, newV, context); 39 | } 40 | } 41 | 42 | private static void writeOutput(long cur, long newV, Context context) 43 | throws IOException, InterruptedException{ 44 | if(cur == newV){ 45 | return; 46 | } 47 | else{ 48 | boolean reverse = (HyperVertex.compare(cur, newV) < 0) ? false : true; 49 | long key2Check = reverse ? getKey(newV, cur) : getKey(cur, newV); 50 | LongWritable output[] = new LongWritable[2]; 51 | if(!outputSet.contains(key2Check)){ 52 | if(reverse){ 53 | output[0] = new LongWritable(newV); 54 | output[1] = new LongWritable(cur); 55 | } 56 | else{ 57 | output[0] = new LongWritable(cur); 58 | output[1] = new LongWritable(newV); 59 | } 60 | outputSet.add(key2Check); 61 | context.write(output[0], output[1]); 62 | } 63 | } 64 | } 65 | 66 | private static long getKey(long v1, long v2){ 67 | long res = (long)HyperVertex.VertexID(v1); 68 | res = res << 32 | (long)(HyperVertex.VertexID(v2)); 69 | return res; 70 | } 71 | 72 | @Override 73 | public void setup(Context context){ 74 | outputSet = new TLongHashSet(); 75 | if(Utility.cliqueMap == null){ 76 | try { 77 | Utility.readCliques(context.getConfiguration(), true); 78 | } catch (InstantiationException e) { 79 | e.printStackTrace(); 80 | } catch (IllegalAccessException e) { 81 | e.printStackTrace(); 82 | } catch (IOException e) { 83 | e.printStackTrace(); 84 | } 85 | } 86 | } 87 | 88 | @Override 89 | public void cleanup(Context context){ 90 | outputSet.clear(); 91 | outputSet = null; 92 | } 93 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/prepare/PrepareDataS2Driver.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.prepare; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.conf.Configured; 7 | import org.apache.hadoop.fs.Path; 8 | import org.apache.hadoop.io.LongWritable; 9 | import org.apache.hadoop.mapred.JobConf; 10 | import org.apache.hadoop.mapreduce.Job; 11 | import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; 12 | import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; 13 | import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; 14 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 15 | import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; 16 | import org.apache.hadoop.util.Tool; 17 | 18 | import dbg.hadoop.subgraphs.io.DoubleIntegerPairComparator; 19 | import dbg.hadoop.subgraphs.io.DoubleIntegerPairGroupComparator; 20 | import dbg.hadoop.subgraphs.io.DoubleIntegerPairPartitioner2; 21 | import dbg.hadoop.subgraphs.io.DoubleIntegerPairWritable; 22 | 23 | public class PrepareDataS2Driver extends Configured implements Tool{ 24 | 25 | public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException { 26 | Configuration conf = new Configuration(); 27 | // The parameters: 28 | String degreeFile = args[0]; 29 | String stageOneOutputDir = args[1]; 30 | 31 | int numReducers = Integer.parseInt(args[3]); 32 | 33 | conf.setBoolean("mapreduce.map.output.compress", true); 34 | conf.set("mapreduce.map.output.compress.codec", "com.hadoop.compression.lzo.LzoCodec"); 35 | //conf.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.DefaultCodec"); 36 | 37 | Job job = new Job(conf, "PrepareData Stage Two"); 38 | ((JobConf)job.getConfiguration()).setJar(args[4]); 39 | 40 | job.setReducerClass(PrepareDataS2Reducer.class); 41 | 42 | job.setOutputFormatClass(SequenceFileOutputFormat.class); 43 | 44 | job.setMapOutputKeyClass(DoubleIntegerPairWritable.class); 45 | job.setOutputKeyClass(LongWritable.class); 46 | job.setOutputValueClass(LongWritable.class); 47 | 48 | job.setSortComparatorClass(DoubleIntegerPairComparator.class); 49 | job.setGroupingComparatorClass(DoubleIntegerPairGroupComparator.class); 50 | job.setPartitionerClass(DoubleIntegerPairPartitioner2.class); 51 | 52 | job.setNumReduceTasks(numReducers); 53 | 54 | MultipleInputs.addInputPath(job, 55 | new Path(degreeFile), 56 | SequenceFileInputFormat.class, 57 | PrepareDataDegreeMapper.class); 58 | 59 | MultipleInputs.addInputPath(job, 60 | new Path(stageOneOutputDir), 61 | SequenceFileInputFormat.class, 62 | PrepareDataS2Mapper.class); 63 | 64 | //FileInputFormat.setInputPaths(job, new Path(args[0])); 65 | FileOutputFormat.setOutputPath(job, new Path(args[2])); 66 | 67 | job.waitForCompletion(true); 68 | return 0; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/prepare/PrepareDataS1Driver.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.prepare; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.conf.Configured; 7 | import org.apache.hadoop.fs.Path; 8 | import org.apache.hadoop.io.IntWritable; 9 | import org.apache.hadoop.io.LongWritable; 10 | import org.apache.hadoop.mapred.JobConf; 11 | import org.apache.hadoop.mapreduce.Job; 12 | import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; 13 | import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; 14 | import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; 15 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 16 | import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; 17 | import org.apache.hadoop.util.Tool; 18 | 19 | import dbg.hadoop.subgraphs.io.DoubleIntegerPairComparator; 20 | import dbg.hadoop.subgraphs.io.DoubleIntegerPairGroupComparator; 21 | import dbg.hadoop.subgraphs.io.DoubleIntegerPairPartitioner2; 22 | import dbg.hadoop.subgraphs.io.DoubleIntegerPairWritable; 23 | 24 | public class PrepareDataS1Driver extends Configured implements Tool{ 25 | 26 | public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException { 27 | Configuration conf = new Configuration(); 28 | // The parameters: 29 | String degreeFile = args[0]; 30 | String graphFile = args[1]; 31 | 32 | int numReducers = Integer.parseInt(args[3]); 33 | 34 | conf.setBoolean("mapreduce.map.output.compress", true); 35 | conf.set("mapreduce.map.output.compress.codec", "com.hadoop.compression.lzo.LzoCodec"); 36 | //conf.set("mapred.map.output.compression.codec", "org.apache.hadoop.io.compress.DefaultCodec"); 37 | 38 | Job job = new Job(conf, "PrepareData Stage One"); 39 | ((JobConf)job.getConfiguration()).setJar(args[4]); 40 | 41 | job.setReducerClass(PrepareDataS1Reducer.class); 42 | 43 | job.setOutputFormatClass(SequenceFileOutputFormat.class); 44 | 45 | job.setMapOutputKeyClass(DoubleIntegerPairWritable.class); 46 | job.setMapOutputValueClass(LongWritable.class); 47 | job.setOutputKeyClass(LongWritable.class); 48 | job.setOutputValueClass(IntWritable.class); 49 | 50 | job.setSortComparatorClass(DoubleIntegerPairComparator.class); 51 | job.setGroupingComparatorClass(DoubleIntegerPairGroupComparator.class); 52 | job.setPartitionerClass(DoubleIntegerPairPartitioner2.class); 53 | 54 | job.setNumReduceTasks(numReducers); 55 | 56 | MultipleInputs.addInputPath(job, 57 | new Path(degreeFile), 58 | SequenceFileInputFormat.class, 59 | PrepareDataDegreeMapper.class); 60 | 61 | MultipleInputs.addInputPath(job, 62 | new Path(graphFile), 63 | SequenceFileInputFormat.class, 64 | PrepareDataS1Mapper2.class); 65 | 66 | //FileInputFormat.setInputPaths(job, new Path(args[0])); 67 | FileOutputFormat.setOutputPath(job, new Path(args[2])); 68 | 69 | job.waitForCompletion(true); 70 | return 0; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/stack/TIntStack.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // Copyright (c) 2009, Rob Eden All Rights Reserved. 4 | // Copyright (c) 2009, Jeff Randall All Rights Reserved. 5 | // 6 | // This library is free software; you can redistribute it and/or 7 | // modify it under the terms of the GNU Lesser General Public 8 | // License as published by the Free Software Foundation; either 9 | // version 2.1 of the License, or (at your option) any later version. 10 | // 11 | // This library is distributed in the hope that it will be useful, 12 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | // GNU General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Lesser General Public 17 | // License along with this program; if not, write to the Free Software 18 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | /////////////////////////////////////////////////////////////////////////////// 20 | 21 | 22 | package gnu.trove.stack; 23 | 24 | ////////////////////////////////////////////////// 25 | // THIS IS A GENERATED CLASS. DO NOT HAND EDIT! // 26 | ////////////////////////////////////////////////// 27 | 28 | import java.io.Serializable; 29 | 30 | 31 | /** 32 | * A stack of int primitives. 33 | */ 34 | public interface TIntStack { 35 | 36 | /** 37 | * Returns the value that is used to represent null. The default 38 | * value is generally zero, but can be changed during construction 39 | * of the collection. 40 | * 41 | * @return the value that represents null 42 | */ 43 | public int getNoEntryValue(); 44 | 45 | 46 | /** 47 | * Pushes the value onto the top of the stack. 48 | * 49 | * @param val an int value 50 | */ 51 | public void push( int val ); 52 | 53 | 54 | /** 55 | * Removes and returns the value at the top of the stack. 56 | * 57 | * @return an int value 58 | */ 59 | public int pop(); 60 | 61 | 62 | /** 63 | * Returns the value at the top of the stack. 64 | * 65 | * @return an int value 66 | */ 67 | public int peek(); 68 | 69 | 70 | /** 71 | * Returns the current depth of the stack. 72 | */ 73 | public int size(); 74 | 75 | 76 | /** 77 | * Clears the stack. 78 | */ 79 | public void clear(); 80 | 81 | 82 | /** 83 | * Copies the contents of the stack into a native array. Note that this will NOT 84 | * pop them out of the stack. 85 | * 86 | * @return an int[] value 87 | */ 88 | public int[] toArray(); 89 | 90 | 91 | /** 92 | * Copies a slice of the list into a native array. Note that this will NOT 93 | * pop them out of the stack. 94 | * 95 | * @param dest the array to copy into. 96 | */ 97 | public void toArray( int[] dest ); 98 | } 99 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/gnu/trove/stack/TLongStack.java: -------------------------------------------------------------------------------- 1 | /////////////////////////////////////////////////////////////////////////////// 2 | // Copyright (c) 2001, Eric D. Friedman All Rights Reserved. 3 | // Copyright (c) 2009, Rob Eden All Rights Reserved. 4 | // Copyright (c) 2009, Jeff Randall All Rights Reserved. 5 | // 6 | // This library is free software; you can redistribute it and/or 7 | // modify it under the terms of the GNU Lesser General Public 8 | // License as published by the Free Software Foundation; either 9 | // version 2.1 of the License, or (at your option) any later version. 10 | // 11 | // This library is distributed in the hope that it will be useful, 12 | // but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | // GNU General Public License for more details. 15 | // 16 | // You should have received a copy of the GNU Lesser General Public 17 | // License along with this program; if not, write to the Free Software 18 | // Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. 19 | /////////////////////////////////////////////////////////////////////////////// 20 | 21 | 22 | package gnu.trove.stack; 23 | 24 | ////////////////////////////////////////////////// 25 | // THIS IS A GENERATED CLASS. DO NOT HAND EDIT! // 26 | ////////////////////////////////////////////////// 27 | 28 | import java.io.Serializable; 29 | 30 | 31 | /** 32 | * A stack of long primitives. 33 | */ 34 | public interface TLongStack { 35 | 36 | /** 37 | * Returns the value that is used to represent null. The default 38 | * value is generally zero, but can be changed during construction 39 | * of the collection. 40 | * 41 | * @return the value that represents null 42 | */ 43 | public long getNoEntryValue(); 44 | 45 | 46 | /** 47 | * Pushes the value onto the top of the stack. 48 | * 49 | * @param val an long value 50 | */ 51 | public void push( long val ); 52 | 53 | 54 | /** 55 | * Removes and returns the value at the top of the stack. 56 | * 57 | * @return an long value 58 | */ 59 | public long pop(); 60 | 61 | 62 | /** 63 | * Returns the value at the top of the stack. 64 | * 65 | * @return an long value 66 | */ 67 | public long peek(); 68 | 69 | 70 | /** 71 | * Returns the current depth of the stack. 72 | */ 73 | public int size(); 74 | 75 | 76 | /** 77 | * Clears the stack. 78 | */ 79 | public void clear(); 80 | 81 | 82 | /** 83 | * Copies the contents of the stack into a native array. Note that this will NOT 84 | * pop them out of the stack. 85 | * 86 | * @return an long[] value 87 | */ 88 | public long[] toArray(); 89 | 90 | 91 | /** 92 | * Copies a slice of the list into a native array. Note that this will NOT 93 | * pop them out of the stack. 94 | * 95 | * @param dest the array to copy into. 96 | */ 97 | public void toArray( long[] dest ); 98 | } 99 | -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/maximalclique/MCliqueS3Driver.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.maximalclique; 2 | 3 | import java.io.IOException; 4 | import java.net.URISyntaxException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.conf.Configured; 8 | import org.apache.hadoop.fs.Path; 9 | import org.apache.hadoop.io.LongWritable; 10 | import org.apache.hadoop.io.SequenceFile.CompressionType; 11 | import org.apache.hadoop.mapred.JobConf; 12 | import org.apache.hadoop.mapreduce.Job; 13 | import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; 14 | import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; 15 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 16 | import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; 17 | //import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 18 | import org.apache.hadoop.util.Tool; 19 | 20 | import com.hadoop.compression.lzo.LzoCodec; 21 | 22 | import dbg.hadoop.subgraphs.io.HVArrayGroupComparator; 23 | import dbg.hadoop.subgraphs.io.HVArraySignComparator; 24 | import dbg.hadoop.subgraphs.io.HVArraySign; 25 | import dbg.hadoop.subgraphs.io.HVArray; 26 | import dbg.hadoop.subgraphs.io.HyperVertexComparator; 27 | 28 | public class MCliqueS3Driver extends Configured implements Tool{ 29 | 30 | public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException, URISyntaxException { 31 | Configuration conf = getConf(); 32 | // The parameters: 33 | int numReducers = Integer.parseInt(args[3]); 34 | 35 | conf.setBoolean("mapreduce.map.output.compress", true); 36 | conf.set("mapreduce.map.output.compress.codec", "com.hadoop.compression.lzo.LzoCodec"); 37 | 38 | Job job = new Job(conf, "Maximal Clique Stage Three"); 39 | ((JobConf)job.getConfiguration()).setJar(args[4]); 40 | //JobConf job = new JobConf(getConf(), this.getClass()); 41 | 42 | job.setReducerClass(MCliqueS3Reducer.class); 43 | job.setCombinerClass(MCliqueS3Reducer.class); 44 | 45 | job.setOutputKeyClass(LongWritable.class); 46 | job.setOutputValueClass(LongWritable.class); 47 | 48 | job.setSortComparatorClass(HyperVertexComparator.class); 49 | 50 | //job.setInputFormatClass(SequenceFileInputFormat.class); 51 | //if(enableOutputCompress){ 52 | job.setOutputFormatClass(SequenceFileOutputFormat.class); 53 | SequenceFileOutputFormat.setOutputCompressionType 54 | (job, CompressionType.BLOCK); 55 | SequenceFileOutputFormat.setOutputCompressorClass(job, LzoCodec.class); 56 | 57 | job.setNumReduceTasks(numReducers); 58 | //FileInputFormat.setInputPaths(job, new Path(args[0])); 59 | MultipleInputs.addInputPath(job, 60 | new Path(args[0]), 61 | SequenceFileInputFormat.class, 62 | MCliqueS3DegreeMapper.class); 63 | 64 | MultipleInputs.addInputPath(job, 65 | new Path(args[1]), 66 | SequenceFileInputFormat.class, 67 | MCliqueS3Mapper.class); 68 | 69 | FileOutputFormat.setOutputPath(job, new Path(args[2])); 70 | 71 | job.waitForCompletion(true); 72 | return 0; 73 | } 74 | } -------------------------------------------------------------------------------- /SubgEnumFramework/src/dbg/hadoop/subgenum/qtcounter/QtS1Driver.java: -------------------------------------------------------------------------------- 1 | package dbg.hadoop.subgenum.qtcounter; 2 | 3 | import java.io.IOException; 4 | import java.net.URISyntaxException; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | import org.apache.hadoop.conf.Configured; 8 | import org.apache.hadoop.fs.Path; 9 | import org.apache.hadoop.io.LongWritable; 10 | import org.apache.hadoop.io.SequenceFile.CompressionType; 11 | import org.apache.hadoop.mapred.JobConf; 12 | import org.apache.hadoop.mapreduce.Job; 13 | import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; 14 | import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; 15 | import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 16 | import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; 17 | //import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; 18 | import org.apache.hadoop.util.Tool; 19 | 20 | import com.hadoop.compression.lzo.LzoCodec; 21 | 22 | import dbg.hadoop.subgraphs.io.HVArrayGroupComparator; 23 | import dbg.hadoop.subgraphs.io.HVArraySignComparator; 24 | import dbg.hadoop.subgraphs.io.HVArraySign; 25 | import dbg.hadoop.subgraphs.io.HVArray; 26 | 27 | public class QtS1Driver extends Configured implements Tool{ 28 | 29 | public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException, URISyntaxException { 30 | Configuration conf = getConf(); 31 | // The parameters: 32 | int numReducers = Integer.parseInt(args[3]); 33 | 34 | conf.setBoolean("mapred.compress.map.output", true); 35 | conf.set("mapred.map.output.compression.codec", "com.hadoop.compression.lzo.LzoCodec"); 36 | 37 | Job job = new Job(conf, "QtCounter Stage One"); 38 | ((JobConf)job.getConfiguration()).setJar(args[4]); 39 | //JobConf job = new JobConf(getConf(), this.getClass()); 40 | 41 | job.setReducerClass(QtS1Reducer.class); 42 | 43 | job.setMapOutputKeyClass(HVArraySign.class); 44 | job.setMapOutputValueClass(LongWritable.class); 45 | job.setOutputKeyClass(HVArray.class); 46 | job.setOutputValueClass(HVArray.class); 47 | 48 | job.setSortComparatorClass(HVArraySignComparator.class); 49 | job.setGroupingComparatorClass(HVArrayGroupComparator.class); 50 | 51 | //job.setInputFormatClass(SequenceFileInputFormat.class); 52 | //if(enableOutputCompress){ 53 | job.setOutputFormatClass(SequenceFileOutputFormat.class); 54 | SequenceFileOutputFormat.setOutputCompressionType 55 | (job, CompressionType.BLOCK); 56 | SequenceFileOutputFormat.setOutputCompressorClass(job, LzoCodec.class); 57 | 58 | job.setNumReduceTasks(numReducers); 59 | //FileInputFormat.setInputPaths(job, new Path(args[0])); 60 | MultipleInputs.addInputPath(job, 61 | new Path(args[0]), 62 | SequenceFileInputFormat.class, 63 | EdgeMapper.class); 64 | 65 | MultipleInputs.addInputPath(job, 66 | new Path(args[1]), 67 | SequenceFileInputFormat.class, 68 | QtS1Mapper.class); 69 | 70 | FileOutputFormat.setOutputPath(job, new Path(args[2])); 71 | 72 | job.waitForCompletion(true); 73 | return 0; 74 | } 75 | } --------------------------------------------------------------------------------