├── .gitignore ├── CHANGES.md ├── LICENSE.txt ├── README.md ├── bin ├── .gitignore ├── README.md ├── buildJars.py ├── client.cfg ├── parameterTuning.py └── runOkapi.py ├── license-header.txt ├── pom.xml └── src ├── main ├── java │ └── ml │ │ └── grafos │ │ └── okapi │ │ ├── aggregators │ │ ├── FloatAvgAggregator.java │ │ ├── TextAggregatorWriter.java │ │ └── ValueTextAggregatorWriter.java │ │ ├── cf │ │ ├── CfId.java │ │ ├── CfLongId.java │ │ ├── CfLongIdFloatTextInputFormat.java │ │ ├── FloatMatrixMessage.java │ │ ├── als │ │ │ └── Als.java │ │ ├── annotations │ │ │ ├── AnnotationsInfo.java │ │ │ ├── HyperParameter.java │ │ │ └── OkapiAutotuning.java │ │ ├── eval │ │ │ ├── CFEvaluationOutputFormat.java │ │ │ ├── CfLongIdBooleanTextInputFormat.java │ │ │ ├── CfModelInputFormat.java │ │ │ ├── FloatBoolean.java │ │ │ └── RankEvaluationComputation.java │ │ ├── ranking │ │ │ ├── AbstractCFRankingComputation.java │ │ │ ├── BPRRankingComputation.java │ │ │ ├── ClimfRankingComputation.java │ │ │ ├── PopularityRankingComputation.java │ │ │ ├── RandomRankingComputation.java │ │ │ └── TFMAPRankingComputation.java │ │ ├── sgd │ │ │ └── Sgd.java │ │ └── svd │ │ │ └── Svdpp.java │ │ ├── clustering │ │ └── kmeans │ │ │ ├── ArrayListOfDoubleArrayListWritableAggregator.java │ │ │ ├── DoubleArrayListWritableAggregator.java │ │ │ ├── KMeansClustering.java │ │ │ ├── KMeansTextInputFormat.java │ │ │ ├── KMeansTextOutputFormat.java │ │ │ └── KMeansVertexValue.java │ │ ├── common │ │ ├── Parameters.java │ │ ├── computation │ │ │ ├── PropagateId.java │ │ │ ├── ReverseEdges.java │ │ │ └── SendFriends.java │ │ ├── data │ │ │ ├── ArrayListOfDoubleArrayListWritable.java │ │ │ ├── DoubleArrayListWritable.java │ │ │ ├── IntArrayListWritable.java │ │ │ ├── LongArrayListWritable.java │ │ │ ├── MapWritable.java │ │ │ └── MessageWrapper.java │ │ ├── graph │ │ │ ├── EdgeValueComparator.java │ │ │ ├── NullOutEdges.java │ │ │ └── SortedOutEdges.java │ │ └── jblas │ │ │ └── FloatMatrixWritable.java │ │ ├── examples │ │ ├── SimpleCountInEdgesVertex.java │ │ ├── SimpleIdentityComputation.java │ │ └── SimpleMasterComputeVertex.java │ │ ├── graphs │ │ ├── ClusteringCoefficient.java │ │ ├── ConnectedComponents.java │ │ ├── KCore.java │ │ ├── MultipleSourceShortestPaths.java │ │ ├── ScalableSemimetric.java │ │ ├── SemiClustering.java │ │ ├── SemimetricTriangles.java │ │ ├── SimplePageRank.java │ │ ├── SingleSourceShortestPaths.java │ │ ├── SybilRank.java │ │ ├── SybilRankWattsStrogatzVertexInputFormat.java │ │ ├── Triangles.java │ │ ├── maxbmatching │ │ │ ├── MBMEdgeValue.java │ │ │ ├── MBMMessage.java │ │ │ ├── MBMTextInputFormat.java │ │ │ ├── MBMTextOutputFormat.java │ │ │ └── MaxBMatching.java │ │ └── similarity │ │ │ ├── AdamicAdar.java │ │ │ └── Jaccard.java │ │ ├── io │ │ └── formats │ │ │ ├── AdjacencyListNoValuesTextVertexOutputFormat.java │ │ │ ├── EdgesWithValuesVertexOutputFormat.java │ │ │ ├── IntDoubleTextEdgeInputFormat.java │ │ │ ├── IntIntDefaultEdgeValueTextEdgeInputFormat.java │ │ │ ├── IntIntTextEdgeInputFormat.java │ │ │ ├── LongDoubleBooleanEdgeInputFormat.java │ │ │ ├── LongDoubleDefaultEdgeValueTextEdgeInputFormat.java │ │ │ ├── LongDoubleDoubleAdjacencyListTextVertexOutputFormat.java │ │ │ ├── LongDoubleReverseTextEdgeInputFormat.java │ │ │ ├── LongDoubleTextEdgeInputFormat.java │ │ │ ├── LongDoubleTextEdgeOutputFormat.java │ │ │ ├── LongDoubleZeroReverseTextEdgeInputFormat.java │ │ │ ├── LongDoubleZerosTextEdgeInputFormat.java │ │ │ ├── LongFloatTextEdgeInputFormat.java │ │ │ ├── LongNullNullAdjacencyListTextVertexOutputFormat.java │ │ │ ├── LongNullTextEdgeInputFormat.java │ │ │ ├── SimpleAdjacencyList.java │ │ │ └── TextDoubleTextEdgeInputFormat.java │ │ ├── spinner │ │ ├── OpenHashMapEdges.java │ │ ├── PartitionedLongWritable.java │ │ ├── PrefixHashPartitionerFactory.java │ │ ├── PrefixHashWorkerPartitioner.java │ │ └── Spinner.java │ │ └── utils │ │ ├── Counters.java │ │ └── WritableUtility.java └── resources │ └── log4j.properties └── test ├── java └── ml │ └── grafos │ └── okapi │ ├── cf │ ├── CfLongIdFloatTextEdgeReaderTest.java │ ├── CfLongIdTest.java │ ├── FloatMatrixMessageTest.java │ ├── als │ │ └── AlsTest.java │ ├── eval │ │ ├── CfModelTestingInputFormatTest.java │ │ └── RankEvaluationComputationTest.java │ ├── ranking │ │ ├── BPRRankingComputationTest.java │ │ ├── PopularityRankingComputationTest.java │ │ ├── RandomRankingComputationTest.java │ │ └── TFMAPRankingComputationTest.java │ ├── sgd │ │ └── SgdTest.java │ └── svd │ │ └── SvdppTest.java │ ├── clustering │ └── kmeans │ │ ├── TestDoubleArrayListWritableAggregator.java │ │ ├── TestKMeansClustering.java │ │ ├── TestKMeansVertexValue.java │ │ └── TestRandomInitializationAggregator.java │ ├── common │ ├── graphs │ │ └── TestSortedOutEdges.java │ └── jblas │ │ └── FloatMatrixWritableTest.java │ └── graphs │ ├── ClusteringCoefficientTest.java │ ├── MultipleSourceShortestPathsTest.java │ ├── ScalableSemimetricTest.java │ ├── SemiClusteringTest.java │ ├── SemimetricTrianglesTest.java │ ├── maxbmatching │ └── MaxBMatchingTest.java │ └── similarity │ ├── AdamicAdarTest.java │ └── JaccardTest.java └── resources └── log4j.properties /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | .settings 3 | .classpath 4 | .idea 5 | .project 6 | graphlib.iml 7 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | # 0.3.5 - 2014-09-22 2 | 3 | * Giraph version updated to trunk:185205703ee4cd886598f5393395f19fc367f65f 4 | * Implemented new triangle-related metrics. 5 | * Added the Maximum B-Matching algorithm. 6 | 7 | 8 | # 0.3.4 - 2014-07-09 9 | 10 | * Giraph version updated to trunk:61cb37ecd50b0d9400873624e46692c3282e4cfc. 11 | * Fix compatibility with latest Giraph. 12 | * K-means clustering algorithm. 13 | * Removed Yarn build since it's not being actively supported by Giraph. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Okapi 2 | ======== 3 | 4 | ## Overview 5 | 6 | Okapi is a library of Machine Learning and graph mining algorithms for Giraph. The library includes state-of-the-art Collaborative Filtering algorithms used in recommendation systems as well as graph algorithms such as partitioning, clustering and sybil-detection for OSNs. 7 | 8 | The Okapi library is developed by the Telefonica Research lab and is available as open source under the Apache License. We invite users to contribute to the library, to make it more robust and rich in features. Okapi is part of the [Grafos.ML](http://grafos.ml) project. 9 | 10 | For a full list of the provided algorithms, documentation, and instructions on how to use Okapi, please visit the [Grafos.ML](http://grafos.ml) page. 11 | 12 | 13 | ## Building 14 | 15 | Although you can find pre-built packages on the [Grafos.ML](http://grafos.ml) for different Hadoop distributions, you may very likely need to build the code youself. Go into $OKAPI_HOME, the directory where you cloned the code, and run: 16 | 17 | mvn package 18 | 19 | This will build the code and also run some tests. If you want to skip the tests, then run: 20 | 21 | mvn package -DskipTests 22 | 23 | After that, under $OKAPI_HOME/target, you should find a jar file with a name of the type: 24 | 25 | okapi-${VERSION}-jar-with-dependencies.jar 26 | 27 | Inside the jar, we package the Okapi library as well as all dependencies for convenience. 28 | 29 | ## Running 30 | 31 | Running an Okapi job does not differ from running an ordinary Giraph job. You can use the pre-built jars or the jar you built yourself to launch a Giraph job as described on the [Giraph](http://giraph.apache.org/) site. On our site, we also provide a web-based tool that helps you construct the command you need to execute. Check it out! 32 | 33 | ## Collaborative Filtering 34 | To run any of the CF algorithms, we suggest to use bin/runOkapi.py, i.e.: 35 | 36 | python runOkapi.py SpitPrecision --local-scheduler --model-name Pop --fraction 1.0 37 | 38 | This gives Precision@5, and uses evaluation procedure where we sample 100 items, mark them 39 | as irrelevant and rank all the list according to the predicted scores. 40 | We also provide some results. These are for reference only (default parameters, no cross-validation). 41 | Note, that many of these algorithms were not designed to run on data sets 42 | with ratings, others optimize rating prediction, therefore, we compare apples and boxes. 43 | 44 | Okapi-0.3.2-SNAPSHOT and Movielens 1M dataset: 45 | 46 | --model-name | Precision@5 47 | ----------------------|------------- 48 | Random | 0.1529 49 | Pop | 0.7554 50 | BPR | ~~0.2412~~ 51 | -------------------------------------------------------------------------------- /bin/.gitignore: -------------------------------------------------------------------------------- 1 | ml-10m.zip 2 | ratings.dat 3 | -------------------------------------------------------------------------------- /bin/README.md: -------------------------------------------------------------------------------- 1 | Luigi based experimentation scripts 2 | =================================== 3 | Try to run one of these: 4 | ``` 5 | python runOkapi.py SpitPrecision --local-scheduler --model-name Random --fraction 0.01 6 | ``` 7 | -------------------------------------------------------------------------------- /bin/buildJars.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # encoding: utf-8 3 | 4 | import os 5 | import sys 6 | import subprocess 7 | import re 8 | import shutil 9 | 10 | if len(sys.argv) <= 1: 11 | print("Usage: {scriptName} ".format( 12 | scriptName=os.path.basename(__file__))) 13 | sys.exit(1) 14 | 15 | giraphDir = os.path.abspath(sys.argv[1]) 16 | okapiDir = os.path.join(os.getcwd(), "..") 17 | okapiTargetDir = os.path.join(okapiDir, "target") 18 | okapiJarDir = os.path.join(okapiDir, "jars") 19 | 20 | if not os.path.isdir(okapiJarDir): 21 | os.mkdir(okapiJarDir) 22 | 23 | okapiJarRegex = re.compile( 24 | "^okapi-([\d.]+)-SNAPSHOT-jar-with-dependencies.jar" 25 | ) 26 | 27 | builds = [ 28 | { 29 | "computation": "mr1", 30 | "withGiraph": True, 31 | "destOkapiJar": "okapi-giraph-mr1-{version}.jar", 32 | "giraphBuildCustomArguments": "", 33 | "okapiBuildCustomArguments": "" 34 | }, 35 | { 36 | "computation": "mr2", 37 | "withGiraph": True, 38 | "destOkapiJar": "okapi-giraph-mr2-{version}.jar", 39 | "giraphBuildCustomArguments": "-Phadoop_2", 40 | "okapiBuildCustomArguments": "-Phadoop_yarn" 41 | }, 42 | #{ 43 | #"computation": "yarn", 44 | #"withGiraph": True, 45 | #"destOkapiJar": "okapi-giraph-yarn-{version}.jar", 46 | #"giraphBuildCustomArguments": "-Phadoop_yarn -Dhadoop.version=2.2.0", 47 | #"okapiBuildCustomArguments": "-Phadoop_yarn" 48 | #}, 49 | { 50 | "computation": "all", 51 | "withGiraph": False, 52 | "destOkapiJar": "okapi-{version}.jar", 53 | "giraphBuildCustomArguments": "", 54 | "okapiBuildCustomArguments": "-Phadoop_yarn -Dgiraph.scope=provided" 55 | }, 56 | ] 57 | 58 | 59 | def compileGiraph(build): 60 | os.chdir(giraphDir) 61 | print("Compiling Giraph for {0}".format(build['computation'])) 62 | subprocess.call( 63 | "mvn {0} -DskipTests clean install" 64 | .format(build['giraphBuildCustomArguments']), shell=True) 65 | 66 | 67 | def compileOkapi(build): 68 | os.chdir(okapiDir) 69 | print("Compiling Okapi for {0}".format(build['computation'])) 70 | subprocess.call( 71 | "mvn {0} -DskipTests clean package" 72 | .format(build['okapiBuildCustomArguments']), shell=True) 73 | 74 | 75 | def moveJar(build): 76 | os.chdir(okapiTargetDir) 77 | 78 | sourceJar = None 79 | version = None 80 | 81 | for f in os.listdir("."): 82 | match = okapiJarRegex.match(f) 83 | 84 | if match: 85 | sourceJar = f 86 | version = match.group(1) 87 | break 88 | 89 | if sourceJar is None: 90 | print("Unable to find original Okapi jar matching {0}".format( 91 | repr(okapiJarRegex))) 92 | return 93 | 94 | destinationJar = build["destOkapiJar"].format(version=version) 95 | 96 | print("Copying {0} to {1}".format(sourceJar, destinationJar)) 97 | 98 | destinationJarFullPath = os.path.join(okapiJarDir, destinationJar) 99 | 100 | shutil.copy(sourceJar, destinationJarFullPath) 101 | 102 | 103 | for build in builds: 104 | if build['withGiraph']: 105 | compileGiraph(build) 106 | compileOkapi(build) 107 | moveJar(build) 108 | 109 | os.chdir(okapiDir) 110 | print("Finished!") 111 | -------------------------------------------------------------------------------- /bin/client.cfg: -------------------------------------------------------------------------------- 1 | ######################################### 2 | # Configuration file for runOkapi.py, 3 | # Edit this to reflect your environment 4 | ######################################### 5 | 6 | [hadoop] 7 | #in luigi-1.0.12 the version can be apache1, snakebite, cdh4, cdh3 8 | version: apache1 9 | streaming-jar: /usr/local/hadoop/contrib/streaming/hadoop-streaming-0.20.203.0.jar 10 | 11 | #you need to tell the zookeeper directory to use. It can be any directory 12 | zookeeper-dir: _bsp 13 | hadoop-mem: -Xmx1g 14 | 15 | [okapi] 16 | # Number of workers (minimum 1) 17 | workers: 1 18 | 19 | # Number of threads per worker 20 | threads: 1 21 | 22 | #paths to the jars of giraph and okapi. Giraph jar can be obtained by checking out giraph and building with mvn package 23 | giraph-jar: /Users/linas/devel/giraph/giraph-core/target/giraph-1.1.0-SNAPSHOT-for-hadoop-0.20.204.0-jar-with-dependencies.jar 24 | okapi-jar: /home/dl/GraphLib/target/okapi-0.3.2-SNAPSHOT-jar-with-dependencies.jar 25 | 26 | #When evaluating ranking, how many negative samples to use. 27 | number-of-negative-samples-in-eval: 100 28 | 29 | [core] 30 | default-scheduler-host: localhost 31 | error-email: email@error.com 32 | -------------------------------------------------------------------------------- /bin/parameterTuning.py: -------------------------------------------------------------------------------- 1 | __author__ = 'linas' 2 | 3 | ''' 4 | Parameter tuning based on the Gaussian Processes. 5 | Idea based on http://arxiv.org/pdf/1206.2944.pdf 6 | Because it is more natural for me, I will use UCB method. 7 | 8 | 0. Estimate method performance at 2 points 9 | 1. Run GP and estimate mean and variance of each parameter at many points 10 | 2. Try the new point where mean+variance is highest. We use 99 percentile to merge these two. 11 | 3. Iterate until the new point is the same as you already tried (this means you found maximum with high confidence) 12 | 13 | ''' 14 | 15 | import numpy as np 16 | import math 17 | from sklearn.gaussian_process import GaussianProcess 18 | from matplotlib import pyplot as pl 19 | 20 | def foo(X): 21 | """The function to evaluate""" 22 | ret = [] 23 | for x in X: 24 | r = 2*math.sqrt(sum([n*n for n in x])); 25 | if r == 0: 26 | ret.append(0) 27 | else: 28 | ret.append(math.sin(r) / r); 29 | return ret 30 | 31 | def get_init_points(dims): 32 | '''gives initial 2 points for the system to train on''' 33 | p1 = tuple([mi for mi,ma in dims]) 34 | p2 = tuple([ma for mi,ma in dims]) 35 | return [p1, p2] 36 | 37 | def get_grid(dims, n_bins=10): 38 | '''This is the full mesh grid and serves as discretization of the parameter search space''' 39 | slices = np.mgrid[[slice(row[0], row[1], (row[1]-row[0])/n_bins) for row in dims.values()]] 40 | return zip(*(x.flat for x in slices)) 41 | 42 | def _subspace(i, max_x, X): 43 | from copy import copy 44 | ret = [] 45 | for x in X: 46 | c = list(copy(max_x)) 47 | c[i] = x 48 | ret.append(c) 49 | return ret 50 | 51 | def plot(dims, max_x, max_y, gp, n_bins=100): 52 | ''' 53 | Plots the current estimation of parameters in 2d space. It is beautiful and not very useful :) 54 | You can imagine, that it is impossible to visualize parameter behaviour in multi-dimensional space. 55 | Therefore, it takes the current maximum (max_x), and varies parameter by parameter while others stay fixed. 56 | ''' 57 | #pl.xlabel('Hyper Parameter Value') 58 | # pl.ylabel('f(x)') 59 | 60 | fig = pl.figure() 61 | #fig.suptitle('GP based parameter estimates') 62 | for i, k in enumerate(dims.keys()): 63 | X = np.arange(dims[k][0], dims[k][1], (dims[k][1]-dims[k][0])/n_bins) 64 | ax = fig.add_subplot(len(dims), 1, i+1) 65 | grid = _subspace(i, max_x, X) 66 | 67 | ax.plot(max_x[i], max_y, 'o', color="red") 68 | #ax.plot(grid, f(grid), 'r:', label=u'$f(x)') 69 | y_pred, MSE = gp.predict(grid, eval_MSE=True) 70 | ax.plot(grid, y_pred, 'b-', label=u'Prediction') 71 | ax.set_xlabel('Parameter '+k) 72 | UCB_u = y_pred + np.sqrt(MSE) * 2.576 73 | UCB_l = y_pred - np.sqrt(MSE) * 2.576 74 | ax.fill(np.concatenate([grid, grid[::-1]]), \ 75 | np.concatenate([UCB_u, 76 | (UCB_l)[::-1]]), \ 77 | alpha=.5, fc='b', ec='None', label='99% confidence interval') 78 | fig.tight_layout() 79 | pl.show() 80 | 81 | def get_next_UCB_point(X, f, depth=0): 82 | print "trial #{}".format(depth) 83 | 84 | gp = GaussianProcess(theta0=0.1, thetaL=.001, thetaU=5.) 85 | gp.fit(np.array(X), np.array(f(X)).T) 86 | y_pred, MSE = gp.predict(grid, eval_MSE=True) 87 | 88 | #get upper confidence interval. 2.576 z-score corresponds to 99th percentile 89 | UCB_u = y_pred + np.sqrt(MSE) * 2.576 90 | 91 | next_list = zip(UCB_u, grid) 92 | next_list.sort(reverse=True) 93 | new_x = next_list[0][1] 94 | if not new_x in X: 95 | X.append(new_x) 96 | return get_next_UCB_point(X, f, depth+1) 97 | else: 98 | return (next_list[0][0], next_list[0][1], gp) 99 | 100 | if __name__ == "__main__": 101 | #init the dimensions to the minimum and maximum 102 | dims = {"u":(-10.0, 10.0), "i":(-10.0, 8.0)} 103 | grid = get_grid(dims) 104 | X = get_init_points(dims.values()) 105 | y,x, gp = get_next_UCB_point(X, foo) 106 | print "y={} at x={}".format(y, x) 107 | plot(dims, x, y, gp) -------------------------------------------------------------------------------- /license-header.txt: -------------------------------------------------------------------------------- 1 | Copyright 2014 Grafos.ml 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/aggregators/FloatAvgAggregator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.aggregators; 17 | 18 | import java.io.DataInput; 19 | import java.io.DataOutput; 20 | import java.io.IOException; 21 | 22 | import ml.grafos.okapi.aggregators.FloatAvgAggregator.PartialAvg; 23 | 24 | import org.apache.giraph.aggregators.BasicAggregator; 25 | import org.apache.hadoop.io.Writable; 26 | 27 | public class FloatAvgAggregator extends BasicAggregator { 28 | 29 | @Override 30 | public void aggregate(PartialAvg value) { 31 | getAggregatedValue().combine(value); 32 | } 33 | 34 | @Override 35 | public PartialAvg createInitialValue() { 36 | return new PartialAvg(0,0); 37 | } 38 | 39 | public static class PartialAvg implements Writable { 40 | public float partialSum; 41 | public int partialCount; 42 | 43 | public PartialAvg() { 44 | partialSum = 0; 45 | partialCount = 0; 46 | } 47 | 48 | public PartialAvg(float sum, int count) { 49 | partialSum = sum; 50 | partialCount = count; 51 | } 52 | 53 | public void combine(PartialAvg other) { 54 | this.partialSum += other.partialSum; 55 | this.partialCount += other.partialCount; 56 | } 57 | 58 | public float get() { 59 | return partialSum/(float)partialCount; 60 | } 61 | 62 | @Override 63 | public void readFields(DataInput input) throws IOException { 64 | partialSum = input.readFloat(); 65 | partialCount = input.readInt(); 66 | } 67 | 68 | @Override 69 | public void write(DataOutput output) throws IOException { 70 | output.writeFloat(partialSum); 71 | output.writeInt(partialCount); 72 | } 73 | 74 | @Override 75 | public String toString() { 76 | return partialCount+" "+partialSum+" "+get(); 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/aggregators/ValueTextAggregatorWriter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.aggregators; 17 | 18 | import org.apache.hadoop.io.Writable; 19 | 20 | /** 21 | * An aggregator writer that only writes the values of aggregators. 22 | * @author dl 23 | * 24 | */ 25 | public class ValueTextAggregatorWriter extends TextAggregatorWriter { 26 | @Override 27 | protected String aggregatorToString(String aggregatorName, Writable value, 28 | long superstep) { 29 | return value+"\n"; 30 | } 31 | } -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/cf/CfId.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf; 17 | 18 | import org.apache.hadoop.io.WritableComparable; 19 | 20 | /** 21 | * Classes that implement this interface represent the ID of a node in a 22 | * user-item graph. Nodes in this case typically represent either users or items 23 | * and are identified by a number or a string (e.g. an item description). To 24 | * avoid conflicts between user and item ids, this interface defines the type 25 | * of the node as well, not just the identifier. The type is a byte value and 26 | * can by set and interpreted in an application-specific manner. 27 | * 28 | * @author dl 29 | * 30 | * @param 31 | */ 32 | public interface CfId extends WritableComparable> { 33 | /** 34 | * Returns the type of the node. 35 | * @return 36 | */ 37 | public byte getType(); 38 | 39 | /** 40 | * Returns the identifier of the node. 41 | * @return 42 | */ 43 | public T getId(); 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/cf/CfLongId.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf; 17 | 18 | import java.io.DataInput; 19 | import java.io.DataOutput; 20 | import java.io.IOException; 21 | 22 | /** 23 | * This class represents the ID of a node in a CF scenario that has an 24 | * identifier of type long. 25 | * 26 | * @author dl 27 | * 28 | */ 29 | public class CfLongId implements CfId { 30 | 31 | private byte type; 32 | private Long id; 33 | 34 | public CfLongId() { 35 | id = 0L; 36 | } 37 | 38 | public CfLongId(byte type, long id) { 39 | this.type = type; 40 | this.id = id; 41 | } 42 | 43 | public boolean isItem(){ 44 | return type == 1; 45 | } 46 | 47 | public boolean isUser(){ 48 | return type == 0; 49 | } 50 | 51 | public boolean isOutput(){ 52 | return type == -1; 53 | } 54 | 55 | public byte getType() { 56 | return type; 57 | } 58 | 59 | public Long getId() { 60 | return id; 61 | } 62 | 63 | @Override 64 | public void readFields(DataInput input) throws IOException { 65 | type = input.readByte(); 66 | id = input.readLong(); 67 | } 68 | 69 | @Override 70 | public void write(DataOutput output) throws IOException { 71 | output.writeByte(type); 72 | output.writeLong(id); 73 | } 74 | 75 | /** 76 | * To objects of this class are the same only if both the type and the id 77 | * are the same. 78 | */ 79 | @Override 80 | public int compareTo(CfId other) { 81 | if (typeother.getType()){ 84 | return 1; 85 | } else if (id==null && other.getId()!=null) { 86 | return 1; 87 | } else { 88 | return id.compareTo(other.getId()); 89 | } 90 | } 91 | 92 | @Override 93 | public int hashCode() { 94 | final int prime = 31; 95 | int result = 1; 96 | result = prime * result + ((id == null) ? 0 : id.hashCode()); 97 | result = prime * result + type; 98 | return result; 99 | } 100 | 101 | @Override 102 | public boolean equals(Object obj) { 103 | if (this == obj) { 104 | return true; 105 | } 106 | if (obj == null) { 107 | return false; 108 | } 109 | if (getClass() != obj.getClass()) { 110 | return false; 111 | } 112 | CfLongId other = (CfLongId) obj; 113 | if (id == null) { 114 | if (other.id != null) { 115 | return false; 116 | } 117 | } else if (!id.equals(other.id)) { 118 | return false; 119 | } 120 | if (type != other.type) { 121 | return false; 122 | } 123 | return true; 124 | } 125 | 126 | /** 127 | * Returns a string of the format: 128 | * <\space> 129 | */ 130 | @Override 131 | public String toString() { 132 | return id+" "+type; 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/cf/CfLongIdFloatTextInputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf; 17 | 18 | import java.io.IOException; 19 | import java.util.regex.Pattern; 20 | 21 | 22 | import org.apache.giraph.io.EdgeReader; 23 | import org.apache.giraph.io.formats.IntNullTextEdgeInputFormat; 24 | import org.apache.giraph.io.formats.TextEdgeInputFormat; 25 | import org.apache.hadoop.io.FloatWritable; 26 | import org.apache.hadoop.io.Text; 27 | import org.apache.hadoop.mapreduce.InputSplit; 28 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 29 | 30 | /** 31 | * Simple text-based {@link org.apache.giraph.io.EdgeInputFormat} to read 32 | * user-item ratings as the input for the CF algorithms. 33 | * 34 | * Each line consists of: 35 | */ 36 | public class CfLongIdFloatTextInputFormat extends 37 | TextEdgeInputFormat { 38 | /** Splitter for endpoints */ 39 | private static final Pattern SEPARATOR = Pattern.compile("[\t ]"); 40 | 41 | @Override 42 | public EdgeReader createEdgeReader( 43 | InputSplit split, TaskAttemptContext context) throws IOException { 44 | return new CfIdFloatTextEdgeReader(); 45 | } 46 | 47 | /** 48 | * {@link org.apache.giraph.io.EdgeReader} associated with 49 | * {@link IntNullTextEdgeInputFormat}. 50 | */ 51 | public class CfIdFloatTextEdgeReader extends 52 | TextEdgeReaderFromEachLineProcessed { 53 | @Override 54 | protected String[] preprocessLine(Text line) throws IOException { 55 | return SEPARATOR.split(line.toString()); 56 | } 57 | 58 | @Override 59 | protected CfLongId getSourceVertexId(String[] tokens) 60 | throws IOException { 61 | // type 0 is user 62 | return new CfLongId((byte)0, Integer.parseInt(tokens[0])); 63 | } 64 | 65 | @Override 66 | protected CfLongId getTargetVertexId(String[] tokens) 67 | throws IOException { 68 | // type 1 is item 69 | return new CfLongId((byte)1, Integer.parseInt(tokens[1])); 70 | } 71 | 72 | @Override 73 | protected FloatWritable getValue(String[] tokens) throws IOException { 74 | return new FloatWritable(Float.parseFloat(tokens[2])); 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/cf/FloatMatrixMessage.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf; 17 | 18 | import java.io.DataInput; 19 | import java.io.DataOutput; 20 | import java.io.IOException; 21 | 22 | import ml.grafos.okapi.common.jblas.FloatMatrixWritable; 23 | 24 | import org.apache.hadoop.io.Writable; 25 | 26 | /** 27 | * Messages send in most of the CF algorithm typically must carry the id of the 28 | * message sender as well as the payload of the message, that is, the latent 29 | * vector. 30 | * @author dl 31 | * 32 | */ 33 | public class FloatMatrixMessage implements Writable { 34 | CfLongId senderId; 35 | FloatMatrixWritable factors; 36 | float score; 37 | 38 | public FloatMatrixMessage() { 39 | } 40 | 41 | public FloatMatrixMessage(FloatMatrixMessage msg) { 42 | this.senderId = msg.senderId; 43 | this.factors = msg.factors; 44 | this.score = msg.score; 45 | } 46 | 47 | public FloatMatrixMessage(CfLongId senderId, FloatMatrixWritable factors, 48 | float score) { 49 | this.senderId = senderId; 50 | this.factors = factors; 51 | this.score = score; 52 | } 53 | 54 | public CfLongId getSenderId() { 55 | return senderId; 56 | } 57 | 58 | public void setSenderId(CfLongId senderId) { 59 | this.senderId = senderId; 60 | } 61 | 62 | public FloatMatrixWritable getFactors() { 63 | return factors; 64 | } 65 | 66 | public void setFactors(FloatMatrixWritable factors) { 67 | this.factors = factors; 68 | } 69 | 70 | public float getScore() { 71 | return score; 72 | } 73 | 74 | public void setScore(float score) { 75 | this.score = score; 76 | } 77 | 78 | public void readFields(DataInput input) throws IOException { 79 | senderId = new CfLongId(); 80 | senderId.readFields(input); 81 | factors = new FloatMatrixWritable(); 82 | factors.readFields(input); 83 | score = input.readFloat(); 84 | 85 | } 86 | 87 | public void write(DataOutput output) throws IOException { 88 | senderId.write(output); 89 | factors.write(output); 90 | output.writeFloat(score); 91 | } 92 | 93 | @Override 94 | public boolean equals(Object matrix) { 95 | if (matrix==null) { 96 | return false; 97 | } 98 | FloatMatrixMessage other = (FloatMatrixMessage) matrix; 99 | if (senderId==null && other.senderId!=null) { 100 | return false; 101 | } else if (!senderId.equals(other.senderId)) { 102 | return false; 103 | } 104 | if (factors==null && other.factors!=null) { 105 | return false; 106 | } else if (!factors.equals(other.factors)) { 107 | return false; 108 | } 109 | if (score!=other.score) { 110 | return false; 111 | } 112 | return true; 113 | } 114 | 115 | @Override 116 | public String toString() { 117 | return "["+senderId+"] "+score+" "+factors; 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/cf/annotations/HyperParameter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf.annotations; 17 | 18 | import java.lang.annotation.Retention; 19 | import java.lang.annotation.RetentionPolicy; 20 | 21 | @Retention(RetentionPolicy.RUNTIME) 22 | public @interface HyperParameter { 23 | String parameterName(); //the name of the parameter that is used to pass to the giraph with ca option, for example delta 24 | String description(); //description of the parameter 25 | float defaultValue(); //default parameter value 26 | float minimumValue(); //minimum value parameter can take (for automatic search) 27 | float maximumValue(); //maximum value parameter can take (for automatic search) 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/cf/annotations/OkapiAutotuning.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf.annotations; 17 | 18 | import java.lang.annotation.Retention; 19 | import java.lang.annotation.RetentionPolicy; 20 | 21 | /** 22 | * Used to indicate that the method could be autotuned by okapi tuning script. 23 | * @author linas 24 | * 25 | */ 26 | @Retention(RetentionPolicy.RUNTIME) 27 | public @interface OkapiAutotuning { 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/cf/eval/CFEvaluationOutputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf.eval; 17 | 18 | import java.io.IOException; 19 | 20 | import ml.grafos.okapi.cf.CfLongId; 21 | import ml.grafos.okapi.common.jblas.FloatMatrixWritable; 22 | 23 | import org.apache.giraph.graph.Vertex; 24 | import org.apache.giraph.io.formats.TextVertexOutputFormat; 25 | import org.apache.hadoop.io.BooleanWritable; 26 | import org.apache.hadoop.io.Text; 27 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 28 | 29 | public class CFEvaluationOutputFormat extends 30 | TextVertexOutputFormat { 31 | 32 | CfLongId outputEdge = new CfLongId((byte)-1, 0); 33 | 34 | @Override 35 | public TextVertexWriter createVertexWriter( 36 | TaskAttemptContext context) throws IOException, 37 | InterruptedException { 38 | 39 | TextVertexWriterToEachLine tvwt = new TextVertexWriterToEachLine() { 40 | 41 | @Override 42 | protected Text convertVertexToLine( 43 | Vertex vertex) 44 | throws IOException { 45 | if (outputEdge.equals(vertex.getId())){ 46 | return new Text(vertex.getValue().toString()); 47 | } 48 | return null; 49 | } 50 | }; 51 | 52 | return tvwt; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/cf/eval/CfLongIdBooleanTextInputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf.eval; 17 | 18 | import java.io.IOException; 19 | import java.util.regex.Pattern; 20 | 21 | import ml.grafos.okapi.cf.CfLongId; 22 | 23 | import org.apache.giraph.io.EdgeReader; 24 | import org.apache.giraph.io.formats.IntNullTextEdgeInputFormat; 25 | import org.apache.giraph.io.formats.TextEdgeInputFormat; 26 | import org.apache.hadoop.io.BooleanWritable; 27 | import org.apache.hadoop.io.FloatWritable; 28 | import org.apache.hadoop.io.Text; 29 | import org.apache.hadoop.mapreduce.InputSplit; 30 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 31 | 32 | /** 33 | * Simple text-based {@link org.apache.giraph.io.EdgeInputFormat} to read 34 | * user-item ratings as the input for the CF algorithms. 35 | * 36 | * Each line consists of: 37 | */ 38 | public class CfLongIdBooleanTextInputFormat extends 39 | TextEdgeInputFormat { 40 | /** Splitter for endpoints */ 41 | private static final Pattern SEPARATOR = Pattern.compile("[\t ]"); 42 | 43 | @Override 44 | public EdgeReader createEdgeReader( 45 | InputSplit split, TaskAttemptContext context) throws IOException { 46 | return new CfIdFloatTextEdgeReader(); 47 | } 48 | 49 | /** 50 | * {@link org.apache.giraph.io.EdgeReader} associated with 51 | * {@link IntNullTextEdgeInputFormat}. 52 | */ 53 | public class CfIdFloatTextEdgeReader extends 54 | TextEdgeReaderFromEachLineProcessed { 55 | @Override 56 | protected String[] preprocessLine(Text line) throws IOException { 57 | return SEPARATOR.split(line.toString()); 58 | } 59 | 60 | @Override 61 | protected CfLongId getSourceVertexId(String[] tokens) 62 | throws IOException { 63 | // type 0 is user 64 | return new CfLongId((byte)0, Integer.parseInt(tokens[0])); 65 | } 66 | 67 | @Override 68 | protected CfLongId getTargetVertexId(String[] tokens) 69 | throws IOException { 70 | // type 1 is item 71 | return new CfLongId((byte)1, Integer.parseInt(tokens[1])); 72 | } 73 | 74 | @Override 75 | protected BooleanWritable getValue(String[] tokens) throws IOException { 76 | return new BooleanWritable(true); 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/cf/eval/CfModelInputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf.eval; 17 | 18 | import java.io.IOException; 19 | import java.util.ArrayList; 20 | 21 | import ml.grafos.okapi.cf.CfLongId; 22 | import ml.grafos.okapi.common.jblas.FloatMatrixWritable; 23 | 24 | import org.apache.giraph.io.formats.TextVertexValueInputFormat; 25 | import org.apache.hadoop.io.FloatWritable; 26 | import org.apache.hadoop.io.Text; 27 | import org.apache.hadoop.mapreduce.InputSplit; 28 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 29 | import org.jblas.FloatMatrix; 30 | 31 | /** 32 | * A custom reader that reads the model formated as following example. 33 | * 34 | * nodeId computed_model 35 | * 32729 0 [0.883140; 0.126675] 36 | * 7563 0 [0.544951; 0.719476] 37 | * 5007 1 [0.726413; 0.968422] 38 | * 1384 1 [0.933587; 0.755566] 39 | * 304 1 [0.368630; 0.468095] 40 | * 41 | * This can be obtained by running runOkapi.py. 42 | * 43 | * @author linas 44 | * 45 | */ 46 | public class CfModelInputFormat extends TextVertexValueInputFormat{ 47 | 48 | public static String DIMENSION = "dim"; 49 | 50 | @Override 51 | public TextVertexValueReader createVertexValueReader( 52 | InputSplit split, TaskAttemptContext context) throws IOException { 53 | return new LongArrayBooleanVertexValueReader(); 54 | } 55 | 56 | public class LongArrayBooleanVertexValueReader extends TextVertexValueReaderFromEachLineProcessed { 57 | 58 | @Override 59 | protected String[] preprocessLine(Text line) throws IOException { 60 | return line.toString().split("\t"); 61 | } 62 | 63 | @Override 64 | protected CfLongId getId(String[] line) throws IOException { 65 | String id = line[0]; 66 | String[] id_type = id.split(" "); 67 | return new CfLongId((byte)Integer.parseInt(id_type[1]), Long.parseLong(id_type[0])); 68 | } 69 | 70 | @Override 71 | protected FloatMatrixWritable getValue(String[] line) 72 | throws IOException { 73 | if (line.length > 1){//for users and items 74 | String[] factors = line[1].split("\\[|\\]|,|;"); 75 | ArrayList factorsFloat = new ArrayList(); 76 | for(int i=0; i{ 19 | private float score; 20 | private boolean relevant; 21 | 22 | public FloatBoolean(float score, boolean relevant) { 23 | this.setScore(score); 24 | this.setRelevant(relevant); 25 | } 26 | 27 | /** 28 | * For sorting from biggest to smallest score! 29 | * Kind of inverse comparator. 30 | */ 31 | public int compareTo(FloatBoolean o) { 32 | if (getScore() == o.getScore()){ 33 | return 0; 34 | }else if (getScore() < o.getScore()){ 35 | return 1; 36 | }else return -1; 37 | } 38 | 39 | public float getScore() { 40 | return score; 41 | } 42 | 43 | public void setScore(float score) { 44 | this.score = score; 45 | } 46 | 47 | public boolean isRelevant() { 48 | return relevant; 49 | } 50 | 51 | public void setRelevant(boolean relevant) { 52 | this.relevant = relevant; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/cf/ranking/PopularityRankingComputation.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf.ranking; 17 | 18 | import java.io.IOException; 19 | 20 | import ml.grafos.okapi.cf.CfLongId; 21 | import ml.grafos.okapi.cf.FloatMatrixMessage; 22 | import ml.grafos.okapi.common.jblas.FloatMatrixWritable; 23 | 24 | import org.apache.giraph.edge.Edge; 25 | import org.apache.giraph.graph.BasicComputation; 26 | import org.apache.giraph.graph.Vertex; 27 | import org.apache.hadoop.io.FloatWritable; 28 | import org.jblas.FloatMatrix; 29 | 30 | 31 | /** 32 | * Computes the popularity ranking based on the number of users rated/seen the item. 33 | * The idea is to represent users and items as bipartite graph. Then compute how many messages 34 | * each item received which is equal to the number of users rated the item. 35 | * @author linas 36 | * 37 | */ 38 | public class PopularityRankingComputation extends BasicComputation { 39 | 40 | static final FloatMatrixWritable emptyList = new FloatMatrixWritable(0); 41 | static final CfLongId nullId = new CfLongId(); 42 | static final FloatMatrixMessage emptyMsg = new FloatMatrixMessage(nullId, emptyList, 0); 43 | 44 | @Override 45 | public void compute(Vertex vertex, Iterable messages) throws IOException { 46 | if (getSuperstep() == 0){//send empty message with the count 47 | vertex.setValue(new FloatMatrixWritable(FloatMatrix.ones(1))); //set all users to 1 48 | if (vertex.getId().isUser()){ 49 | Iterable> edges = vertex.getEdges(); 50 | sendMessage(vertex.getId(), emptyMsg); //send message to myself in order to be executed in the next super step 51 | for (Edge edge : edges) { 52 | sendMessage(edge.getTargetVertexId(), new FloatMatrixMessage(vertex.getId(), emptyList, edge.getValue().get())); 53 | } 54 | } 55 | }else if(getSuperstep() == 1){//compute how many messages were received by an item (how many users rated this item) 56 | if (vertex.getId().isItem()){ 57 | int cnt = 0; 58 | for (@SuppressWarnings("unused") FloatMatrixMessage msg : messages) { 59 | cnt+= 1; 60 | } 61 | FloatMatrix output = FloatMatrix.zeros(1); 62 | output.put(0, cnt); 63 | vertex.setValue(new FloatMatrixWritable(output)); 64 | } 65 | } 66 | vertex.voteToHalt(); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/cf/ranking/RandomRankingComputation.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf.ranking; 17 | 18 | import java.io.IOException; 19 | 20 | import ml.grafos.okapi.cf.CfLongId; 21 | import ml.grafos.okapi.cf.FloatMatrixMessage; 22 | import ml.grafos.okapi.cf.annotations.OkapiAutotuning; 23 | import ml.grafos.okapi.common.jblas.FloatMatrixWritable; 24 | 25 | import org.apache.giraph.edge.Edge; 26 | import org.apache.giraph.graph.BasicComputation; 27 | import org.apache.giraph.graph.Vertex; 28 | import org.apache.hadoop.io.FloatWritable; 29 | import org.jblas.FloatMatrix; 30 | 31 | 32 | /** 33 | * Computes the random ranking 34 | * @author linas 35 | * 36 | */ 37 | @OkapiAutotuning 38 | public class RandomRankingComputation extends BasicComputation { 39 | 40 | static final FloatMatrixWritable emptyList = new FloatMatrixWritable(0); 41 | static final CfLongId nullId = new CfLongId(); 42 | static final FloatMatrixMessage emptyMsg = new FloatMatrixMessage(nullId, emptyList, 0); 43 | 44 | static int DIM = 2; 45 | 46 | public static void setDim(int dim) { 47 | RandomRankingComputation.DIM = dim; 48 | } 49 | 50 | @Override 51 | public void compute(Vertex vertex, Iterable messages) throws IOException { 52 | vertex.setValue(new FloatMatrixWritable(FloatMatrix.rand(DIM))); 53 | if(getSuperstep() == 0){ 54 | Iterable> edges = vertex.getEdges(); 55 | sendMessage(vertex.getId(), emptyMsg); //send message to myself in order to be executed in the next super step 56 | for (Edge edge : edges) { 57 | sendMessage(edge.getTargetVertexId(), new FloatMatrixMessage(vertex.getId(), emptyList, edge.getValue().get())); 58 | } 59 | } 60 | vertex.voteToHalt(); 61 | } 62 | 63 | public static int getDim() { 64 | return DIM; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/clustering/kmeans/ArrayListOfDoubleArrayListWritableAggregator.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.clustering.kmeans; 2 | 3 | import java.util.Random; 4 | 5 | import ml.grafos.okapi.common.data.ArrayListOfDoubleArrayListWritable; 6 | 7 | import org.apache.giraph.aggregators.Aggregator; 8 | import org.apache.giraph.conf.DefaultImmutableClassesGiraphConfigurable; 9 | 10 | @SuppressWarnings("rawtypes") 11 | public class ArrayListOfDoubleArrayListWritableAggregator extends 12 | DefaultImmutableClassesGiraphConfigurable 13 | implements Aggregator { 14 | 15 | public static final String CLUSTER_CENTERS_COUNT = "kmeans.cluster.centers.count"; 16 | public static final String POINTS_COUNT = "kmeans.points.count"; 17 | 18 | private int k; // the number of the cluster centers 19 | private int pointsCount; // the number of input points 20 | private ArrayListOfDoubleArrayListWritable value; 21 | 22 | public ArrayListOfDoubleArrayListWritableAggregator() { 23 | setAggregatedValue(createInitialValue()); 24 | } 25 | 26 | /** 27 | * Used to randomly select initial points for k-means 28 | * If the size of the current list is less than k (#centers) 29 | * then the element is appended in the list 30 | * else it replaces an element in a random position 31 | * with probability k/N, where N is the total number of points 32 | * 33 | * @param other 34 | */ 35 | @Override 36 | public void aggregate(ArrayListOfDoubleArrayListWritable other) { 37 | k = getConf().getInt(CLUSTER_CENTERS_COUNT, 0); 38 | pointsCount = getConf().getInt(POINTS_COUNT, 0); 39 | for ( int i = 0; i < other.size(); i++ ) { 40 | if ( getAggregatedValue().size() < k ) { 41 | value.add(other.get(i)); 42 | } 43 | else { 44 | Random ran = new Random(); 45 | int index = ran.nextInt(k); 46 | if (Math.random() > ((double) k / (double) pointsCount) ) { 47 | value.set(index, other.get(i)); 48 | } 49 | } 50 | } 51 | 52 | } 53 | 54 | @Override 55 | public ArrayListOfDoubleArrayListWritable createInitialValue() { 56 | return new ArrayListOfDoubleArrayListWritable(); 57 | } 58 | 59 | @Override 60 | public ArrayListOfDoubleArrayListWritable getAggregatedValue() { 61 | return value; 62 | } 63 | 64 | @Override 65 | public void setAggregatedValue(ArrayListOfDoubleArrayListWritable value) { 66 | this.value = value; 67 | } 68 | 69 | @Override 70 | public void reset() { 71 | value = createInitialValue(); 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/clustering/kmeans/DoubleArrayListWritableAggregator.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.clustering.kmeans; 2 | 3 | import ml.grafos.okapi.common.data.DoubleArrayListWritable; 4 | import org.apache.giraph.aggregators.BasicAggregator; 5 | import org.apache.hadoop.io.DoubleWritable; 6 | 7 | public class DoubleArrayListWritableAggregator extends BasicAggregator { 8 | 9 | @Override 10 | public void aggregate(DoubleArrayListWritable other) { 11 | DoubleArrayListWritable aggrValue = getAggregatedValue(); 12 | if ( aggrValue.size() == 0 ) { 13 | // first-time creation 14 | for ( int i = 0; i < other.size(); i ++ ) { 15 | aggrValue.add(other.get(i)); 16 | } 17 | setAggregatedValue(aggrValue); 18 | } 19 | else if ( aggrValue.size() < other.size() ) { 20 | throw new IndexOutOfBoundsException("The value to be aggregated " + 21 | "cannot have larger size than the aggregator value"); 22 | } 23 | else { 24 | for ( int i = 0; i < other.size(); i ++ ) { 25 | DoubleWritable element = new DoubleWritable(aggrValue.get(i).get() + other.get(i).get()); 26 | aggrValue.set(i, element); 27 | } 28 | setAggregatedValue(aggrValue); 29 | } 30 | } 31 | 32 | @Override 33 | public DoubleArrayListWritable createInitialValue() { 34 | return new DoubleArrayListWritable(); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/clustering/kmeans/KMeansTextInputFormat.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.clustering.kmeans; 2 | 3 | import java.io.IOException; 4 | import java.util.regex.Pattern; 5 | import ml.grafos.okapi.common.data.DoubleArrayListWritable; 6 | import org.apache.giraph.edge.Edge; 7 | import org.apache.giraph.io.formats.TextVertexInputFormat; 8 | import org.apache.hadoop.io.DoubleWritable; 9 | import org.apache.hadoop.io.IntWritable; 10 | import org.apache.hadoop.io.LongWritable; 11 | import org.apache.hadoop.io.NullWritable; 12 | import org.apache.hadoop.io.Text; 13 | import org.apache.hadoop.mapreduce.InputSplit; 14 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 15 | 16 | /** 17 | * The input format for the k-means algorithm. 18 | * It reads form a file, where each line contains one point. 19 | * The line is formed by an integer id and the double coordinates of the input point. 20 | * The id is separated from the coordinates by comma, while the coordinates are tab-separated. 21 | * The produced vertex value contains the point coordinates and an empty initial cluster center. 22 | */ 23 | public class KMeansTextInputFormat extends 24 | TextVertexInputFormat { 25 | /** Separator of the vertex id and its coordinates */ 26 | private static final Pattern ID_SEPARATOR = Pattern.compile("[,]"); 27 | /** Separator of the coordinates values */ 28 | private static final Pattern COORD_SEPARATOR = Pattern.compile("[\t ]"); 29 | 30 | @Override 31 | public TextVertexReader createVertexReader( 32 | InputSplit split, TaskAttemptContext context) throws IOException { 33 | return new LongDoubleArrayListNullVertexReader(); 34 | } 35 | 36 | public class LongDoubleArrayListNullVertexReader extends 37 | TextVertexReaderFromEachLineProcessed { 38 | /** Vertex id for the current line */ 39 | private LongWritable id; 40 | /** The vertex coordinates*/ 41 | private String coordinatesString; 42 | 43 | @Override 44 | protected String[] preprocessLine(Text line) throws IOException { 45 | String[] tokens = ID_SEPARATOR.split(line.toString()); 46 | id = new LongWritable(Long.parseLong(tokens[0])); 47 | coordinatesString = tokens[1]; 48 | return tokens; 49 | } 50 | 51 | @Override 52 | protected LongWritable getId(String[] line) throws IOException { 53 | return id; 54 | } 55 | 56 | @Override 57 | protected KMeansVertexValue getValue(String[] line) 58 | throws IOException { 59 | DoubleArrayListWritable coordinates = new DoubleArrayListWritable(); 60 | String[] tokens = COORD_SEPARATOR.split(coordinatesString); 61 | for ( int i = 0; i < tokens.length; i++ ) { 62 | coordinates.add(new DoubleWritable(Double.parseDouble(tokens[i]))); 63 | } 64 | IntWritable clusterId = new IntWritable(); 65 | return new KMeansVertexValue(coordinates, clusterId); 66 | } 67 | 68 | @Override 69 | protected Iterable> getEdges( 70 | String[] line) throws IOException { 71 | // no edges 72 | return null; 73 | } 74 | 75 | } 76 | 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/clustering/kmeans/KMeansTextOutputFormat.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.clustering.kmeans; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.giraph.graph.Vertex; 6 | import org.apache.giraph.io.formats.TextVertexOutputFormat; 7 | import org.apache.hadoop.io.LongWritable; 8 | import org.apache.hadoop.io.NullWritable; 9 | import org.apache.hadoop.io.Text; 10 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 11 | 12 | 13 | public class KMeansTextOutputFormat extends 14 | TextVertexOutputFormat { 15 | 16 | @Override 17 | public TextVertexWriter createVertexWriter( 18 | TaskAttemptContext context) throws IOException, 19 | InterruptedException { 20 | return new LongIntVertexNullVertexWriter(); 21 | } 22 | 23 | protected class LongIntVertexNullVertexWriter extends 24 | TextVertexWriterToEachLine { 25 | private String delimiter = ","; 26 | 27 | @Override 28 | protected Text convertVertexToLine( 29 | Vertex vertex) 30 | throws IOException { 31 | StringBuffer sb = new StringBuffer(vertex.getId().toString()); 32 | sb.append(delimiter); 33 | sb.append(vertex.getValue().getClusterId().toString()); 34 | return new Text(sb.toString()); 35 | } 36 | 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/clustering/kmeans/KMeansVertexValue.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.clustering.kmeans; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | 7 | import ml.grafos.okapi.common.data.DoubleArrayListWritable; 8 | import org.apache.hadoop.io.IntWritable; 9 | import org.apache.hadoop.io.Writable; 10 | 11 | /** 12 | * The type of the vertex value in K-means 13 | * It stores the coordinates of the point 14 | * and the currently assigned cluster id 15 | * 16 | */ 17 | public class KMeansVertexValue implements Writable { 18 | private final DoubleArrayListWritable pointCoordinates; 19 | private IntWritable clusterId; 20 | 21 | public KMeansVertexValue(DoubleArrayListWritable coordinates, 22 | IntWritable id) { 23 | this.pointCoordinates = coordinates; 24 | this.clusterId = id; 25 | } 26 | 27 | public KMeansVertexValue() { 28 | this.pointCoordinates = new DoubleArrayListWritable(); 29 | this.clusterId = new IntWritable(); 30 | } 31 | 32 | public DoubleArrayListWritable getPointCoordinates() { 33 | return this.pointCoordinates; 34 | } 35 | 36 | public IntWritable getClusterId() { 37 | return this.clusterId; 38 | } 39 | 40 | public void setClusterId(IntWritable id) { 41 | this.clusterId = id; 42 | } 43 | 44 | @Override 45 | public void readFields(DataInput in) throws IOException { 46 | pointCoordinates.readFields(in); 47 | clusterId.readFields(in); 48 | } 49 | 50 | @Override 51 | public void write(DataOutput out) throws IOException { 52 | pointCoordinates.write(out); 53 | clusterId.write(out); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/common/Parameters.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.common; 17 | 18 | import org.apache.giraph.conf.BooleanConfOption; 19 | import org.apache.giraph.conf.LongConfOption; 20 | 21 | /** 22 | * Common Okapi parameters. 23 | */ 24 | public class Parameters { 25 | 26 | public static BooleanConfOption DEBUG = 27 | new BooleanConfOption("debug", false, 28 | "Enable debugging information printing"); 29 | 30 | public static LongConfOption RANDOM_SEED = 31 | new LongConfOption("random.seed", -1, 32 | "Random number generator seed"); 33 | 34 | public static BooleanConfOption GRAPH_DIRECTED = 35 | new BooleanConfOption("graph.directed", true, 36 | "Defines whether the graph is directed or not. " 37 | + "By default a graph is considered directed."); 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/common/computation/PropagateId.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.common.computation; 17 | 18 | import java.io.IOException; 19 | 20 | import org.apache.giraph.graph.AbstractComputation; 21 | import org.apache.giraph.graph.Vertex; 22 | import org.apache.hadoop.io.Writable; 23 | import org.apache.hadoop.io.WritableComparable; 24 | 25 | /** 26 | * This computation is used simply to propagate the ID of a vertex to all its 27 | * neighbors. 28 | * 29 | * @author dl 30 | * 31 | * @param 32 | * @param 33 | * @param 34 | * @param 35 | */ 36 | public class PropagateId 38 | extends AbstractComputation { 39 | 40 | @Override 41 | public void compute(Vertex vertex, 42 | Iterable messages) throws IOException { 43 | sendMessageToAllEdges(vertex, vertex.getId()); 44 | } 45 | } -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/common/computation/ReverseEdges.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.common.computation; 17 | 18 | import java.io.IOException; 19 | 20 | import ml.grafos.okapi.common.data.MessageWrapper; 21 | 22 | import org.apache.giraph.edge.Edge; 23 | import org.apache.giraph.edge.EdgeFactory; 24 | import org.apache.giraph.graph.AbstractComputation; 25 | import org.apache.giraph.graph.Vertex; 26 | import org.apache.hadoop.io.Writable; 27 | import org.apache.hadoop.io.WritableComparable; 28 | import org.apache.hadoop.util.ReflectionUtils; 29 | 30 | 31 | public class ReverseEdges 34 | extends AbstractComputation, M2> { 35 | 36 | @Override 37 | public void compute(Vertex vertex, 38 | Iterable> messages) throws IOException { 39 | for (MessageWrapper msg : messages) { 40 | E edgeValue = vertex.getEdgeValue(msg.getSourceId()); 41 | 42 | if (edgeValue == null) { 43 | I clonedId = null; 44 | E clonedEdgeValue = null; 45 | 46 | try { 47 | clonedId = (I)msg.getSourceId().getClass().newInstance(); 48 | clonedEdgeValue = (E)msg.getMessage().getClass().newInstance(); 49 | } catch (InstantiationException e) { 50 | throw new IOException(e); 51 | } catch (IllegalAccessException e) { 52 | throw new IOException(e); 53 | } 54 | 55 | ReflectionUtils.copy( 56 | getContext().getConfiguration(), msg.getSourceId(), clonedId); 57 | ReflectionUtils.copy( 58 | getContext().getConfiguration(), msg.getMessage(), clonedEdgeValue); 59 | 60 | Edge edge = EdgeFactory.create(clonedId, clonedEdgeValue); 61 | vertex.addEdge(edge); 62 | } 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/common/computation/SendFriends.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.common.computation; 17 | 18 | import java.io.IOException; 19 | 20 | import ml.grafos.okapi.common.data.MessageWrapper; 21 | 22 | import org.apache.giraph.edge.Edge; 23 | import org.apache.giraph.graph.BasicComputation; 24 | import org.apache.giraph.graph.Vertex; 25 | import org.apache.giraph.utils.ArrayListWritable; 26 | import org.apache.hadoop.io.Writable; 27 | import org.apache.hadoop.io.WritableComparable; 28 | import org.apache.hadoop.io.WritableUtils; 29 | 30 | /** 31 | * Many applications require a vertex to send a list of its friends to all 32 | * its neighbors in the graph. This is common in algorithms that require a 33 | * vertex to know its 2-hop neighborhood, such as triangle counting or computing 34 | * the clustering coefficient. 35 | * 36 | * Such algorithms can re-use this computation class. They only need to extend 37 | * it and specify the parameter types. 38 | * 39 | * @author dl 40 | * 41 | * @param 42 | * @param 43 | * @param 44 | * @param 45 | */ 46 | public abstract class SendFriends> 49 | extends BasicComputation { 50 | 51 | @Override 52 | public void compute(Vertex vertex, Iterable messages) 53 | throws IOException { 54 | 55 | final Vertex _vertex = vertex; 56 | 57 | final ArrayListWritable friends = new ArrayListWritable() { 58 | @Override 59 | public void setClass() { 60 | setClass(_vertex.getId().getClass()); 61 | } 62 | }; 63 | 64 | for (Edge edge : vertex.getEdges()) { 65 | friends.add(WritableUtils.clone(edge.getTargetVertexId(), getConf())); 66 | } 67 | 68 | MessageWrapper> msg = 69 | new MessageWrapper>() { 70 | 71 | @Override 72 | public Class getVertexIdClass() { 73 | return _vertex.getClass(); 74 | } 75 | 76 | @Override 77 | public Class getMessageClass() { 78 | return friends.getClass(); 79 | } 80 | }; 81 | 82 | msg.setSourceId(vertex.getId()); 83 | msg.setMessage(friends); 84 | sendMessageToAllEdges(vertex, (M)msg); 85 | } 86 | } -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/common/data/ArrayListOfDoubleArrayListWritable.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.common.data; 2 | 3 | import org.apache.giraph.utils.ArrayListWritable; 4 | 5 | public class ArrayListOfDoubleArrayListWritable extends ArrayListWritable { 6 | 7 | /** 8 | * 9 | */ 10 | private static final long serialVersionUID = -1547484036891955472L; 11 | 12 | @Override 13 | public void setClass() { 14 | setClass(DoubleArrayListWritable.class); 15 | } 16 | 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/common/data/DoubleArrayListWritable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.common.data; 17 | 18 | import org.apache.giraph.utils.ArrayListWritable; 19 | import org.apache.hadoop.io.DoubleWritable; 20 | import org.apache.hadoop.io.WritableComparable; 21 | 22 | public class DoubleArrayListWritable 23 | extends ArrayListWritable 24 | implements WritableComparable{ 25 | 26 | /** 27 | * Serialization number. 28 | */ 29 | private static final long serialVersionUID = 5968423975161632117L; 30 | 31 | /** Default constructor for reflection */ 32 | public DoubleArrayListWritable() { 33 | super(); 34 | } 35 | 36 | /** 37 | * Constructor 38 | * 39 | * @param other Other 40 | */ 41 | public DoubleArrayListWritable(DoubleArrayListWritable other) { 42 | super(other); 43 | } 44 | 45 | @Override 46 | public void setClass() { 47 | setClass(DoubleWritable.class); 48 | } 49 | 50 | /** 51 | * Print the Double Array List 52 | */ 53 | public void print() { 54 | System.out.print("["); 55 | for (int i = 0; i < this.size(); i++) { 56 | System.out.print(this.get(i)); 57 | if (i < this.size() - 1) { 58 | System.out.print("; "); 59 | } 60 | } 61 | System.out.println("]"); 62 | } 63 | 64 | /** 65 | * Compare function 66 | * 67 | * @param other Other array to be compared 68 | * 69 | * @return 0 70 | */ 71 | public int compareTo(DoubleArrayListWritable other) { 72 | if (other==null) { 73 | return 1; 74 | } 75 | if (this.size() < other.size()) { 76 | return -1; 77 | } 78 | if (this.size() > other.size()) { 79 | return 1; 80 | } 81 | for (int i=0; i < this.size(); i++) { 82 | if (this.get(i) == null && other.get(i) == null) { 83 | continue; 84 | } 85 | if (this.get(i) == null) { 86 | return -1; 87 | } 88 | if (other.get(i) == null) { 89 | return 1; 90 | } 91 | if (this.get(i).get() < other.get(i).get()) { 92 | return -1; 93 | } 94 | if (this.get(i).get() > other.get(i).get()) { 95 | return 1; 96 | } 97 | } 98 | return 0; 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/common/data/IntArrayListWritable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.common.data; 17 | 18 | import org.apache.giraph.utils.ArrayListWritable; 19 | import org.apache.hadoop.io.IntWritable; 20 | import org.apache.hadoop.io.WritableComparable; 21 | 22 | public class IntArrayListWritable 23 | extends ArrayListWritable 24 | implements WritableComparable { 25 | /** Default constructor for reflection */ 26 | public IntArrayListWritable() { 27 | super(); 28 | } 29 | 30 | @Override 31 | public void setClass() { 32 | setClass(IntWritable.class); 33 | } 34 | 35 | /** 36 | * public int compareTo(MessageWrapper wrapper) { if (this == wrapper ) 37 | * return 0; 38 | * 39 | * if (this.sourceId.compareTo(wrapper.getSourceId()) == 0) return 40 | * this.message.compareTo(wrapper.getMessage()); 41 | * else return this.sourceId.compareTo(wrapper.getSourceId()); } 42 | * 43 | * @param message Message to be compared 44 | * @return 0 value 45 | */ 46 | @Override 47 | public int compareTo(IntArrayListWritable message) { 48 | if (message==null) { 49 | return 1; 50 | } 51 | if (this.size()message.size()) { 55 | return 1; 56 | } 57 | for (int i=0; imessage.get(i).get()) { 71 | return 1; 72 | } 73 | } 74 | return 0; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/common/data/LongArrayListWritable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.common.data; 17 | 18 | import org.apache.giraph.utils.ArrayListWritable; 19 | import org.apache.hadoop.io.LongWritable; 20 | import org.apache.hadoop.io.WritableComparable; 21 | 22 | public class LongArrayListWritable 23 | extends ArrayListWritable 24 | implements WritableComparable { 25 | /** Default constructor for reflection */ 26 | public LongArrayListWritable() { 27 | super(); 28 | } 29 | 30 | @Override 31 | public void setClass() { 32 | setClass(LongWritable.class); 33 | } 34 | 35 | /** 36 | * public int compareTo(MessageWrapper wrapper) { if (this == wrapper ) 37 | * return 0; 38 | * 39 | * if (this.sourceId.compareTo(wrapper.getSourceId()) == 0) return 40 | * this.message.compareTo(wrapper.getMessage()); 41 | * else return this.sourceId.compareTo(wrapper.getSourceId()); } 42 | * 43 | * @param message Message to be compared 44 | * @return 0 value 45 | */ 46 | @Override 47 | public int compareTo(LongArrayListWritable message) { 48 | if (message==null) { 49 | return 1; 50 | } 51 | if (this.size()message.size()) { 55 | return 1; 56 | } 57 | for (int i=0; imessage.get(i).get()) { 71 | return 1; 72 | } 73 | } 74 | return 0; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/common/data/MapWritable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.common.data; 17 | 18 | /** 19 | * Extends the MapWritable from Hadoop and adds extra functionality. 20 | * @author dl 21 | * 22 | */ 23 | public class MapWritable extends org.apache.hadoop.io.MapWritable { 24 | 25 | @Override 26 | public String toString() { 27 | StringBuilder s = new StringBuilder(); 28 | for (Entry e : entrySet()) { 29 | s.append("("+e.getKey()+","+e.getValue()+")"); 30 | } 31 | return s.toString(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/common/graph/EdgeValueComparator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.common.graph; 17 | 18 | import java.util.Comparator; 19 | import java.util.Map; 20 | import java.util.Map.Entry; 21 | 22 | @SuppressWarnings("rawtypes") 23 | public class EdgeValueComparator 24 | implements Comparator> { 25 | 26 | @SuppressWarnings("unchecked") 27 | @Override 28 | public int compare(Entry e1, Entry e2) { 29 | int res = e1.getValue().compareTo(e2.getValue()); 30 | if (res == 0) { 31 | // equal values, compare vertex ids 32 | return e1.getKey().compareTo(e2.getKey()); 33 | } 34 | else { 35 | return res; 36 | } 37 | } 38 | 39 | 40 | } -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/common/graph/NullOutEdges.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.common.graph; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | import java.util.Iterator; 7 | 8 | import org.apache.giraph.edge.Edge; 9 | import org.apache.giraph.edge.OutEdges; 10 | import org.apache.hadoop.io.LongWritable; 11 | import org.apache.hadoop.io.NullWritable; 12 | 13 | public class NullOutEdges implements OutEdges { 14 | 15 | @Override 16 | public Iterator> iterator() { 17 | return null; 18 | } 19 | 20 | @Override 21 | public void readFields(DataInput arg0) throws IOException { 22 | // TODO Auto-generated method stub 23 | 24 | } 25 | 26 | @Override 27 | public void write(DataOutput arg0) throws IOException { 28 | // TODO Auto-generated method stub 29 | 30 | } 31 | 32 | @Override 33 | public void initialize(Iterable> edges) { 34 | // TODO Auto-generated method stub 35 | 36 | } 37 | 38 | @Override 39 | public void initialize(int capacity) { 40 | // TODO Auto-generated method stub 41 | 42 | } 43 | 44 | @Override 45 | public void initialize() { 46 | // TODO Auto-generated method stub 47 | 48 | } 49 | 50 | @Override 51 | public void add(Edge edge) { 52 | // TODO Auto-generated method stub 53 | 54 | } 55 | 56 | @Override 57 | public void remove(LongWritable targetVertexId) { 58 | // TODO Auto-generated method stub 59 | 60 | } 61 | 62 | @Override 63 | public int size() { 64 | // TODO Auto-generated method stub 65 | return 0; 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/common/jblas/FloatMatrixWritable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.common.jblas; 17 | 18 | import java.io.DataInput; 19 | import java.io.DataOutput; 20 | import java.io.IOException; 21 | import java.nio.ByteBuffer; 22 | import java.nio.FloatBuffer; 23 | 24 | import org.apache.hadoop.io.Writable; 25 | import org.jblas.FloatMatrix; 26 | import org.jblas.JavaBlas; 27 | 28 | /** 29 | * Extension to the jblas FloatMatrix class that also implements the Writable 30 | * interface. 31 | * 32 | * This class currently implements only a basic subset of the FloatMatrix 33 | * constructors. 34 | * 35 | * @author dl 36 | * 37 | */ 38 | public class FloatMatrixWritable extends FloatMatrix implements Writable { 39 | 40 | public FloatMatrixWritable() { 41 | super(); 42 | } 43 | 44 | public FloatMatrixWritable(int rows) { 45 | super(rows); 46 | } 47 | 48 | public FloatMatrixWritable(int rows, int columns, float... array) { 49 | super(rows, columns, array); 50 | } 51 | 52 | public FloatMatrixWritable(int rows, int columns) { 53 | super(rows, columns); 54 | } 55 | 56 | /** 57 | * Creates a copy of the input matrix. 58 | * 59 | * @param m 60 | */ 61 | public FloatMatrixWritable(FloatMatrix m) { 62 | super(m.rows, m.columns); 63 | JavaBlas.rcopy(m.length, m.data, 0, 1, this.data, 0, 1); 64 | } 65 | 66 | @Override 67 | public void readFields(DataInput input) throws IOException { 68 | int length = input.readInt(); 69 | this.rows = input.readInt(); 70 | this.columns = input.readInt(); 71 | byte[] array = new byte[length]; 72 | input.readFully(array); 73 | this.data = toFloatArray(array); 74 | this.length = data.length; 75 | } 76 | 77 | @Override 78 | public void write(DataOutput output) throws IOException { 79 | byte[] array = toByteArray(data); 80 | output.writeInt(array.length); 81 | output.writeInt(rows); 82 | output.writeInt(columns); 83 | output.write(array); 84 | } 85 | 86 | public byte[] toByteArray(float[] floatArray) { 87 | byte byteArray[] = new byte[floatArray.length*4]; 88 | ByteBuffer byteBuf = ByteBuffer.wrap(byteArray); 89 | FloatBuffer floatBuf = byteBuf.asFloatBuffer(); 90 | floatBuf.put (floatArray); 91 | return byteArray; 92 | } 93 | 94 | public float[] toFloatArray(byte[] byteArray) { 95 | float floatArray[] = new float[byteArray.length/4]; 96 | ByteBuffer byteBuf = ByteBuffer.wrap(byteArray); 97 | FloatBuffer floatBuf = byteBuf.asFloatBuffer(); 98 | floatBuf.get (floatArray); 99 | return floatArray; 100 | } 101 | 102 | } 103 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/examples/SimpleCountInEdgesVertex.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.examples; 17 | 18 | import org.apache.giraph.edge.Edge; 19 | import org.apache.giraph.examples.Algorithm; 20 | import org.apache.giraph.graph.BasicComputation; 21 | import org.apache.giraph.graph.Vertex; 22 | import org.apache.hadoop.io.DoubleWritable; 23 | import org.apache.hadoop.io.FloatWritable; 24 | import org.apache.hadoop.io.LongWritable; 25 | import org.apache.log4j.Logger; 26 | 27 | /** 28 | * Demonstrates the basic Pregel shortest paths implementation. 29 | */ 30 | @Algorithm( 31 | name = "Count input edges", 32 | description = "Counts input edges for each vertex") 33 | public class SimpleCountInEdgesVertex extends 34 | BasicComputation { 36 | /** Class logger */ 37 | private static final Logger LOG = 38 | Logger.getLogger(SimpleCountInEdgesVertex.class); 39 | 40 | /** 41 | * Compute method 42 | * @param messages Messages received 43 | */ 44 | public void compute( 45 | Vertex vertex, 46 | Iterable messages) { 47 | /** Initialize vertex value to zero */ 48 | if (getSuperstep() == 0) { 49 | vertex.setValue(new DoubleWritable(0d)); 50 | } 51 | /** Initialize counter */ 52 | double count = 0d; 53 | /** Count messages */ 54 | for (@SuppressWarnings("unused") 55 | DoubleWritable message : messages) { 56 | count += 1d; 57 | } 58 | if (LOG.isDebugEnabled()) { 59 | LOG.debug("Vertex " + vertex.getId() + 60 | " got a message. New total = " + count); 61 | } 62 | /** Save count into vertex value */ 63 | vertex.setValue(new DoubleWritable(count)); 64 | 65 | /** Send to all neighbors a message */ 66 | for (Edge edge : vertex.getEdges()) { 67 | if (LOG.isDebugEnabled()) { 68 | LOG.debug("Vertex " + vertex.getId() + " sent a message to " + 69 | edge.getTargetVertexId()); 70 | } 71 | if (getSuperstep() < 2) { 72 | sendMessage(edge.getTargetVertexId(), new DoubleWritable(1d)); 73 | } 74 | } 75 | vertex.voteToHalt(); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/examples/SimpleIdentityComputation.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.examples; 17 | 18 | import org.apache.giraph.examples.IdentityComputation; 19 | import org.apache.hadoop.io.DoubleWritable; 20 | import org.apache.hadoop.io.LongWritable; 21 | 22 | /** 23 | * This is simply an extension of the 24 | * {@link org.apache.giraph.examples.IdentityComputation IdentityComputation} 25 | * with specific parameter types. 26 | * 27 | * @author dl 28 | * 29 | */ 30 | public class SimpleIdentityComputation extends IdentityComputation { 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/examples/SimpleMasterComputeVertex.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.examples; 17 | 18 | import org.apache.giraph.aggregators.DoubleOverwriteAggregator; 19 | import org.apache.giraph.graph.BasicComputation; 20 | import org.apache.giraph.graph.Vertex; 21 | import org.apache.giraph.master.DefaultMasterCompute; 22 | import org.apache.giraph.worker.WorkerContext; 23 | import org.apache.hadoop.io.DoubleWritable; 24 | import org.apache.hadoop.io.FloatWritable; 25 | import org.apache.hadoop.io.LongWritable; 26 | import org.apache.log4j.Logger; 27 | 28 | /** 29 | * Demonstrates a computation with a centralized part implemented via a 30 | * MasterCompute. 31 | */ 32 | public class SimpleMasterComputeVertex extends BasicComputation { 34 | /** Aggregator to get values from the master to the workers */ 35 | public static final String SMC_AGG = "simplemastercompute.aggregator"; 36 | /** Logger */ 37 | private static final Logger LOG = 38 | Logger.getLogger(SimpleMasterComputeVertex.class); 39 | 40 | @Override 41 | public void compute( 42 | Vertex vertex, 43 | Iterable messages) { 44 | double oldSum = getSuperstep() == 0 ? 0 : vertex.getValue().get(); 45 | double newValue = this.getAggregatedValue(SMC_AGG).get(); 46 | double newSum = oldSum + newValue; 47 | vertex.setValue(new DoubleWritable(newSum)); 48 | 49 | SimpleMasterComputeWorkerContext workerContext = 50 | (SimpleMasterComputeWorkerContext) getWorkerContext(); 51 | workerContext.setFinalSum(newSum); 52 | LOG.info("Current sum: " + newSum); 53 | } 54 | 55 | /** 56 | * Worker context used with {@link SimpleMasterComputeVertex}. 57 | */ 58 | public static class SimpleMasterComputeWorkerContext 59 | extends WorkerContext { 60 | /** Final sum value for verification for local jobs */ 61 | private static double FINAL_SUM; 62 | 63 | @Override 64 | public void preApplication() 65 | throws InstantiationException, IllegalAccessException { 66 | } 67 | 68 | @Override 69 | public void preSuperstep() { 70 | } 71 | 72 | @Override 73 | public void postSuperstep() { 74 | } 75 | 76 | @Override 77 | public void postApplication() { 78 | } 79 | 80 | public void setFinalSum(double sum) { 81 | FINAL_SUM = sum; 82 | } 83 | 84 | public static double getFinalSum() { 85 | return FINAL_SUM; 86 | } 87 | } 88 | 89 | /** 90 | * MasterCompute used with {@link SimpleMasterComputeVertex}. 91 | */ 92 | public static class SimpleMasterCompute 93 | extends DefaultMasterCompute { 94 | @Override 95 | public void compute() { 96 | setAggregatedValue(SMC_AGG, 97 | new DoubleWritable(((double) getSuperstep()) / 2 + 1)); 98 | if (getSuperstep() == 10) { 99 | haltComputation(); 100 | } 101 | } 102 | 103 | @Override 104 | public void initialize() throws InstantiationException, 105 | IllegalAccessException { 106 | registerAggregator(SMC_AGG, DoubleOverwriteAggregator.class); 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/graphs/ConnectedComponents.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.graphs; 17 | 18 | import java.io.IOException; 19 | 20 | import org.apache.giraph.edge.Edge; 21 | import org.apache.giraph.graph.BasicComputation; 22 | import org.apache.giraph.graph.Vertex; 23 | import org.apache.hadoop.io.LongWritable; 24 | import org.apache.hadoop.io.NullWritable; 25 | 26 | /** 27 | * Implementation of the HCC algorithm that identifies connected components and 28 | * assigns each vertex its "component identifier" (the smallest vertex id 29 | * in the component) 30 | * 31 | * The idea behind the algorithm is very simple: propagate the smallest 32 | * vertex id along the edges to all vertices of a connected component. The 33 | * number of supersteps necessary is equal to the length of the maximum 34 | * diameter of all components + 1 35 | * 36 | * The original Hadoop-based variant of this algorithm was proposed by Kang, 37 | * Charalampos, Tsourakakis and Faloutsos in 38 | * "PEGASUS: Mining Peta-Scale Graphs", 2010 39 | * 40 | * http://www.cs.cmu.edu/~ukang/papers/PegasusKAIS.pdf 41 | */ 42 | public class ConnectedComponents extends BasicComputation { 44 | /** 45 | * Propagates the smallest vertex id to all neighbors. Will always choose to 46 | * halt and only reactivate if a smaller id has been sent to it. 47 | * 48 | * @param messages Iterator of messages from the previous superstep. 49 | * @throws IOException 50 | */ 51 | @Override 52 | public void compute( 53 | Vertex vertex, 54 | Iterable messages) throws IOException { 55 | long currentComponent = vertex.getValue().get(); 56 | 57 | // First superstep is special, because we can simply look at the neighbors 58 | if (getSuperstep() == 0) { 59 | for (Edge edge : vertex.getEdges()) { 60 | long neighbor = edge.getTargetVertexId().get(); 61 | if (neighbor < currentComponent) { 62 | currentComponent = neighbor; 63 | } 64 | } 65 | // Only need to send value if it is not the own id 66 | if (currentComponent != vertex.getValue().get()) { 67 | vertex.setValue(new LongWritable(currentComponent)); 68 | for (Edge edge : vertex.getEdges()) { 69 | LongWritable neighbor = edge.getTargetVertexId(); 70 | if (neighbor.get() > currentComponent) { 71 | sendMessage(neighbor, vertex.getValue()); 72 | } 73 | } 74 | } 75 | 76 | vertex.voteToHalt(); 77 | return; 78 | } 79 | 80 | boolean changed = false; 81 | // did we get a smaller id ? 82 | for (LongWritable message : messages) { 83 | long candidateComponent = message.get(); 84 | if (candidateComponent < currentComponent) { 85 | currentComponent = candidateComponent; 86 | changed = true; 87 | } 88 | } 89 | 90 | // propagate new component id to the neighbors 91 | if (changed) { 92 | vertex.setValue(new LongWritable(currentComponent)); 93 | sendMessageToAllEdges(vertex, vertex.getValue()); 94 | } 95 | vertex.voteToHalt(); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/graphs/KCore.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.graphs; 17 | 18 | import java.io.IOException; 19 | import java.util.HashSet; 20 | import java.util.Iterator; 21 | 22 | import org.apache.giraph.edge.MutableEdge; 23 | import org.apache.giraph.graph.BasicComputation; 24 | import org.apache.giraph.graph.Vertex; 25 | import org.apache.giraph.graph.VertexChanges; 26 | import org.apache.giraph.graph.VertexResolver; 27 | import org.apache.hadoop.io.LongWritable; 28 | import org.apache.hadoop.io.NullWritable; 29 | 30 | /** 31 | *

32 | * The k-core of a graph is the subgraph in which all vertices have 33 | * degree of at least k. 34 | *

35 | *

36 | * We find the k-core of a graph by iteratively removing vertices with degree 37 | * less than k. 38 | *

39 | *

40 | * The algorithms stops when there are no more vertex removals. At the end of 41 | * the execution, the remaining graph represents the k-core. It is possible that 42 | * the result in an empty graph. 43 | *

44 | *

45 | * http://en.wikipedia.org/wiki/Degeneracy_(graph_theory) 46 | *

47 | * @author dl 48 | * 49 | */ 50 | public class KCore { 51 | 52 | /** 53 | * K-value property. 54 | */ 55 | public static String K_VALUE = "core.k"; 56 | 57 | /** 58 | * Default K-value. 59 | */ 60 | public static int K_VALUE_DEFAULT = -1; 61 | 62 | public static class KCoreComputation extends BasicComputation{ 64 | 65 | @Override 66 | public void compute( 67 | Vertex vertex, 68 | Iterable messages) throws IOException { 69 | 70 | HashSet toDelete = new HashSet(); 71 | 72 | for (LongWritable id : messages) { 73 | toDelete.add(new LongWritable(id.get())); 74 | } 75 | 76 | Iterator> edgeIterator = 77 | vertex.getMutableEdges().iterator(); 78 | 79 | while (edgeIterator.hasNext()) { 80 | if (toDelete.contains(edgeIterator.next().getTargetVertexId())) { 81 | edgeIterator.remove(); 82 | } 83 | } 84 | 85 | if (vertex.getNumEdges() { 101 | 102 | @Override 103 | public Vertex resolve(LongWritable vertexId, Vertex vertex, 104 | VertexChanges vertexChanges, boolean hasMessages) { 105 | return null; 106 | } 107 | 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/graphs/SimplePageRank.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.graphs; 17 | 18 | import org.apache.giraph.graph.BasicComputation; 19 | import org.apache.giraph.graph.Vertex; 20 | import org.apache.hadoop.io.DoubleWritable; 21 | import org.apache.hadoop.io.FloatWritable; 22 | import org.apache.hadoop.io.LongWritable; 23 | import org.apache.log4j.Logger; 24 | 25 | /** 26 | * Basic Pregel PageRank implementation. 27 | * 28 | * This version initializes the value of every vertex to 1/N, where N is the 29 | * total number of vertices. 30 | * 31 | * The maximum number of supersteps is configurable. 32 | */ 33 | public class SimplePageRank extends BasicComputation { 35 | /** Default number of supersteps */ 36 | public static final int MAX_SUPERSTEPS_DEFAULT = 30; 37 | /** Property name for number of supersteps */ 38 | public static final String MAX_SUPERSTEPS = "pagerank.max.supersteps"; 39 | 40 | /** Logger */ 41 | private static final Logger LOG = 42 | Logger.getLogger(SimplePageRank.class); 43 | 44 | @Override 45 | public void compute( 46 | Vertex vertex, 47 | Iterable messages) { 48 | if (getSuperstep() == 0) { 49 | vertex.setValue(new DoubleWritable(1f / getTotalNumVertices())); 50 | } 51 | if (getSuperstep() >= 1) { 52 | double sum = 0; 53 | for (DoubleWritable message : messages) { 54 | sum += message.get(); 55 | } 56 | DoubleWritable vertexValue = 57 | new DoubleWritable((0.15f / getTotalNumVertices()) + 0.85f * sum); 58 | vertex.setValue(vertexValue); 59 | } 60 | 61 | if (getSuperstep() < getContext().getConfiguration().getInt( 62 | MAX_SUPERSTEPS, MAX_SUPERSTEPS_DEFAULT)) { 63 | 64 | long edges = vertex.getNumEdges(); 65 | sendMessageToAllEdges(vertex, 66 | new DoubleWritable(vertex.getValue().get() / edges)); 67 | } else { 68 | vertex.voteToHalt(); 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/graphs/SingleSourceShortestPaths.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.graphs; 17 | 18 | import org.apache.giraph.edge.Edge; 19 | import org.apache.giraph.examples.Algorithm; 20 | import org.apache.giraph.graph.BasicComputation; 21 | import org.apache.giraph.graph.Vertex; 22 | import org.apache.hadoop.io.DoubleWritable; 23 | import org.apache.hadoop.io.FloatWritable; 24 | import org.apache.hadoop.io.LongWritable; 25 | import org.apache.log4j.Logger; 26 | 27 | /** 28 | * Implementation of the single-source shortest paths algorithm. It finds the 29 | * shortest distances from a specified source to all other nodes in the graph. 30 | * The input graph can be directed or undirected. 31 | */ 32 | @Algorithm( 33 | name = "Shortest paths", 34 | description = "Finds all shortest paths from a selected vertex" 35 | ) 36 | public class SingleSourceShortestPaths extends BasicComputation { 38 | /** The shortest paths id */ 39 | public static final String SOURCE_ID = "sssp.source.id"; 40 | /** Default shortest paths id */ 41 | public static final long SOURCE_ID_DEFAULT = 1; 42 | /** Class logger */ 43 | private static final Logger LOG = 44 | Logger.getLogger(SingleSourceShortestPaths.class); 45 | 46 | /** 47 | * Is this vertex the source id? 48 | * 49 | * @return True if the source id 50 | */ 51 | private boolean isSource( 52 | Vertex vertex) { 53 | return vertex.getId().get() == 54 | getContext().getConfiguration().getLong(SOURCE_ID, 55 | SOURCE_ID_DEFAULT); 56 | } 57 | 58 | @Override 59 | public void compute( 60 | Vertex vertex, 61 | Iterable messages) { 62 | if (getSuperstep() == 0) { 63 | vertex.setValue(new DoubleWritable(Double.MAX_VALUE)); 64 | } 65 | 66 | // In directed graphs, vertices that have no outgoing edges will be created 67 | // in the 1st superstep as a result of messages sent to them. 68 | if (getSuperstep() == 1 && vertex.getNumEdges() == 0) { 69 | vertex.setValue(new DoubleWritable(Double.MAX_VALUE)); 70 | } 71 | 72 | double minDist = isSource(vertex) ? 0d : Double.MAX_VALUE; 73 | for (DoubleWritable message : messages) { 74 | minDist = Math.min(minDist, message.get()); 75 | } 76 | if (LOG.isDebugEnabled()) { 77 | LOG.debug("Vertex " + vertex.getId() + " got minDist = " + minDist + 78 | " vertex value = " + vertex.getValue()); 79 | } 80 | if (minDist < vertex.getValue().get()) { 81 | vertex.setValue(new DoubleWritable(minDist)); 82 | for (Edge edge : vertex.getEdges()) { 83 | double distance = minDist + edge.getValue().get(); 84 | if (LOG.isDebugEnabled()) { 85 | LOG.debug("Vertex " + vertex.getId() + " sent to " + 86 | edge.getTargetVertexId() + " = " + distance); 87 | } 88 | sendMessage(edge.getTargetVertexId(), new DoubleWritable(distance)); 89 | } 90 | } 91 | vertex.voteToHalt(); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/graphs/maxbmatching/MBMEdgeValue.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.graphs.maxbmatching; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | 9 | import org.apache.hadoop.io.Writable; 10 | 11 | public class MBMEdgeValue implements Writable { 12 | private double weight; 13 | private State state; 14 | 15 | public MBMEdgeValue() { 16 | this(0, State.DEFAULT); 17 | } 18 | 19 | public MBMEdgeValue(double weight) { 20 | this(weight, State.DEFAULT); 21 | } 22 | 23 | public MBMEdgeValue(double weight, State state) { 24 | this.setWeight(weight); 25 | this.setState(state); 26 | } 27 | 28 | public double getWeight() { 29 | return weight; 30 | } 31 | 32 | public void setWeight(double weight) { 33 | this.weight = weight; 34 | } 35 | 36 | public State getState() { 37 | return state; 38 | } 39 | 40 | public void setState(State state) { 41 | this.state = state; 42 | } 43 | 44 | @Override 45 | public String toString() { 46 | return String.valueOf(weight) + "\t" + state.toString(); 47 | } 48 | 49 | @Override 50 | public int hashCode() { 51 | final int prime = 31; 52 | int result = 1; 53 | result = prime * result + ((state == null) ? 0 : state.hashCode()); 54 | long temp; 55 | temp = Double.doubleToLongBits(weight); 56 | result = prime * result + (int) (temp ^ (temp >>> 32)); 57 | return result; 58 | } 59 | 60 | @Override 61 | public boolean equals(Object obj) { 62 | if (this == obj) 63 | return true; 64 | if (obj == null) 65 | return false; 66 | if (!(obj instanceof MBMEdgeValue)) 67 | return false; 68 | MBMEdgeValue other = (MBMEdgeValue) obj; 69 | if (state != other.state) 70 | return false; 71 | if (Double.doubleToLongBits(weight) != Double.doubleToLongBits(other.weight)) 72 | return false; 73 | return true; 74 | } 75 | 76 | @Override 77 | public void readFields(DataInput in) throws IOException { 78 | weight = in.readDouble(); 79 | state = State.fromValue(in.readByte()); 80 | } 81 | 82 | @Override 83 | public void write(DataOutput out) throws IOException { 84 | out.writeDouble(weight); 85 | out.writeByte(state.value()); 86 | } 87 | 88 | public static enum State { 89 | DEFAULT ((byte) 0), // starting state 90 | PROPOSED((byte) 1), // proposed for inclusion in the matching 91 | REMOVED ((byte) 2), // cannot be included in the matching 92 | INCLUDED((byte) 3); // included in the matching 93 | 94 | private final byte value; 95 | private static final Map lookup = new HashMap(); 96 | static { 97 | for (State s : values()) 98 | lookup.put(s.value, s); 99 | } 100 | 101 | State(byte value) { 102 | this.value = value; 103 | } 104 | 105 | public static State fromValue(byte value) { 106 | State result = lookup.get(value); 107 | if (result == null) 108 | throw new IllegalArgumentException("Cannot build edge State from illegal value: " + value); 109 | return result; 110 | } 111 | 112 | public byte value() { 113 | return value; 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/graphs/maxbmatching/MBMMessage.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.graphs.maxbmatching; 2 | 3 | import java.io.DataInput; 4 | import java.io.DataOutput; 5 | import java.io.IOException; 6 | 7 | import ml.grafos.okapi.graphs.maxbmatching.MBMEdgeValue.State; 8 | 9 | import org.apache.hadoop.io.LongWritable; 10 | import org.apache.hadoop.io.Writable; 11 | 12 | public class MBMMessage implements Writable { 13 | private LongWritable vertexID; 14 | private State state; 15 | 16 | public MBMMessage() { 17 | } 18 | 19 | public MBMMessage(LongWritable id, State proposed) { 20 | this.vertexID = id; 21 | this.state = proposed; 22 | } 23 | 24 | public LongWritable getId() { 25 | return vertexID; 26 | } 27 | 28 | public void setId(LongWritable id) { 29 | this.vertexID = id; 30 | } 31 | 32 | public State getState() { 33 | return state; 34 | } 35 | 36 | public void setState(State state) { 37 | this.state = state; 38 | } 39 | 40 | @Override 41 | public String toString() { 42 | return "(" + vertexID + ", " + state + ")"; 43 | } 44 | 45 | @Override 46 | public int hashCode() { 47 | final int prime = 31; 48 | int result = 1; 49 | result = prime * result + ((state == null) ? 0 : state.hashCode()); 50 | result = prime * result + ((vertexID == null) ? 0 : vertexID.hashCode()); 51 | return result; 52 | } 53 | 54 | @Override 55 | public boolean equals(Object obj) { 56 | if (this == obj) 57 | return true; 58 | if (obj == null) 59 | return false; 60 | if (!(obj instanceof MBMMessage)) 61 | return false; 62 | MBMMessage other = (MBMMessage) obj; 63 | if (state != other.state) 64 | return false; 65 | if (vertexID == null) { 66 | if (other.vertexID != null) 67 | return false; 68 | } else if (!vertexID.equals(other.vertexID)) 69 | return false; 70 | return true; 71 | } 72 | 73 | @Override 74 | public void readFields(DataInput in) throws IOException { 75 | vertexID = new LongWritable(in.readLong()); 76 | state = State.fromValue(in.readByte()); 77 | } 78 | 79 | @Override 80 | public void write(DataOutput out) throws IOException { 81 | out.writeLong(vertexID.get()); 82 | out.writeByte(state.value()); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/graphs/maxbmatching/MBMTextInputFormat.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.graphs.maxbmatching; 2 | 3 | import org.apache.giraph.edge.Edge; 4 | import org.apache.giraph.edge.EdgeFactory; 5 | import org.apache.giraph.io.formats.AdjacencyListTextVertexInputFormat; 6 | import org.apache.hadoop.io.IntWritable; 7 | import org.apache.hadoop.io.LongWritable; 8 | import org.apache.hadoop.mapreduce.InputSplit; 9 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 10 | 11 | /** 12 | * The input format for the maximum b-matching algorithm. The input is an undirected graph in adjacency list format, one vertex per line. The algorithm assumes 13 | * that each edge is present twice in the input, once for each end-point. The line is composed by a vertex, and a list of edges. The vertex is composed by a long 14 | * id, and an integer capacity. Each edge is composed by a long id (the destination vertex id), and a double weight. The elements of the list are tab or space 15 | * separated. 16 | * 17 | * E.g., "1 5 2 0.5 3 0.1" creates a vertex with id=1, capacity=5, and two edges to vertices 2 and 3, with weights 0.5 and 0.1, respectively. 18 | */ 19 | public class MBMTextInputFormat extends AdjacencyListTextVertexInputFormat { 20 | 21 | @Override 22 | public AdjacencyListTextVertexReader createVertexReader(InputSplit split, TaskAttemptContext context) { 23 | return new MBMVertexReader(); 24 | } 25 | 26 | public class MBMVertexReader extends AdjacencyListTextVertexReader { 27 | 28 | @Override 29 | public Edge decodeEdge(String e, String w) { 30 | return EdgeFactory.create(new LongWritable(Long.parseLong(e)), new MBMEdgeValue(Double.parseDouble(w))); 31 | } 32 | 33 | @Override 34 | public LongWritable decodeId(String s) { 35 | return new LongWritable(Long.parseLong(s)); 36 | } 37 | 38 | @Override 39 | public IntWritable decodeValue(String s) { 40 | return new IntWritable(Integer.parseInt(s)); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/graphs/maxbmatching/MBMTextOutputFormat.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.graphs.maxbmatching; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.giraph.edge.Edge; 6 | import org.apache.giraph.graph.Vertex; 7 | import org.apache.giraph.io.formats.AdjacencyListTextVertexOutputFormat; 8 | import org.apache.hadoop.io.IntWritable; 9 | import org.apache.hadoop.io.LongWritable; 10 | import org.apache.hadoop.io.Text; 11 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 12 | 13 | public class MBMTextOutputFormat extends AdjacencyListTextVertexOutputFormat { 14 | 15 | @Override 16 | public AdjacencyListTextVertexOutputFormat.AdjacencyListTextVertexWriter createVertexWriter( 17 | TaskAttemptContext context) { 18 | return new MBMVertexWriter(); 19 | } 20 | 21 | public class MBMVertexWriter extends AdjacencyListTextVertexWriter { 22 | protected String delimiter; 23 | 24 | @Override 25 | public void initialize(TaskAttemptContext context) throws IOException, InterruptedException { 26 | super.initialize(context); 27 | delimiter = getConf().get(LINE_TOKENIZE_VALUE, LINE_TOKENIZE_VALUE_DEFAULT); 28 | } 29 | 30 | @Override 31 | public Text convertVertexToLine(Vertex vertex) throws IOException { 32 | StringBuffer sb = new StringBuffer(vertex.getId().toString()); 33 | sb.append(delimiter); 34 | sb.append(vertex.getValue()); 35 | 36 | for (Edge edge : vertex.getEdges()) { 37 | sb.append(delimiter).append(edge.getTargetVertexId()); 38 | sb.append(delimiter).append(edge.getValue().getWeight()); 39 | // skip the state, which is always INCLUDED 40 | } 41 | return new Text(sb.toString()); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/AdjacencyListNoValuesTextVertexOutputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import java.io.IOException; 19 | 20 | import org.apache.giraph.edge.Edge; 21 | import org.apache.giraph.graph.Vertex; 22 | import org.apache.giraph.io.formats.AdjacencyListTextVertexInputFormat; 23 | import org.apache.giraph.io.formats.AdjacencyListTextVertexOutputFormat; 24 | import org.apache.giraph.io.formats.TextVertexOutputFormat; 25 | import org.apache.hadoop.io.Text; 26 | import org.apache.hadoop.io.Writable; 27 | import org.apache.hadoop.io.WritableComparable; 28 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 29 | 30 | /** 31 | * OutputFormat to write out the graph nodes as text, value-separated (by 32 | * tabs, by default). With the default delimiter, a vertex is written out as: 33 | * 34 | * []+ 35 | * 36 | * This is similar to {@link AdjacencyListTextVertexOutputFormat}, only it does 37 | * not write values. 38 | * 39 | * @param Vertex index value 40 | * @param Vertex value 41 | * @param Edge value 42 | */ 43 | @SuppressWarnings("rawtypes") 44 | public class AdjacencyListNoValuesTextVertexOutputFormat 46 | extends TextVertexOutputFormat { 47 | /** Split delimiter */ 48 | public static final String LINE_TOKENIZE_VALUE = "output.delimiter"; 49 | /** Default split delimiter */ 50 | public static final String LINE_TOKENIZE_VALUE_DEFAULT = 51 | AdjacencyListTextVertexInputFormat.LINE_TOKENIZE_VALUE_DEFAULT; 52 | 53 | @Override 54 | public AdjacencyListTextVertexWriter createVertexWriter( 55 | TaskAttemptContext context) { 56 | return new AdjacencyListTextVertexWriter(); 57 | } 58 | 59 | /** 60 | * Vertex writer associated with {@link AdjacencyListNoValuesTextVertexOutputFormat}. 61 | */ 62 | protected class AdjacencyListTextVertexWriter extends 63 | TextVertexWriterToEachLine { 64 | /** Cached split delimeter */ 65 | private String delimiter; 66 | 67 | @Override 68 | public void initialize(TaskAttemptContext context) throws IOException, 69 | InterruptedException { 70 | super.initialize(context); 71 | delimiter = 72 | getConf().get(LINE_TOKENIZE_VALUE, LINE_TOKENIZE_VALUE_DEFAULT); 73 | } 74 | 75 | @Override 76 | public Text convertVertexToLine(Vertex vertex) 77 | throws IOException { 78 | StringBuffer sb = new StringBuffer(vertex.getId().toString()); 79 | sb.append(delimiter); 80 | 81 | for (Edge edge : vertex.getEdges()) { 82 | sb.append(delimiter).append(edge.getTargetVertexId()); 83 | } 84 | 85 | return new Text(sb.toString()); 86 | } 87 | } 88 | 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/EdgesWithValuesVertexOutputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import org.apache.giraph.edge.Edge; 19 | import org.apache.giraph.graph.Vertex; 20 | import org.apache.giraph.io.formats.TextVertexOutputFormat; 21 | import org.apache.hadoop.io.Text; 22 | import org.apache.hadoop.io.Writable; 23 | import org.apache.hadoop.io.WritableComparable; 24 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 25 | 26 | import java.io.IOException; 27 | 28 | /** 29 | * OutputFormat to write out the graph edges with their values, 30 | * one edge per line: 31 | * 32 | * 33 | * 34 | * @param Vertex index value 35 | * @param Vertex value 36 | * @param Edge value 37 | */ 38 | @SuppressWarnings("rawtypes") 39 | public class EdgesWithValuesVertexOutputFormat 41 | extends TextVertexOutputFormat { 42 | /** Split delimiter */ 43 | public static final String LINE_TOKENIZE_VALUE = "output.delimiter"; 44 | /** Default split delimiter */ 45 | public static final String LINE_TOKENIZE_VALUE_DEFAULT = "\t"; 46 | 47 | @Override 48 | public EdgesWithValuesTextVertexWriter createVertexWriter( 49 | TaskAttemptContext context) { 50 | return new EdgesWithValuesTextVertexWriter(); 51 | } 52 | 53 | /** 54 | * Vertex writer associated with {@link EdgesWithValuesVertexOutputFormat}. 55 | */ 56 | protected class EdgesWithValuesTextVertexWriter extends TextVertexWriterToEachLine { 57 | /** Cached split delimeter */ 58 | private String delimiter; 59 | 60 | @Override 61 | public void initialize(TaskAttemptContext context) throws IOException, 62 | InterruptedException { 63 | super.initialize(context); 64 | delimiter = 65 | getConf().get(LINE_TOKENIZE_VALUE, LINE_TOKENIZE_VALUE_DEFAULT); 66 | } 67 | 68 | @Override 69 | public Text convertVertexToLine(Vertex vertex) 70 | throws IOException { 71 | StringBuffer sb = new StringBuffer(); 72 | 73 | for (Edge edge : vertex.getEdges()) { 74 | sb.append(vertex.getId().toString()); 75 | sb.append(delimiter).append(edge.getTargetVertexId()); 76 | sb.append(delimiter).append(edge.getValue()); 77 | sb.append('\n'); 78 | } 79 | sb.deleteCharAt(sb.length()-1); 80 | 81 | return new Text(sb.toString()); 82 | } 83 | } 84 | 85 | } -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/IntDoubleTextEdgeInputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import java.io.IOException; 19 | import java.util.regex.Pattern; 20 | 21 | import org.apache.giraph.io.EdgeReader; 22 | import org.apache.giraph.io.formats.IntNullTextEdgeInputFormat; 23 | import org.apache.giraph.io.formats.TextEdgeInputFormat; 24 | import org.apache.hadoop.io.DoubleWritable; 25 | import org.apache.hadoop.io.IntWritable; 26 | import org.apache.hadoop.io.Text; 27 | import org.apache.hadoop.mapreduce.InputSplit; 28 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 29 | 30 | /** 31 | * Simple text-based {@link org.apache.giraph.io.EdgeInputFormat} for 32 | * weighted graphs with int ids double values. 33 | * 34 | * Each line consists of: 35 | */ 36 | public class IntDoubleTextEdgeInputFormat extends 37 | TextEdgeInputFormat { 38 | /** Splitter for endpoints */ 39 | private static final Pattern SEPARATOR = Pattern.compile("[\t ]"); 40 | 41 | @Override 42 | public EdgeReader createEdgeReader( 43 | InputSplit split, TaskAttemptContext context) throws IOException { 44 | return new IntDoubleTextEdgeReader(); 45 | } 46 | 47 | /** 48 | * {@link org.apache.giraph.io.EdgeReader} associated with 49 | * {@link IntNullTextEdgeInputFormat}. 50 | */ 51 | public class IntDoubleTextEdgeReader extends 52 | TextEdgeReaderFromEachLineProcessed { 53 | @Override 54 | protected String[] preprocessLine(Text line) throws IOException { 55 | return SEPARATOR.split(line.toString()); 56 | } 57 | 58 | @Override 59 | protected IntWritable getSourceVertexId(String[] tokens) 60 | throws IOException { 61 | return new IntWritable(Integer.parseInt(tokens[0])); 62 | } 63 | 64 | @Override 65 | protected IntWritable getTargetVertexId(String[] tokens) 66 | throws IOException { 67 | return new IntWritable(Integer.parseInt(tokens[1])); 68 | } 69 | 70 | @Override 71 | protected DoubleWritable getValue(String[] tokens) throws IOException { 72 | return new DoubleWritable(Double.parseDouble(tokens[2])); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/IntIntDefaultEdgeValueTextEdgeInputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import java.io.IOException; 19 | import java.util.regex.Pattern; 20 | 21 | import org.apache.giraph.io.EdgeReader; 22 | import org.apache.giraph.io.formats.IntNullTextEdgeInputFormat; 23 | import org.apache.giraph.io.formats.TextEdgeInputFormat; 24 | import org.apache.hadoop.io.IntWritable; 25 | import org.apache.hadoop.io.Text; 26 | import org.apache.hadoop.mapreduce.InputSplit; 27 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 28 | 29 | /** 30 | * Simple text-based {@link org.apache.giraph.io.EdgeInputFormat} for 31 | * weighted graphs with int ids int values. In this format however, 32 | * the value of the edge is not expected in the input. 33 | * 34 | * Each line consists of: 35 | */ 36 | public class IntIntDefaultEdgeValueTextEdgeInputFormat extends 37 | TextEdgeInputFormat { 38 | /** Splitter for endpoints */ 39 | private static final Pattern SEPARATOR = Pattern.compile("[\t ]"); 40 | 41 | @Override 42 | public EdgeReader createEdgeReader( 43 | InputSplit split, TaskAttemptContext context) throws IOException { 44 | return new IntIntTextEdgeReader(); 45 | } 46 | 47 | /** 48 | * {@link org.apache.giraph.io.EdgeReader} associated with 49 | * {@link IntNullTextEdgeInputFormat}. 50 | */ 51 | public class IntIntTextEdgeReader extends 52 | TextEdgeReaderFromEachLineProcessed { 53 | @Override 54 | protected String[] preprocessLine(Text line) throws IOException { 55 | return SEPARATOR.split(line.toString()); 56 | } 57 | 58 | @Override 59 | protected IntWritable getSourceVertexId(String[] tokens) 60 | throws IOException { 61 | return new IntWritable(Integer.parseInt(tokens[0])); 62 | } 63 | 64 | @Override 65 | protected IntWritable getTargetVertexId(String[] tokens) 66 | throws IOException { 67 | return new IntWritable(Integer.parseInt(tokens[1])); 68 | } 69 | 70 | @Override 71 | protected IntWritable getValue(String tokens[]) throws IOException { 72 | return new IntWritable(-1); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/IntIntTextEdgeInputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import java.io.IOException; 19 | import java.util.regex.Pattern; 20 | 21 | import org.apache.giraph.io.EdgeReader; 22 | import org.apache.giraph.io.formats.IntNullTextEdgeInputFormat; 23 | import org.apache.giraph.io.formats.TextEdgeInputFormat; 24 | import org.apache.hadoop.io.IntWritable; 25 | import org.apache.hadoop.io.Text; 26 | import org.apache.hadoop.mapreduce.InputSplit; 27 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 28 | 29 | /** 30 | * Simple text-based {@link org.apache.giraph.io.EdgeInputFormat} for 31 | * weighted graphs with int ids int values. 32 | * 33 | * Each line consists of: 34 | */ 35 | public class IntIntTextEdgeInputFormat extends 36 | TextEdgeInputFormat { 37 | /** Splitter for endpoints */ 38 | private static final Pattern SEPARATOR = Pattern.compile("[\t ]"); 39 | 40 | @Override 41 | public EdgeReader createEdgeReader( 42 | InputSplit split, TaskAttemptContext context) throws IOException { 43 | return new IntIntTextEdgeReader(); 44 | } 45 | 46 | /** 47 | * {@link org.apache.giraph.io.EdgeReader} associated with 48 | * {@link IntNullTextEdgeInputFormat}. 49 | */ 50 | public class IntIntTextEdgeReader extends 51 | TextEdgeReaderFromEachLineProcessed { 52 | @Override 53 | protected String[] preprocessLine(Text line) throws IOException { 54 | return SEPARATOR.split(line.toString()); 55 | } 56 | 57 | @Override 58 | protected IntWritable getSourceVertexId(String[] tokens) 59 | throws IOException { 60 | return new IntWritable(Integer.parseInt(tokens[0])); 61 | } 62 | 63 | @Override 64 | protected IntWritable getTargetVertexId(String[] tokens) 65 | throws IOException { 66 | return new IntWritable(Integer.parseInt(tokens[1])); 67 | } 68 | 69 | @Override 70 | protected IntWritable getValue(String[] tokens) throws IOException { 71 | return new IntWritable(Integer.parseInt(tokens[2])); 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/LongDoubleBooleanEdgeInputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import java.io.IOException; 19 | import java.util.regex.Pattern; 20 | 21 | import ml.grafos.okapi.graphs.ScalableSemimetric.DoubleBooleanPair; 22 | 23 | import org.apache.giraph.io.EdgeReader; 24 | import org.apache.giraph.io.formats.TextEdgeInputFormat; 25 | import org.apache.hadoop.io.LongWritable; 26 | import org.apache.hadoop.io.Text; 27 | import org.apache.hadoop.mapreduce.InputSplit; 28 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 29 | 30 | /** 31 | * Simple text-based {@link org.apache.giraph.io.EdgeInputFormat} for 32 | * weighted graphs with long IDs edges with double-boolean pair values. 33 | * The boolean value of each edge is false by default. 34 | * 35 | * Each line consists of: 36 | */ 37 | public class LongDoubleBooleanEdgeInputFormat extends 38 | TextEdgeInputFormat { 39 | 40 | private static final Pattern SEPARATOR = Pattern.compile("[\t ]"); 41 | 42 | @Override 43 | public EdgeReader createEdgeReader( 44 | InputSplit split, TaskAttemptContext context) throws IOException { 45 | return new LongLongDoubleBooleanTextEdgeReader(); 46 | } 47 | 48 | /** 49 | * {@link org.apache.giraph.io.EdgeReader} associated with 50 | * {@link LongLongDoubleTextEdgeInputFormat}. 51 | */ 52 | public class LongLongDoubleBooleanTextEdgeReader extends 53 | TextEdgeReaderFromEachLineProcessed { 54 | @Override 55 | protected String[] preprocessLine(Text line) throws IOException { 56 | return SEPARATOR.split(line.toString()); 57 | } 58 | 59 | @Override 60 | protected LongWritable getSourceVertexId(String[] tokens) 61 | throws IOException { 62 | return new LongWritable(Long.parseLong(tokens[0])); 63 | } 64 | 65 | @Override 66 | protected LongWritable getTargetVertexId(String[] tokens) 67 | throws IOException { 68 | return new LongWritable(Long.parseLong(tokens[1])); 69 | } 70 | 71 | @Override 72 | protected DoubleBooleanPair getValue(String[] tokens) 73 | throws IOException { 74 | return new DoubleBooleanPair(Double.parseDouble(tokens[2]), false); 75 | } 76 | } 77 | } -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/LongDoubleDefaultEdgeValueTextEdgeInputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import java.io.IOException; 19 | import java.util.regex.Pattern; 20 | 21 | import org.apache.giraph.io.EdgeReader; 22 | import org.apache.giraph.io.formats.IntNullTextEdgeInputFormat; 23 | import org.apache.giraph.io.formats.TextEdgeInputFormat; 24 | import org.apache.hadoop.io.DoubleWritable; 25 | import org.apache.hadoop.io.LongWritable; 26 | import org.apache.hadoop.io.Text; 27 | import org.apache.hadoop.mapreduce.InputSplit; 28 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 29 | 30 | /** 31 | * Simple text-based {@link org.apache.giraph.io.EdgeInputFormat} for 32 | * weighted graphs with int ids int values. In this format however, 33 | * the value of the edge is not expected in the input. 34 | * 35 | * Each line consists of: 36 | */ 37 | public class LongDoubleDefaultEdgeValueTextEdgeInputFormat extends 38 | TextEdgeInputFormat { 39 | /** Splitter for endpoints */ 40 | private static final Pattern SEPARATOR = Pattern.compile("[\t ]"); 41 | 42 | @Override 43 | public EdgeReader createEdgeReader( 44 | InputSplit split, TaskAttemptContext context) throws IOException { 45 | return new LongDoubleTextEdgeReader(context); 46 | } 47 | 48 | /** 49 | * {@link org.apache.giraph.io.EdgeReader} associated with 50 | * {@link IntNullTextEdgeInputFormat}. 51 | */ 52 | public class LongDoubleTextEdgeReader extends 53 | TextEdgeReaderFromEachLineProcessed { 54 | 55 | DoubleWritable defaultEdgeValue; 56 | 57 | public LongDoubleTextEdgeReader(TaskAttemptContext context) { 58 | defaultEdgeValue = new DoubleWritable( 59 | Double.parseDouble( 60 | context.getConfiguration().get("edge.default.value"))); 61 | } 62 | 63 | @Override 64 | protected String[] preprocessLine(Text line) throws IOException { 65 | return SEPARATOR.split(line.toString()); 66 | } 67 | 68 | @Override 69 | protected LongWritable getSourceVertexId(String[] tokens) 70 | throws IOException { 71 | return new LongWritable(Long.parseLong(tokens[0])); 72 | } 73 | 74 | @Override 75 | protected LongWritable getTargetVertexId(String[] tokens) 76 | throws IOException { 77 | return new LongWritable(Long.parseLong(tokens[1])); 78 | } 79 | 80 | @Override 81 | protected DoubleWritable getValue(String tokens[]) throws IOException { 82 | return defaultEdgeValue; 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/LongDoubleDoubleAdjacencyListTextVertexOutputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import org.apache.giraph.io.formats.AdjacencyListTextVertexOutputFormat; 19 | import org.apache.hadoop.io.DoubleWritable; 20 | import org.apache.hadoop.io.LongWritable; 21 | 22 | public class LongDoubleDoubleAdjacencyListTextVertexOutputFormat extends 23 | AdjacencyListTextVertexOutputFormat { 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/LongDoubleReverseTextEdgeInputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import java.io.IOException; 19 | 20 | import org.apache.giraph.io.EdgeReader; 21 | import org.apache.giraph.io.ReverseEdgeDuplicator; 22 | import org.apache.hadoop.io.DoubleWritable; 23 | import org.apache.hadoop.io.LongWritable; 24 | import org.apache.hadoop.mapreduce.InputSplit; 25 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 26 | 27 | /** 28 | * Simple text-based {@link org.apache.giraph.io.EdgeInputFormat} for 29 | * weighted graphs with long IDs and double values. 30 | * 31 | * Note that this version also creates the reverse edges. 32 | * 33 | * Each line consists of: 34 | * 35 | */ 36 | public class LongDoubleReverseTextEdgeInputFormat 37 | extends LongDoubleTextEdgeInputFormat { 38 | @Override 39 | public EdgeReader createEdgeReader( 40 | InputSplit split, TaskAttemptContext context) throws IOException { 41 | EdgeReader edgeReader = 42 | super.createEdgeReader(split, context); 43 | edgeReader.setConf(getConf()); 44 | return new ReverseEdgeDuplicator(edgeReader); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/LongDoubleTextEdgeInputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import java.io.IOException; 19 | import java.util.regex.Pattern; 20 | 21 | import org.apache.giraph.io.EdgeReader; 22 | import org.apache.giraph.io.formats.TextEdgeInputFormat; 23 | import org.apache.hadoop.io.DoubleWritable; 24 | import org.apache.hadoop.io.LongWritable; 25 | import org.apache.hadoop.io.Text; 26 | import org.apache.hadoop.mapreduce.InputSplit; 27 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 28 | 29 | /** 30 | * Simple text-based {@link org.apache.giraph.io.EdgeInputFormat} for 31 | * weighted graphs with long IDs and double values. 32 | * 33 | * Each line consists of: 34 | */ 35 | public class LongDoubleTextEdgeInputFormat extends 36 | TextEdgeInputFormat { 37 | /** Splitter for endpoints */ 38 | private static final Pattern SEPARATOR = Pattern.compile("[\t ]"); 39 | 40 | @Override 41 | public EdgeReader createEdgeReader( 42 | InputSplit split, TaskAttemptContext context) throws IOException { 43 | return new LongLongDoubleTextEdgeReader(); 44 | } 45 | 46 | /** 47 | * {@link org.apache.giraph.io.EdgeReader} associated with 48 | * {@link LongLongDoubleTextEdgeInputFormat}. 49 | */ 50 | public class LongLongDoubleTextEdgeReader extends 51 | TextEdgeReaderFromEachLineProcessed { 52 | @Override 53 | protected String[] preprocessLine(Text line) throws IOException { 54 | return SEPARATOR.split(line.toString()); 55 | } 56 | 57 | @Override 58 | protected LongWritable getSourceVertexId(String[] tokens) 59 | throws IOException { 60 | return new LongWritable(Long.parseLong(tokens[0])); 61 | } 62 | 63 | @Override 64 | protected LongWritable getTargetVertexId(String[] tokens) 65 | throws IOException { 66 | return new LongWritable(Long.parseLong(tokens[1])); 67 | } 68 | 69 | @Override 70 | protected DoubleWritable getValue(String[] tokens) 71 | throws IOException { 72 | return new DoubleWritable(Double.parseDouble(tokens[2])); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/LongDoubleTextEdgeOutputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import org.apache.giraph.io.formats.SrcIdDstIdEdgeValueTextOutputFormat; 19 | import org.apache.hadoop.io.DoubleWritable; 20 | import org.apache.hadoop.io.LongWritable; 21 | 22 | /** 23 | * Used to output the edges of a graph along with their values. It assumes an 24 | * id of type long, a vertex value of type double (not used) and an edge value 25 | * of type double. 26 | * 27 | * @author dl 28 | * 29 | */ 30 | public class LongDoubleTextEdgeOutputFormat extends 31 | SrcIdDstIdEdgeValueTextOutputFormat { 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/LongDoubleZeroReverseTextEdgeInputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import java.io.IOException; 19 | 20 | import org.apache.giraph.io.EdgeReader; 21 | import org.apache.giraph.io.ReverseEdgeDuplicator; 22 | import org.apache.hadoop.io.DoubleWritable; 23 | import org.apache.hadoop.io.LongWritable; 24 | import org.apache.hadoop.mapreduce.InputSplit; 25 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 26 | 27 | /** 28 | * Simple text-based {@link org.apache.giraph.io.EdgeInputFormat} for 29 | * weighted graphs with long IDs and double values. 30 | * 31 | * Note that this version also creates the reverse edges. 32 | * 33 | * Each line consists of: 34 | * Reads an edge and also creates its reverse. 35 | * This format reads the src and trg ids and adds a zero weight on the edge. 36 | * 37 | */ 38 | public class LongDoubleZeroReverseTextEdgeInputFormat 39 | extends LongDoubleZerosTextEdgeInputFormat { 40 | @Override 41 | public EdgeReader createEdgeReader( 42 | InputSplit split, TaskAttemptContext context) throws IOException { 43 | EdgeReader edgeReader = 44 | super.createEdgeReader(split, context); 45 | edgeReader.setConf(getConf()); 46 | return new ReverseEdgeDuplicator(edgeReader); 47 | } 48 | } -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/LongDoubleZerosTextEdgeInputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import java.io.IOException; 19 | import java.util.regex.Pattern; 20 | 21 | import org.apache.giraph.io.EdgeReader; 22 | import org.apache.giraph.io.formats.TextEdgeInputFormat; 23 | import org.apache.hadoop.io.DoubleWritable; 24 | import org.apache.hadoop.io.LongWritable; 25 | import org.apache.hadoop.io.Text; 26 | import org.apache.hadoop.mapreduce.InputSplit; 27 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 28 | 29 | /** 30 | * Simple text-based {@link org.apache.giraph.io.EdgeInputFormat} for 31 | * weighted graphs with long IDs and double values. 32 | * 33 | * Each line consists of: 34 | * This format reads the src and trg ids and adds a zero weight on the edge 35 | */ 36 | public class LongDoubleZerosTextEdgeInputFormat extends 37 | TextEdgeInputFormat { 38 | /** Splitter for endpoints */ 39 | private static final Pattern SEPARATOR = Pattern.compile("[\t ]"); 40 | 41 | @Override 42 | public EdgeReader createEdgeReader( 43 | InputSplit split, TaskAttemptContext context) throws IOException { 44 | return new LongLongDoubleTextEdgeReader(); 45 | } 46 | 47 | /** 48 | * {@link org.apache.giraph.io.EdgeReader} associated with 49 | * {@link LongLongDoubleTextEdgeInputFormat}. 50 | */ 51 | public class LongLongDoubleTextEdgeReader extends 52 | TextEdgeReaderFromEachLineProcessed { 53 | @Override 54 | protected String[] preprocessLine(Text line) throws IOException { 55 | return SEPARATOR.split(line.toString()); 56 | } 57 | 58 | @Override 59 | protected LongWritable getSourceVertexId(String[] tokens) 60 | throws IOException { 61 | return new LongWritable(Long.parseLong(tokens[0])); 62 | } 63 | 64 | @Override 65 | protected LongWritable getTargetVertexId(String[] tokens) 66 | throws IOException { 67 | return new LongWritable(Long.parseLong(tokens[1])); 68 | } 69 | 70 | @Override 71 | protected DoubleWritable getValue(String[] tokens) 72 | throws IOException { 73 | return new DoubleWritable(0.0); 74 | } 75 | } 76 | } -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/LongFloatTextEdgeInputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import java.io.IOException; 19 | import java.util.regex.Pattern; 20 | 21 | import org.apache.giraph.io.EdgeReader; 22 | import org.apache.giraph.io.formats.TextEdgeInputFormat; 23 | import org.apache.hadoop.io.FloatWritable; 24 | import org.apache.hadoop.io.LongWritable; 25 | import org.apache.hadoop.io.Text; 26 | import org.apache.hadoop.mapreduce.InputSplit; 27 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 28 | 29 | /** 30 | * Simple text-based {@link org.apache.giraph.io.EdgeInputFormat} for 31 | * weighted graphs with long ids float values. 32 | * 33 | * Each line consists of: 34 | */ 35 | public class LongFloatTextEdgeInputFormat extends 36 | TextEdgeInputFormat { 37 | /** Splitter for endpoints */ 38 | private static final Pattern SEPARATOR = Pattern.compile("[\t ]"); 39 | 40 | @Override 41 | public EdgeReader createEdgeReader( 42 | InputSplit split, TaskAttemptContext context) throws IOException { 43 | return new LongLongFloatTextEdgeReader(); 44 | } 45 | 46 | /** 47 | * {@link org.apache.giraph.io.EdgeReader} associated with 48 | * {@link LongLongDoubleTextEdgeInputFormat}. 49 | */ 50 | public class LongLongFloatTextEdgeReader extends 51 | TextEdgeReaderFromEachLineProcessed { 52 | @Override 53 | protected String[] preprocessLine(Text line) throws IOException { 54 | return SEPARATOR.split(line.toString()); 55 | } 56 | 57 | @Override 58 | protected LongWritable getSourceVertexId(String[] tokens) 59 | throws IOException { 60 | return new LongWritable(Long.parseLong(tokens[0])); 61 | } 62 | 63 | @Override 64 | protected LongWritable getTargetVertexId(String[] tokens) 65 | throws IOException { 66 | return new LongWritable(Long.parseLong(tokens[1])); 67 | } 68 | 69 | @Override 70 | protected FloatWritable getValue(String[] tokens) 71 | throws IOException { 72 | return new FloatWritable(Float.parseFloat(tokens[2])); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/LongNullNullAdjacencyListTextVertexOutputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import org.apache.hadoop.io.LongWritable; 19 | import org.apache.hadoop.io.NullWritable; 20 | 21 | public class LongNullNullAdjacencyListTextVertexOutputFormat extends 22 | AdjacencyListNoValuesTextVertexOutputFormat< 23 | LongWritable, NullWritable, NullWritable> { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/LongNullTextEdgeInputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import java.io.IOException; 19 | import java.util.regex.Pattern; 20 | 21 | import org.apache.giraph.io.EdgeReader; 22 | import org.apache.giraph.io.formats.TextEdgeInputFormat; 23 | import org.apache.hadoop.io.LongWritable; 24 | import org.apache.hadoop.io.NullWritable; 25 | import org.apache.hadoop.io.Text; 26 | import org.apache.hadoop.mapreduce.InputSplit; 27 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 28 | 29 | /** 30 | * Simple text-based {@link org.apache.giraph.io.EdgeInputFormat} for 31 | * unweighted graphs with long ids. 32 | * 33 | * Each line consists of: 34 | */ 35 | public class LongNullTextEdgeInputFormat extends 36 | TextEdgeInputFormat { 37 | /** Splitter for endpoints */ 38 | private static final Pattern SEPARATOR = Pattern.compile("[\t ]"); 39 | 40 | @Override 41 | public EdgeReader createEdgeReader( 42 | InputSplit split, TaskAttemptContext context) throws IOException { 43 | return new LongNullTextEdgeReader(); 44 | } 45 | 46 | /** 47 | * {@link org.apache.giraph.io.EdgeReader} associated with 48 | * {@link LongNullTextEdgeInputFormat}. 49 | */ 50 | public class LongNullTextEdgeReader extends 51 | TextEdgeReaderFromEachLineProcessed { 52 | @Override 53 | protected String[] preprocessLine(Text line) throws IOException { 54 | return SEPARATOR.split(line.toString()); 55 | } 56 | 57 | @Override 58 | protected LongWritable getSourceVertexId(String[] tokens) 59 | throws IOException { 60 | return new LongWritable(Long.parseLong(tokens[0])); 61 | } 62 | 63 | @Override 64 | protected LongWritable getTargetVertexId(String[] tokens) 65 | throws IOException { 66 | return new LongWritable(Long.parseLong(tokens[1])); 67 | } 68 | 69 | @Override 70 | protected NullWritable getValue(String[] tokens) throws IOException { 71 | return NullWritable.get(); 72 | } 73 | } 74 | 75 | /** 76 | * A pair of longs 77 | */ 78 | public class LongPair { 79 | /** First element. */ 80 | private long first; 81 | /** Second element. */ 82 | private long second; 83 | 84 | /** Constructor. 85 | * 86 | * @param fst First element 87 | * @param snd Second element 88 | */ 89 | public LongPair(long fst, long snd) { 90 | first = fst; 91 | second = snd; 92 | } 93 | 94 | /** 95 | * Get the first element. 96 | * 97 | * @return The first element 98 | */ 99 | public long getFirst() { 100 | return first; 101 | } 102 | 103 | /** 104 | * Set the first element. 105 | * 106 | * @param first The first element 107 | */ 108 | public void setFirst(long first) { 109 | this.first = first; 110 | } 111 | 112 | /** 113 | * Get the second element. 114 | * 115 | * @return The second element 116 | */ 117 | public long getSecond() { 118 | return second; 119 | } 120 | 121 | /** 122 | * Set the second element. 123 | * 124 | * @param second The second element 125 | */ 126 | public void setSecond(long second) { 127 | this.second = second; 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/SimpleAdjacencyList.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import org.apache.hadoop.io.DoubleWritable; 19 | import org.apache.hadoop.io.LongWritable; 20 | 21 | /** 22 | * Writes the graph in an adjacency list format without any value or edge 23 | * values. This simply extends 24 | * {@link AdjacencyListNoValuesTextVertexOutputFormat} with specific types. 25 | * 26 | * @author dl 27 | * 28 | */ 29 | public class SimpleAdjacencyList extends 30 | AdjacencyListNoValuesTextVertexOutputFormat { 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/io/formats/TextDoubleTextEdgeInputFormat.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.io.formats; 17 | 18 | import java.io.IOException; 19 | import java.util.regex.Pattern; 20 | 21 | import org.apache.giraph.io.EdgeReader; 22 | import org.apache.giraph.io.formats.TextEdgeInputFormat; 23 | import org.apache.hadoop.io.DoubleWritable; 24 | import org.apache.hadoop.io.Text; 25 | import org.apache.hadoop.mapreduce.InputSplit; 26 | import org.apache.hadoop.mapreduce.TaskAttemptContext; 27 | 28 | /** 29 | * Simple text-based {@link org.apache.giraph.io.EdgeInputFormat} for 30 | * weighted graphs with Text ids and Double as weights 31 | * 32 | * Each line consists of: source_vertex, target_vertex, edge value 33 | */ 34 | public class TextDoubleTextEdgeInputFormat extends 35 | TextEdgeInputFormat { 36 | /** Splitter for endpoints */ 37 | private static final Pattern SEPARATOR = Pattern.compile("[\t ]"); 38 | 39 | @Override 40 | public EdgeReader createEdgeReader( 41 | InputSplit split, TaskAttemptContext context) throws IOException { 42 | return new TextDoubleTextEdgeReader(); 43 | } 44 | 45 | /** 46 | * {@link org.apache.giraph.io.EdgeReader} associated with 47 | * {@link TextDoubleTextEdgeInputFormat}. 48 | */ 49 | public class TextDoubleTextEdgeReader extends 50 | TextEdgeReaderFromEachLineProcessed { 51 | @Override 52 | protected String preprocessLine(Text line) throws IOException { 53 | return line.toString(); 54 | } 55 | 56 | @Override 57 | protected Text getSourceVertexId(String line) throws IOException { 58 | String[] tokens = SEPARATOR.split(line); 59 | return new Text(tokens[0]); 60 | } 61 | 62 | @Override 63 | protected Text getTargetVertexId(String line) throws IOException { 64 | String[] tokens = SEPARATOR.split(line); 65 | return new Text(tokens[1]); 66 | } 67 | 68 | @Override 69 | protected DoubleWritable getValue(String line) throws IOException { 70 | String[] tokens = SEPARATOR.split(line); 71 | return new DoubleWritable(Double.parseDouble(tokens[2])); 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/spinner/PartitionedLongWritable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.spinner; 17 | 18 | import java.io.DataInput; 19 | import java.io.DataOutput; 20 | import java.io.IOException; 21 | 22 | import org.apache.hadoop.io.WritableComparable; 23 | 24 | public class PartitionedLongWritable implements WritableComparable { 25 | public static final String DELIMITER = "_"; 26 | private short partition; 27 | private long id; 28 | 29 | public PartitionedLongWritable() { 30 | } 31 | 32 | public PartitionedLongWritable(String id) { 33 | String[] tokens = id.split(DELIMITER); 34 | this.partition = Short.parseShort(tokens[0]); 35 | this.id = Long.parseLong(tokens[1]); 36 | } 37 | 38 | @Override 39 | public void readFields(DataInput in) throws IOException { 40 | partition = in.readShort(); 41 | id = in.readLong(); 42 | } 43 | 44 | @Override 45 | public void write(DataOutput out) throws IOException { 46 | out.writeShort(partition); 47 | out.writeLong(id); 48 | } 49 | 50 | @Override 51 | public boolean equals(Object o) { 52 | if (this == o) { 53 | return true; 54 | } 55 | if (o == null || getClass() != o.getClass()) { 56 | return false; 57 | } 58 | PartitionedLongWritable other = (PartitionedLongWritable) o; 59 | if (this.partition == other.partition && this.id == other.id) { 60 | return true; 61 | } 62 | return false; 63 | } 64 | 65 | @Override 66 | public String toString() { 67 | return partition + DELIMITER + id; 68 | } 69 | 70 | @Override 71 | public int hashCode() { 72 | return (int) id; 73 | } 74 | 75 | public short getPartition() { 76 | return partition; 77 | } 78 | 79 | public long getId() { 80 | return id; 81 | } 82 | 83 | @Override 84 | public int compareTo(Object o) { 85 | if (o == this) { 86 | return 0; 87 | } 88 | PartitionedLongWritable other = (PartitionedLongWritable) o; 89 | return this.id > other.id ? +1 : this.id < other.id ? -1 : 0; 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/spinner/PrefixHashPartitionerFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.spinner; 17 | 18 | import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; 19 | import org.apache.giraph.partition.GraphPartitionerFactory; 20 | import org.apache.giraph.partition.HashMasterPartitioner; 21 | import org.apache.giraph.partition.MasterGraphPartitioner; 22 | import org.apache.giraph.partition.WorkerGraphPartitioner; 23 | import org.apache.giraph.worker.LocalData; 24 | import org.apache.hadoop.io.Writable; 25 | import org.apache.hadoop.io.WritableComparable; 26 | 27 | /* 28 | * Set it through giraph.graphPartitionerFactoryClass 29 | * expects I as PrefixIntWritable 30 | */ 31 | @SuppressWarnings("rawtypes") 32 | public class PrefixHashPartitionerFactory 33 | implements GraphPartitionerFactory { 34 | /** Saved configuration */ 35 | private ImmutableClassesGiraphConfiguration conf; 36 | 37 | @Override 38 | public MasterGraphPartitioner createMasterGraphPartitioner() { 39 | return new HashMasterPartitioner(getConf()); 40 | } 41 | 42 | @Override 43 | public WorkerGraphPartitioner createWorkerGraphPartitioner() { 44 | return new PrefixHashWorkerPartitioner(); 45 | } 46 | 47 | @Override 48 | public ImmutableClassesGiraphConfiguration getConf() { 49 | return conf; 50 | } 51 | 52 | @Override 53 | public void setConf(ImmutableClassesGiraphConfiguration conf) { 54 | this.conf = conf; 55 | } 56 | 57 | @Override 58 | public void initialize(LocalData localData) { } 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/spinner/PrefixHashWorkerPartitioner.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.spinner; 17 | 18 | import org.apache.giraph.partition.HashWorkerPartitioner; 19 | import org.apache.giraph.partition.PartitionOwner; 20 | import org.apache.hadoop.io.Writable; 21 | import org.apache.hadoop.io.WritableComparable; 22 | 23 | /* 24 | * expects I as PrefixIntWritable 25 | */ 26 | @SuppressWarnings("rawtypes") 27 | public class PrefixHashWorkerPartitioner 28 | extends HashWorkerPartitioner { 29 | 30 | @Override 31 | public PartitionOwner getPartitionOwner(I vertexId) { 32 | PartitionedLongWritable id = (PartitionedLongWritable) vertexId; 33 | return partitionOwnerList.get(Math.abs(id.getPartition() 34 | % partitionOwnerList.size())); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/utils/Counters.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.utils; 17 | 18 | import org.apache.hadoop.mapreduce.Counter; 19 | import org.apache.hadoop.mapreduce.Mapper.Context; 20 | 21 | /** 22 | * Utility class that helps maintain Hadoop counters. 23 | * 24 | * @author dl 25 | * 26 | */ 27 | public class Counters { 28 | 29 | /** 30 | * Replaces the value of a counter with a new one. 31 | * 32 | * @param context 33 | * @param counterGroup 34 | * @param counterName 35 | * @param newValue 36 | */ 37 | public static void updateCounter(Context context, String counterGroup, 38 | String counterName, long newValue) { 39 | 40 | Counter counter = context.getCounter(counterGroup, counterName); 41 | long oldValue = counter.getValue(); 42 | counter.increment(newValue-oldValue); 43 | } 44 | 45 | /** 46 | * Increments the value of a counter. 47 | * 48 | * @param context 49 | * @param counterGroup 50 | * @param counterName 51 | * @param increment 52 | */ 53 | public static void incrementCounter(Context context, String counterGroup, 54 | String counterName, long increment) { 55 | 56 | context.getCounter(counterGroup, counterName).increment(increment); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/ml/grafos/okapi/utils/WritableUtility.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.utils; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.io.Writable; 7 | import org.apache.hadoop.util.ReflectionUtils; 8 | 9 | /** 10 | * Contains helper methods for manipulating Writable objects. 11 | * 12 | * @author dl 13 | * 14 | */ 15 | public class WritableUtility { 16 | 17 | /** 18 | * Clones a writable object using the Hadoop ReflectionUtils object. 19 | * @param object The object to clone 20 | * @param conf The configuration 21 | * @return The cloned object 22 | * @throws InstantiationException 23 | * @throws IllegalAccessException 24 | * @throws IOException 25 | */ 26 | public static Writable clone(Writable object, Configuration conf) 27 | throws InstantiationException, IllegalAccessException, IOException { 28 | Writable cloned = null; 29 | cloned = object.getClass().newInstance(); 30 | ReflectionUtils.copy(conf, object, cloned); 31 | return cloned; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Set root logger level to DEBUG and its only appender to A1. 2 | log4j.rootLogger=ERROR, A1 3 | 4 | # A1 is set to be a ConsoleAppender. 5 | log4j.appender.A1=org.apache.log4j.ConsoleAppender 6 | 7 | # A1 uses PatternLayout. 8 | log4j.appender.A1.layout=org.apache.log4j.PatternLayout 9 | log4j.appender.A1.layout.ConversionPattern=%c %x - %m%n 10 | log4j.logger.org.apache=ERROR 11 | log4j.logger.ml.grafos=DEBUG -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/cf/CfLongIdFloatTextEdgeReaderTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf; 17 | 18 | import static org.junit.Assert.assertEquals; 19 | 20 | import java.io.IOException; 21 | 22 | import ml.grafos.okapi.cf.CfLongIdFloatTextInputFormat.CfIdFloatTextEdgeReader; 23 | 24 | import org.apache.hadoop.io.FloatWritable; 25 | import org.apache.hadoop.io.Text; 26 | import org.junit.Test; 27 | 28 | public class CfLongIdFloatTextEdgeReaderTest extends CfIdFloatTextEdgeReader { 29 | 30 | public CfLongIdFloatTextEdgeReaderTest() { 31 | new CfLongIdFloatTextInputFormat().super(); 32 | } 33 | 34 | @Test 35 | public void test() throws IOException { 36 | Text inputLine1 = new Text("1 2 5.0"); 37 | Text inputLine2 = new Text("1\t2\t5.000 "); 38 | 39 | CfLongId user = new CfLongId((byte)0, 1); 40 | CfLongId item = new CfLongId((byte)1, 2); 41 | FloatWritable rating = new FloatWritable(5f); 42 | 43 | String tokens[] = this.preprocessLine(inputLine1); 44 | assertEquals(user, this.getSourceVertexId(tokens)); 45 | assertEquals(item, this.getTargetVertexId(tokens)); 46 | assertEquals(rating, this.getValue(tokens)); 47 | 48 | tokens = this.preprocessLine(inputLine2); 49 | assertEquals(user, this.getSourceVertexId(tokens)); 50 | assertEquals(item, this.getTargetVertexId(tokens)); 51 | assertEquals(rating, this.getValue(tokens)); 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/cf/CfLongIdTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf; 17 | 18 | import static org.junit.Assert.assertEquals; 19 | import static org.junit.Assert.assertTrue; 20 | 21 | import java.io.ByteArrayInputStream; 22 | import java.io.ByteArrayOutputStream; 23 | import java.io.DataInputStream; 24 | import java.io.DataOutput; 25 | import java.io.DataOutputStream; 26 | import java.io.IOException; 27 | 28 | import org.junit.Test; 29 | 30 | public class CfLongIdTest { 31 | 32 | @Test 33 | public void testEmptySerialization() throws IOException { 34 | ByteArrayOutputStream baos = new ByteArrayOutputStream(10000); 35 | DataOutput output = new DataOutputStream(baos); 36 | 37 | CfLongId id = new CfLongId(); 38 | id.write(output); 39 | 40 | DataInputStream input = new DataInputStream(new ByteArrayInputStream( 41 | baos.toByteArray())); 42 | 43 | CfLongId idCopy = new CfLongId(); 44 | idCopy.readFields(input); 45 | 46 | assertTrue(id.equals(idCopy)); 47 | } 48 | 49 | @Test 50 | public void testSerialization() throws IOException { 51 | ByteArrayOutputStream baos = new ByteArrayOutputStream(10000); 52 | DataOutput output = new DataOutputStream(baos); 53 | 54 | CfLongId id = new CfLongId((byte)100, 200); 55 | id.write(output); 56 | 57 | DataInputStream input = new DataInputStream(new ByteArrayInputStream( 58 | baos.toByteArray())); 59 | 60 | CfLongId idCopy = new CfLongId(); 61 | idCopy.readFields(input); 62 | 63 | assertTrue(idCopy.getType()==100); 64 | assertTrue(idCopy.getId()==200); 65 | assertTrue(id.equals(idCopy)); 66 | } 67 | 68 | @Test 69 | public void testCompare() { 70 | CfLongId id1 = new CfLongId((byte)100, 200); 71 | CfLongId id2 = new CfLongId((byte)100, 200); 72 | CfLongId id3 = new CfLongId((byte)101, 200); 73 | CfLongId id4 = new CfLongId((byte)100, 199); 74 | assertEquals(id1, id2); 75 | assertTrue(!id1.equals(id3)); 76 | assertTrue(!id1.equals(null)); 77 | assertTrue(id1.compareTo(id2)==0); 78 | assertTrue(id1.compareTo(id3)==-1); 79 | assertTrue(id1.compareTo(id4)==1); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/cf/FloatMatrixMessageTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf; 17 | 18 | import static org.junit.Assert.assertEquals; 19 | import static org.junit.Assert.assertTrue; 20 | 21 | import java.io.ByteArrayInputStream; 22 | import java.io.ByteArrayOutputStream; 23 | import java.io.DataInputStream; 24 | import java.io.DataOutput; 25 | import java.io.DataOutputStream; 26 | import java.io.IOException; 27 | 28 | import ml.grafos.okapi.common.jblas.FloatMatrixWritable; 29 | 30 | import org.junit.Test; 31 | 32 | public class FloatMatrixMessageTest { 33 | 34 | @Test 35 | public void testSerialization() throws IOException { 36 | ByteArrayOutputStream baos = new ByteArrayOutputStream(10000); 37 | DataOutput output = new DataOutputStream(baos); 38 | 39 | CfLongId id = new CfLongId((byte)9, 111); 40 | FloatMatrixWritable fmw = 41 | new FloatMatrixWritable(2, 2, 0.1f, 0.5f, Float.NaN, Float.MAX_VALUE); 42 | FloatMatrixMessage msg = new FloatMatrixMessage(id, fmw, 1.2f); 43 | 44 | msg.write(output); 45 | 46 | DataInputStream input = new DataInputStream(new ByteArrayInputStream( 47 | baos.toByteArray())); 48 | 49 | FloatMatrixMessage msgCopy = new FloatMatrixMessage(); 50 | msgCopy.readFields(input); 51 | 52 | assertTrue(msg.equals(msgCopy)); 53 | } 54 | 55 | 56 | @Test 57 | public void testCompare() { 58 | FloatMatrixMessage msg1 = new FloatMatrixMessage( 59 | new CfLongId((byte)9,111), 60 | new FloatMatrixWritable(2, 2, 0.1f, 0.5f, Float.NaN, Float.MAX_VALUE), 61 | 1.2f); 62 | FloatMatrixMessage msg2 = new FloatMatrixMessage( 63 | new CfLongId((byte)9,111), 64 | new FloatMatrixWritable(2, 2, 0.1f, 0.5f, Float.NaN, Float.MAX_VALUE), 65 | 1.2f); 66 | FloatMatrixMessage msg3 = new FloatMatrixMessage( 67 | new CfLongId((byte)10,111), 68 | new FloatMatrixWritable(2, 2, 0.1f, 0.5f, Float.NaN, Float.MAX_VALUE), 69 | 1.2f); 70 | FloatMatrixMessage msg4 = new FloatMatrixMessage( 71 | new CfLongId((byte)9,111), 72 | new FloatMatrixWritable(2, 2, 0.2f, 0.5f, Float.NaN, Float.MAX_VALUE), 73 | 1.2f); 74 | FloatMatrixMessage msg5 = new FloatMatrixMessage( 75 | new CfLongId((byte)9,111), 76 | new FloatMatrixWritable(2, 2, 0.1f, 0.5f, Float.NaN, Float.MAX_VALUE), 77 | 1.3f); 78 | assertEquals(msg1, msg2); 79 | assertTrue(!msg1.equals(msg3)); 80 | assertTrue(!msg1.equals(msg4)); 81 | assertTrue(!msg1.equals(msg5)); 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/cf/als/AlsTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf.als; 17 | 18 | import java.util.LinkedList; 19 | import java.util.List; 20 | 21 | import ml.grafos.okapi.cf.CfLongIdFloatTextInputFormat; 22 | 23 | import org.apache.giraph.conf.GiraphConfiguration; 24 | import org.apache.giraph.io.formats.IdWithValueTextOutputFormat; 25 | import org.apache.giraph.utils.InternalVertexRunner; 26 | import org.jblas.FloatMatrix; 27 | import org.junit.Assert; 28 | import org.junit.Test; 29 | 30 | public class AlsTest { 31 | 32 | @Test 33 | public void testUpdateValue() { 34 | Als als = new Als(); 35 | 36 | float lambda = 0.01f; 37 | 38 | //user = (0.1, 0.2, 0.3) 39 | //item1 = (0.2, 0.1, 0.4) 40 | //item2 = (0.1, 0.1, 0.1) 41 | //item3 = (0.3, 0.1, 0.3) 42 | //item4 = (0.1, 0.1, 0.3) 43 | //ratings: 1.0 2.0 3.0 4.0 44 | 45 | FloatMatrix user = new FloatMatrix(1, 3, new float[]{0.1f, 0.2f, 0.3f}); 46 | FloatMatrix item1 = new FloatMatrix(1, 3, new float[]{0.2f, 0.1f, 0.4f}); 47 | FloatMatrix item2 = new FloatMatrix(1, 3, new float[]{0.1f, 0.1f, 0.1f}); 48 | FloatMatrix item3 = new FloatMatrix(1, 3, new float[]{0.3f, 0.1f, 0.3f}); 49 | FloatMatrix item4 = new FloatMatrix(1, 3, new float[]{0.1f, 0.1f, 0.3f}); 50 | 51 | FloatMatrix mat_M = new FloatMatrix(3,4); 52 | mat_M.putColumn(0, item1); 53 | mat_M.putColumn(1, item2); 54 | mat_M.putColumn(2, item3); 55 | mat_M.putColumn(3, item4); 56 | FloatMatrix mat_R = 57 | new FloatMatrix(4,1, new float[]{1.0f, 2.0f, 3.0f, 4.0f}); 58 | 59 | als.updateValue(user, mat_M, mat_R, lambda); 60 | 61 | Assert.assertArrayEquals(user.data, 62 | new float[] {2.598314f, 4.297752f, 4.311797f}, 0.00001f); 63 | } 64 | 65 | @Test 66 | public void testEndToEnd() throws Exception { 67 | String[] graph = { 68 | "1 1 1.0", 69 | "1 2 2.0", 70 | "2 1 3.0", 71 | "2 2 4.0" 72 | }; 73 | 74 | GiraphConfiguration conf = new GiraphConfiguration(); 75 | conf.setComputationClass(Als.InitUsersComputation.class); 76 | conf.setMasterComputeClass(Als.MasterCompute.class); 77 | conf.setEdgeInputFormatClass(CfLongIdFloatTextInputFormat.class); 78 | conf.setFloat(Als.LAMBDA, 0.01f); 79 | conf.setInt(Als.VECTOR_SIZE, 2); 80 | conf.setInt(Als.ITERATIONS, 4); 81 | conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class); 82 | Iterable results = InternalVertexRunner.run(conf, null, graph); 83 | List res = new LinkedList(); 84 | for (String string : results) { 85 | res.add(string); 86 | } 87 | Assert.assertEquals(4, res.size()); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/cf/eval/RankEvaluationComputationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf.eval; 17 | 18 | import static org.junit.Assert.assertEquals; 19 | 20 | import java.util.HashMap; 21 | import java.util.Map; 22 | 23 | import org.apache.giraph.conf.GiraphConfiguration; 24 | import org.apache.giraph.utils.InternalVertexRunner; 25 | import org.junit.After; 26 | import org.junit.Before; 27 | 28 | public class RankEvaluationComputationTest { 29 | 30 | @Before 31 | public void setUp() throws Exception { 32 | } 33 | 34 | @After 35 | public void tearDown() throws Exception { 36 | } 37 | 38 | 39 | public void testFullComputation() throws Exception { 40 | String[] graph = { 41 | "0 -1", 42 | "1 0 [1;0;0] -1,-2", 43 | "2 0 [2;0;0] -1", 44 | "-1 1 [1;1;1]", 45 | "-2 1 [0.5;1;1]", 46 | "-3 1 [0;0;0]", 47 | "-4 1 [0;0;0]"}; 48 | 49 | GiraphConfiguration conf = new GiraphConfiguration(); 50 | conf.setComputationClass(RankEvaluationComputation.class); 51 | conf.setVertexInputFormatClass(CfModelInputFormat.class); 52 | conf.setVertexOutputFormatClass(CFEvaluationOutputFormat.class); 53 | conf.set("minItemId", "-4"); 54 | conf.set("maxItemId", "-1"); 55 | conf.set("numberSamples", "1"); 56 | conf.set("k", "2"); 57 | Iterable results = InternalVertexRunner.run(conf, graph); 58 | 59 | String line = results.iterator().next(); 60 | line = line.replace("[", ""); 61 | line = line.replace("]", ""); 62 | line = line.replace(";", ""); 63 | assertEquals((1+0.5)/2.0, Double.parseDouble(line), 0.01); 64 | 65 | } 66 | 67 | 68 | private static Map parseResults(Iterable results) { 69 | Map values = new HashMap(); 70 | for (String line : results) { 71 | String[] tokens = line.split("\\s+"); 72 | int id = Integer.valueOf(tokens[0]); 73 | int value = Integer.valueOf(tokens[1]); 74 | values.put(id, value); 75 | } 76 | return values; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/cf/ranking/BPRRankingComputationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf.ranking; 17 | 18 | import java.util.LinkedList; 19 | import java.util.List; 20 | 21 | import ml.grafos.okapi.cf.CfLongIdFloatTextInputFormat; 22 | 23 | import org.apache.giraph.conf.GiraphConfiguration; 24 | import org.apache.giraph.io.formats.IdWithValueTextOutputFormat; 25 | import org.apache.giraph.utils.InternalVertexRunner; 26 | import org.junit.After; 27 | import org.junit.Assert; 28 | import org.junit.Before; 29 | import org.junit.Test; 30 | 31 | 32 | public class BPRRankingComputationTest { 33 | BPRRankingComputation bpr; 34 | 35 | @Before 36 | public void setUp() throws Exception { 37 | bpr = new BPRRankingComputation(); 38 | } 39 | 40 | @After 41 | public void tearDown() throws Exception { 42 | } 43 | 44 | @Test 45 | public void testFull() throws Exception{ 46 | String[] graph = { 47 | "1 1 1", 48 | "2 2 1", 49 | "3 3 1", 50 | "4 4 1", 51 | "4 5 1", 52 | "5 5 1", 53 | }; 54 | 55 | GiraphConfiguration conf = new GiraphConfiguration(); 56 | conf.setComputationClass(BPRRankingComputation.class); 57 | conf.setEdgeInputFormatClass(CfLongIdFloatTextInputFormat.class); 58 | conf.set("minItemId", "1"); 59 | conf.set("maxItemId", "5"); 60 | conf.set("iter", "1"); 61 | conf.set("dim", "5"); 62 | conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class); 63 | Iterable results = InternalVertexRunner.run(conf, null, graph); 64 | List res = new LinkedList(); 65 | for (String string : results) { 66 | res.add(string); 67 | Assert.assertEquals(5+1, string.split(";").length);//6 factors! 68 | } 69 | Assert.assertEquals(10, res.size()); 70 | 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/cf/ranking/PopularityRankingComputationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf.ranking; 17 | 18 | import java.util.LinkedList; 19 | import java.util.List; 20 | 21 | import junit.framework.Assert; 22 | import ml.grafos.okapi.cf.CfLongIdFloatTextInputFormat; 23 | 24 | import org.apache.giraph.conf.GiraphConfiguration; 25 | import org.apache.giraph.io.formats.IdWithValueTextOutputFormat; 26 | import org.apache.giraph.utils.InternalVertexRunner; 27 | import org.junit.Test; 28 | 29 | public class PopularityRankingComputationTest { 30 | 31 | @Test 32 | public void testFullComputation() throws Exception { 33 | String[] graph = { 34 | "1 1 1", 35 | "2 1 1" 36 | }; 37 | 38 | GiraphConfiguration conf = new GiraphConfiguration(); 39 | conf.setComputationClass(PopularityRankingComputation.class); 40 | conf.setEdgeInputFormatClass(CfLongIdFloatTextInputFormat.class); 41 | conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class); 42 | 43 | Iterable results = InternalVertexRunner.run(conf, null, graph); 44 | List res = new LinkedList(); 45 | for (String string : results) { 46 | res.add(string); 47 | //System.out.println(string); 48 | } 49 | Assert.assertEquals(3, res.size()); 50 | Assert.assertTrue(res.contains("1 0\t[1.000000]")); 51 | Assert.assertTrue(res.contains("2 0\t[1.000000]")); 52 | Assert.assertTrue(res.contains("1 1\t[2.000000]")); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/cf/ranking/RandomRankingComputationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf.ranking; 17 | 18 | import java.util.LinkedList; 19 | import java.util.List; 20 | 21 | import junit.framework.Assert; 22 | import ml.grafos.okapi.cf.CfLongIdFloatTextInputFormat; 23 | 24 | import org.apache.giraph.conf.GiraphConfiguration; 25 | import org.apache.giraph.io.formats.IdWithValueTextOutputFormat; 26 | import org.apache.giraph.utils.InternalVertexRunner; 27 | import org.junit.Before; 28 | import org.junit.Test; 29 | 30 | /** 31 | * @author User: linas 32 | * Date: 11/25/13 33 | */ 34 | public class RandomRankingComputationTest { 35 | 36 | static List res; 37 | static List res2; 38 | 39 | @Before 40 | public void setUp() throws Exception { 41 | String[] graph = { 42 | "1 1 1", 43 | "2 1 1" 44 | }; 45 | 46 | 47 | GiraphConfiguration conf = new GiraphConfiguration(); 48 | conf.setComputationClass(RandomRankingComputation.class); 49 | conf.setEdgeInputFormatClass(CfLongIdFloatTextInputFormat.class); 50 | conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class); 51 | 52 | RandomRankingComputation.setDim(2); 53 | Iterable results = InternalVertexRunner.run(conf, null, graph); 54 | res = new LinkedList(); 55 | for (String string : results) { 56 | res.add(string); 57 | } 58 | 59 | RandomRankingComputation.setDim(10); 60 | results = InternalVertexRunner.run(conf, null, graph); 61 | res2 = new LinkedList(); 62 | for (String string : results) { 63 | res2.add(string); 64 | } 65 | 66 | } 67 | 68 | @Test 69 | public void testFullComputation() throws Exception { 70 | Assert.assertEquals(3, res.size()); 71 | Assert.assertTrue(res.get(0).startsWith("1 0\t[")); 72 | Assert.assertTrue(res.get(1).startsWith("2 0\t[")); 73 | Assert.assertTrue(res.get(2).startsWith("1 1\t[")); 74 | } 75 | 76 | @Test 77 | public void testDimensionality() throws Exception { 78 | Assert.assertEquals(2, res.get(0).split(";").length); 79 | Assert.assertEquals(2, res.get(1).split(";").length); 80 | Assert.assertEquals(2, res.get(2).split(";").length); 81 | 82 | Assert.assertEquals(10, res2.get(0).split(";").length); 83 | Assert.assertEquals(10, res2.get(1).split(";").length); 84 | Assert.assertEquals(10, res2.get(2).split(";").length); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/cf/ranking/TFMAPRankingComputationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.cf.ranking; 17 | 18 | import org.junit.After; 19 | import org.junit.Before; 20 | 21 | 22 | public class TFMAPRankingComputationTest { 23 | 24 | TFMAPRankingComputation tfmap; 25 | 26 | @Before 27 | public void setUp() throws Exception { 28 | tfmap = new TFMAPRankingComputation(); 29 | } 30 | 31 | @After 32 | public void tearDown() throws Exception { 33 | } 34 | 35 | // @Test 36 | // public void testFull() throws Exception{ 37 | // String[] graph = { 38 | // "1 -1 1", 39 | // "1 -2 1", 40 | // "2 -1 1", 41 | // "3 -2 1", 42 | // "4 -1 1", 43 | // "4 -2 1", 44 | // "5 -3 1", 45 | // "6 -4 1", 46 | // "7 -5 1", 47 | // "8 -6 1", 48 | // "9 -7 1", 49 | // "10 -8 1" 50 | // }; 51 | // 52 | // GiraphConfiguration conf = new GiraphConfiguration(); 53 | // conf.setComputationClass(TFMAPRankingComputation.class); 54 | // conf.setEdgeInputFormatClass(LongLongIntInputFormat.class); 55 | // conf.set("minItemId", "-8"); 56 | // conf.set("maxItemId", "-1"); 57 | // conf.set("iter", "2"); 58 | // conf.set("bufferSize", "2"); 59 | // conf.setVertexOutputFormatClass(LongDoubleArrayListLongArrayListOutputFormat.class); 60 | // Iterable results = InternalVertexRunner.run(conf, null, graph); 61 | // List res = new LinkedList(); 62 | // for (String string : results) { 63 | // res.add(string); 64 | // System.out.println(string); 65 | // } 66 | // Assert.assertEquals(18, res.size()); 67 | // } 68 | // 69 | // @Test 70 | // public void testsampleRelevantAndIrrelevantEdges(){ 71 | // //lets setup vertex mock with one edge 72 | // 73 | // tfmap.setMaxItemId(-1); 74 | // tfmap.setMinItemId(-4); 75 | // TFMAPRankingComputation tfmapMock = spy(tfmap); 76 | // doNothing().when(tfmapMock).sendMessage(any(LongWritable.class), any(LongDoubleArrayListMessage.class)); 77 | // 78 | // Vertex vertex = mock(Vertex.class); 79 | // List> edges = new LinkedList>(); 80 | // edges.add(EdgeFactory.create(new LongWritable(-3), new IntWritable(1))); 81 | // edges.add(EdgeFactory.create(new LongWritable(-4), new IntWritable(1))); 82 | // 83 | // when(vertex.getId()).thenReturn(new LongWritable(1)); 84 | // when(vertex.getNumEdges()).thenReturn(2); 85 | // when(vertex.getEdges()).thenReturn(edges); 86 | // 87 | // tfmapMock.sampleRelevantAndIrrelevantEdges(vertex, 2); 88 | // verify(tfmapMock).sendRequestForFactors(-3, 1, true); 89 | // verify(tfmapMock).sendRequestForFactors(-4, 1, true); 90 | // verify(tfmapMock).sendRequestForFactors(-1, 1, false); 91 | // verify(tfmapMock).sendRequestForFactors(-2, 1, false); 92 | // 93 | // } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/clustering/kmeans/TestDoubleArrayListWritableAggregator.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.clustering.kmeans; 2 | 3 | import static org.junit.Assert.*; 4 | import ml.grafos.okapi.clustering.kmeans.DoubleArrayListWritableAggregator; 5 | import ml.grafos.okapi.common.data.DoubleArrayListWritable; 6 | 7 | import org.apache.hadoop.io.DoubleWritable; 8 | import org.junit.Test; 9 | 10 | public class TestDoubleArrayListWritableAggregator { 11 | private static double E = 0.0001f; 12 | 13 | @Test 14 | public void test() { 15 | DoubleArrayListWritableAggregator aggr = new DoubleArrayListWritableAggregator(); 16 | DoubleArrayListWritable other = new DoubleArrayListWritable(); 17 | other.add(new DoubleWritable(1.0)); 18 | other.add(new DoubleWritable(2.0)); 19 | aggr.aggregate(other); 20 | assertEquals(1.0, aggr.getAggregatedValue().get(0).get(), E); 21 | assertEquals(2.0, aggr.getAggregatedValue().get(1).get(), E); 22 | aggr.aggregate(other); 23 | assertEquals(2.0, aggr.getAggregatedValue().get(0).get(), E); 24 | assertEquals(4.0, aggr.getAggregatedValue().get(1).get(), E); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/clustering/kmeans/TestKMeansVertexValue.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.clustering.kmeans; 2 | 3 | import static org.junit.Assert.*; 4 | import ml.grafos.okapi.clustering.kmeans.KMeansVertexValue; 5 | import ml.grafos.okapi.common.data.DoubleArrayListWritable; 6 | 7 | import org.apache.giraph.utils.WritableUtils; 8 | import org.apache.hadoop.io.DoubleWritable; 9 | import org.apache.hadoop.io.IntWritable; 10 | import org.junit.Test; 11 | 12 | public class TestKMeansVertexValue { 13 | private static double E = 0.0001f; 14 | 15 | @Test 16 | public void testSerialize() { 17 | DoubleArrayListWritable coordinates = new DoubleArrayListWritable(); 18 | IntWritable clusterId = new IntWritable(5); 19 | coordinates.add(new DoubleWritable(1.0)); 20 | coordinates.add(new DoubleWritable(2.0)); 21 | coordinates.add(new DoubleWritable(3.0)); 22 | 23 | // Serialize from 24 | KMeansVertexValue from = new KMeansVertexValue(coordinates, clusterId); 25 | byte[] data = WritableUtils.writeToByteArray(from, from); 26 | 27 | // De-serialize to 28 | KMeansVertexValue to1 = new KMeansVertexValue(); 29 | KMeansVertexValue to2 = new KMeansVertexValue(); 30 | 31 | WritableUtils.readFieldsFromByteArray(data, to1, to2); 32 | 33 | // all coordinates should be equal 34 | assertEquals(from.getPointCoordinates().get(0).get(), to1.getPointCoordinates().get(0).get(), E); 35 | assertEquals(from.getPointCoordinates().get(1).get(), to1.getPointCoordinates().get(1).get(), E); 36 | assertEquals(from.getPointCoordinates().get(2).get(), to1.getPointCoordinates().get(2).get(), E); 37 | 38 | assertEquals(from.getPointCoordinates().get(0).get(), to2.getPointCoordinates().get(0).get(), E); 39 | assertEquals(from.getPointCoordinates().get(1).get(), to2.getPointCoordinates().get(1).get(), E); 40 | assertEquals(from.getPointCoordinates().get(2).get(), to2.getPointCoordinates().get(2).get(), E); 41 | 42 | // cluster ids should be equal 43 | assertEquals(from.getClusterId().get(), to1.getClusterId().get()); 44 | assertEquals(from.getClusterId().get(), to1.getClusterId().get()); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/clustering/kmeans/TestRandomInitializationAggregator.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.clustering.kmeans; 2 | 3 | import static org.junit.Assert.*; 4 | import ml.grafos.okapi.clustering.kmeans.ArrayListOfDoubleArrayListWritableAggregator; 5 | import ml.grafos.okapi.common.data.ArrayListOfDoubleArrayListWritable; 6 | import ml.grafos.okapi.common.data.DoubleArrayListWritable; 7 | 8 | import org.apache.giraph.conf.GiraphConfiguration; 9 | import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; 10 | import org.apache.hadoop.io.DoubleWritable; 11 | import org.junit.Test; 12 | 13 | public class TestRandomInitializationAggregator { 14 | private static double E = 0.0001f; 15 | 16 | @SuppressWarnings({ "unchecked", "rawtypes" }) 17 | @Test 18 | public void test() { 19 | 20 | GiraphConfiguration conf = new GiraphConfiguration(); 21 | ImmutableClassesGiraphConfiguration immConf = new ImmutableClassesGiraphConfiguration(conf); 22 | immConf.setInt(ArrayListOfDoubleArrayListWritableAggregator.CLUSTER_CENTERS_COUNT, 2); 23 | immConf.setInt(ArrayListOfDoubleArrayListWritableAggregator.POINTS_COUNT, 7); 24 | 25 | ArrayListOfDoubleArrayListWritableAggregator aggr = new ArrayListOfDoubleArrayListWritableAggregator(); 26 | aggr.setConf(immConf); 27 | 28 | ArrayListOfDoubleArrayListWritable other = new ArrayListOfDoubleArrayListWritable(); 29 | DoubleArrayListWritable value = new DoubleArrayListWritable(); 30 | value.add(new DoubleWritable(1.0)); 31 | value.add(new DoubleWritable(2.0)); 32 | other.add(value); 33 | 34 | aggr.aggregate(other); 35 | assertEquals(1.0, aggr.getAggregatedValue().get(0).get(0).get(), E); 36 | assertEquals(2.0, aggr.getAggregatedValue().get(0).get(1).get(), E); 37 | 38 | ArrayListOfDoubleArrayListWritable other2 = new ArrayListOfDoubleArrayListWritable(); 39 | DoubleArrayListWritable value2 = new DoubleArrayListWritable(); 40 | value2.add(new DoubleWritable(3.0)); 41 | value2.add(new DoubleWritable(4.0)); 42 | other2.add(value2); 43 | 44 | aggr.aggregate(other2); 45 | assertEquals(3.0, aggr.getAggregatedValue().get(1).get(0).get(), E); 46 | assertEquals(4.0, aggr.getAggregatedValue().get(1).get(1).get(), E); 47 | 48 | ArrayListOfDoubleArrayListWritable other3 = new ArrayListOfDoubleArrayListWritable(); 49 | DoubleArrayListWritable value3 = new DoubleArrayListWritable(); 50 | value3.add(new DoubleWritable(5.0)); 51 | value3.add(new DoubleWritable(6.0)); 52 | other3.add(value3); 53 | 54 | ArrayListOfDoubleArrayListWritable other4 = new ArrayListOfDoubleArrayListWritable(); 55 | DoubleArrayListWritable value4 = new DoubleArrayListWritable(); 56 | value4.add(new DoubleWritable(7.0)); 57 | value4.add(new DoubleWritable(8.0)); 58 | other4.add(value4); 59 | 60 | aggr.aggregate(other3); 61 | aggr.aggregate(other4); 62 | 63 | assertEquals(2, aggr.getAggregatedValue().size()); 64 | System.out.println(aggr.getAggregatedValue().get(0).get(0).get() + ", " +aggr.getAggregatedValue().get(0).get(1).get()); 65 | System.out.println(aggr.getAggregatedValue().get(1).get(0).get() + ", " +aggr.getAggregatedValue().get(1).get(1).get()); 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/common/jblas/FloatMatrixWritableTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.common.jblas; 17 | 18 | import static org.junit.Assert.assertArrayEquals; 19 | import static org.junit.Assert.assertTrue; 20 | 21 | import java.io.ByteArrayInputStream; 22 | import java.io.ByteArrayOutputStream; 23 | import java.io.DataInputStream; 24 | import java.io.DataOutput; 25 | import java.io.DataOutputStream; 26 | import java.io.IOException; 27 | 28 | import junit.framework.Assert; 29 | 30 | import org.jblas.FloatMatrix; 31 | import org.junit.Before; 32 | import org.junit.Test; 33 | 34 | public class FloatMatrixWritableTest { 35 | 36 | private FloatMatrixWritable fmw; 37 | 38 | @Before 39 | public void setUp() throws Exception { 40 | fmw = new FloatMatrixWritable(); 41 | } 42 | 43 | @Test 44 | public void testReadWrite() throws IOException { 45 | ByteArrayOutputStream baos = new ByteArrayOutputStream(10000); 46 | fmw = new FloatMatrixWritable(2, 2, 0.1f, 0.5f, Float.NaN, Float.MAX_VALUE); 47 | DataOutput output = new DataOutputStream(baos); 48 | fmw.write(output); 49 | FloatMatrixWritable fmwCopy = new FloatMatrixWritable(); 50 | DataInputStream input = new DataInputStream(new ByteArrayInputStream( 51 | baos.toByteArray())); 52 | fmwCopy.readFields(input); 53 | assertArrayEquals(fmw.toArray(), fmwCopy.toArray(), 0.001f); 54 | assertTrue(fmw.equals(fmwCopy)); 55 | 56 | FloatMatrixWritable row = new FloatMatrixWritable(1, 2, 0.1f, 0.5f); 57 | assertTrue(row.equals(fmwCopy.getRow(0))); 58 | } 59 | 60 | @Test 61 | public void testFloatToByteAndBack() { 62 | float[] input = { 0.1f, 0.5f, Float.NaN, Float.MAX_VALUE, Float.MIN_VALUE, 63 | Float.NEGATIVE_INFINITY }; 64 | assertArrayEquals(input, fmw.toFloatArray(fmw.toByteArray(input)), 0.001f); 65 | } 66 | 67 | @Test 68 | public void testConstructor() { 69 | FloatMatrix fm = 70 | new FloatMatrix(2, 2, 0.1f, 0.5f, Float.NaN, Float.MAX_VALUE); 71 | FloatMatrixWritable fmw = 72 | new FloatMatrixWritable(2, 2, 0.1f, 0.5f, Float.NaN, Float.MAX_VALUE); 73 | FloatMatrixWritable fmwDup = new FloatMatrixWritable(fm); 74 | Assert.assertEquals(fmwDup, fmw); 75 | } 76 | 77 | } 78 | -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/graphs/ClusteringCoefficientTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.graphs; 17 | 18 | import static org.junit.Assert.*; 19 | 20 | import java.util.LinkedList; 21 | import java.util.List; 22 | 23 | import junit.framework.Assert; 24 | import ml.grafos.okapi.io.formats.LongNullTextEdgeInputFormat; 25 | 26 | import org.apache.giraph.conf.GiraphConfiguration; 27 | import org.apache.giraph.io.formats.IdWithValueTextOutputFormat; 28 | import org.apache.giraph.utils.InternalVertexRunner; 29 | import org.junit.Test; 30 | 31 | public class ClusteringCoefficientTest { 32 | 33 | @Test 34 | public void test() { 35 | String[] graph = { 36 | "1 2 1.0", 37 | "2 1 1.0", 38 | "1 3 1.0", 39 | "3 1 1.0", 40 | "2 3 2.0", 41 | "3 2 2.0", 42 | "3 4 2.0", 43 | "4 3 2.0", 44 | "3 5 1.0", 45 | "5 3 1.0", 46 | "4 5 1.0", 47 | "5 4 1.0" 48 | }; 49 | 50 | GiraphConfiguration conf = new GiraphConfiguration(); 51 | conf.setComputationClass(ClusteringCoefficient.SendFriendsList.class); 52 | conf.setMasterComputeClass(ClusteringCoefficient.MasterCompute.class); 53 | conf.setEdgeInputFormatClass(LongNullTextEdgeInputFormat.class); 54 | conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class); 55 | Iterable results; 56 | try { 57 | results = InternalVertexRunner.run(conf, null, graph); 58 | } catch (Exception e) { 59 | e.printStackTrace(); 60 | fail("Exception occurred"); 61 | return; 62 | } 63 | List res = new LinkedList(); 64 | for (String string : results) { 65 | res.add(string); 66 | System.out.println(string); 67 | } 68 | Assert.assertEquals(5, res.size()); 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/graphs/MultipleSourceShortestPathsTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.graphs; 17 | 18 | import static org.junit.Assert.*; 19 | 20 | import java.util.LinkedList; 21 | import java.util.List; 22 | 23 | import junit.framework.Assert; 24 | import ml.grafos.okapi.common.Parameters; 25 | import ml.grafos.okapi.io.formats.LongFloatTextEdgeInputFormat; 26 | 27 | import org.apache.giraph.conf.GiraphConfiguration; 28 | import org.apache.giraph.io.formats.IdWithValueTextOutputFormat; 29 | import org.apache.giraph.utils.InternalVertexRunner; 30 | import org.junit.Test; 31 | 32 | public class MultipleSourceShortestPathsTest { 33 | 34 | @Test 35 | public void test() { 36 | String[] graph = { 37 | "1 2 1.0", 38 | "2 1 1.0", 39 | "1 3 1.0", 40 | "3 1 1.0", 41 | "2 3 2.0", 42 | "3 2 2.0", 43 | "3 4 2.0", 44 | "4 3 2.0", 45 | "3 5 1.0", 46 | "5 3 1.0", 47 | "4 5 1.0", 48 | "5 4 1.0" 49 | }; 50 | 51 | GiraphConfiguration conf = new GiraphConfiguration(); 52 | conf.setComputationClass(MultipleSourceShortestPaths.InitSources.class); 53 | conf.setMasterComputeClass(MultipleSourceShortestPaths.MasterCompute.class); 54 | conf.setEdgeInputFormatClass(LongFloatTextEdgeInputFormat.class); 55 | conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class); 56 | conf.setFloat(MultipleSourceShortestPaths.SOURCES_FRACTION, 0.4f); 57 | conf.setLong(Parameters.RANDOM_SEED.getKey(), 0); 58 | Iterable results; 59 | try { 60 | results = InternalVertexRunner.run(conf, null, graph); 61 | } catch (Exception e) { 62 | e.printStackTrace(); 63 | fail("Exception occurred"); 64 | return; 65 | } 66 | List res = new LinkedList(); 67 | for (String string : results) { 68 | res.add(string); 69 | System.out.println(string); 70 | } 71 | Assert.assertEquals(5, res.size()); 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/graphs/ScalableSemimetricTest.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.graphs; 2 | 3 | import java.util.ArrayList; 4 | 5 | import junit.framework.Assert; 6 | import ml.grafos.okapi.io.formats.EdgesWithValuesVertexOutputFormat; 7 | import ml.grafos.okapi.io.formats.LongDoubleBooleanEdgeInputFormat; 8 | 9 | import org.apache.giraph.conf.GiraphConfiguration; 10 | import org.apache.giraph.edge.HashMapEdges; 11 | import org.apache.giraph.utils.InternalVertexRunner; 12 | import org.junit.Test; 13 | 14 | public class ScalableSemimetricTest { 15 | 16 | @Test 17 | public void testTwoSemimetric() throws Exception { 18 | String[] graph = new String[] { 19 | "1 2 1.0", 20 | "2 1 1.0", 21 | "2 3 1.0", 22 | "3 2 1.0", 23 | "3 1 10.0", 24 | "1 3 10.0", 25 | "3 6 1.0", 26 | "6 3 1.0", 27 | "5 6 10.0", 28 | "6 5 10.0", 29 | "4 5 1.0", 30 | "5 4 1.0", 31 | "3 5 1.0", 32 | "5 3 1.0", 33 | "2 4 1.0", 34 | "4 2 1.0" 35 | }; 36 | 37 | // run to check results correctness 38 | GiraphConfiguration conf = new GiraphConfiguration(); 39 | conf.setMasterComputeClass(ScalableSemimetric.SemimetricMasterCompute.class); 40 | conf.setComputationClass(ScalableSemimetric.PropagateId.class); 41 | conf.setEdgeInputFormatClass(LongDoubleBooleanEdgeInputFormat.class); 42 | conf.setVertexOutputFormatClass(EdgesWithValuesVertexOutputFormat.class); 43 | conf.setOutEdgesClass(HashMapEdges.class); 44 | conf.setInt("semimetric.subsupersteps", 3); 45 | 46 | // run internally 47 | Iterable results = InternalVertexRunner.run(conf, null, graph); 48 | System.out.println("Testing two semimetric..."); 49 | for (String s: results) { 50 | String[] tokens = s.split("[\t ]"); 51 | Assert.assertEquals(false, ((Integer.parseInt(tokens[0]) == 1) && (Integer.parseInt(tokens[1]) == 3))); 52 | Assert.assertEquals(false, ((Integer.parseInt(tokens[0]) == 3) && (Integer.parseInt(tokens[1]) == 1))); 53 | Assert.assertEquals(false, ((Integer.parseInt(tokens[0]) == 5) && (Integer.parseInt(tokens[1]) == 6))); 54 | Assert.assertEquals(false, ((Integer.parseInt(tokens[0]) == 6) && (Integer.parseInt(tokens[1]) == 5))); 55 | Assert.assertEquals(1.0, Double.parseDouble(tokens[2])); 56 | System.out.println(s); 57 | } 58 | System.out.println(); 59 | } 60 | 61 | @Test 62 | public void testAllMetric() throws Exception { 63 | String[] graph = new String[] { 64 | "1 2 1.0", 65 | "2 1 1.0", 66 | "2 3 1.0", 67 | "3 2 1.0", 68 | "3 1 1.0", 69 | "1 3 1.0", 70 | "3 6 1.0", 71 | "6 3 1.0", 72 | "5 6 1.0", 73 | "6 5 1.0", 74 | "4 5 1.0", 75 | "5 4 1.0", 76 | "3 5 1.0", 77 | "5 3 1.0", 78 | "2 4 1.0", 79 | "4 2 1.0" 80 | }; 81 | 82 | // run to check results correctness 83 | GiraphConfiguration conf = new GiraphConfiguration(); 84 | conf.setMasterComputeClass(ScalableSemimetric.SemimetricMasterCompute.class); 85 | conf.setComputationClass(ScalableSemimetric.PropagateId.class); 86 | conf.setEdgeInputFormatClass(LongDoubleBooleanEdgeInputFormat.class); 87 | conf.setVertexOutputFormatClass(EdgesWithValuesVertexOutputFormat.class); 88 | conf.setOutEdgesClass(HashMapEdges.class); 89 | conf.setInt("semimetric.subsupersteps", 3); 90 | 91 | // run internally 92 | Iterable results = InternalVertexRunner.run(conf, null, graph); 93 | System.out.println("Testing all metric..."); 94 | ArrayList resultsList = new ArrayList(); 95 | for (String s: results) { 96 | resultsList.add(s); 97 | } 98 | Assert.assertEquals(16, resultsList.size()); 99 | } 100 | } -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/graphs/SemiClusteringTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.graphs; 17 | 18 | import static org.junit.Assert.*; 19 | 20 | import java.util.LinkedList; 21 | import java.util.List; 22 | 23 | import junit.framework.Assert; 24 | import ml.grafos.okapi.graphs.SemiClustering; 25 | import ml.grafos.okapi.io.formats.LongDoubleTextEdgeInputFormat; 26 | 27 | import org.apache.giraph.conf.GiraphConfiguration; 28 | import org.apache.giraph.io.formats.IdWithValueTextOutputFormat; 29 | import org.apache.giraph.utils.InternalVertexRunner; 30 | import org.junit.Test; 31 | 32 | public class SemiClusteringTest { 33 | 34 | @Test 35 | public void test() { 36 | String[] graph = { 37 | "1 2 1.0", 38 | "2 1 1.0", 39 | "1 3 1.0", 40 | "3 1 1.0", 41 | "2 3 2.0", 42 | "3 2 2.0", 43 | "3 4 2.0", 44 | "4 3 2.0", 45 | "3 5 1.0", 46 | "5 3 1.0", 47 | "4 5 1.0", 48 | "5 4 1.0" 49 | }; 50 | 51 | GiraphConfiguration conf = new GiraphConfiguration(); 52 | conf.setComputationClass(SemiClustering.class); 53 | conf.setEdgeInputFormatClass(LongDoubleTextEdgeInputFormat.class); 54 | conf.setInt(SemiClustering.ITERATIONS, 10); 55 | conf.setInt(SemiClustering.MAX_CLUSTERS, 2); 56 | conf.setInt(SemiClustering.CLUSTER_CAPACITY, 2); 57 | conf.setVertexOutputFormatClass(IdWithValueTextOutputFormat.class); 58 | Iterable results; 59 | try { 60 | results = InternalVertexRunner.run(conf, null, graph); 61 | } catch (Exception e) { 62 | e.printStackTrace(); 63 | fail("Exception occurred"); 64 | return; 65 | } 66 | List res = new LinkedList(); 67 | for (String string : results) { 68 | res.add(string); 69 | System.out.println(string); 70 | } 71 | Assert.assertEquals(5, res.size()); 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/graphs/SemimetricTrianglesTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2014 Grafos.ml 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package ml.grafos.okapi.graphs; 17 | 18 | import static org.junit.Assert.*; 19 | 20 | import java.util.LinkedList; 21 | import java.util.List; 22 | 23 | import ml.grafos.okapi.io.formats.LongDoubleTextEdgeInputFormat; 24 | import org.apache.giraph.conf.GiraphConfiguration; 25 | import org.apache.giraph.edge.HashMapEdges; 26 | import org.apache.giraph.io.formats.AdjacencyListTextVertexOutputFormat; 27 | import org.apache.giraph.utils.InternalVertexRunner; 28 | import org.junit.Test; 29 | 30 | public class SemimetricTrianglesTest { 31 | 32 | final double delta = 0.0001; 33 | 34 | @Test 35 | public void testSemimetricRemoval() { 36 | String[] graph = { 37 | "1 2 10.0", 38 | "1 4 1.0", 39 | "2 3 3.0", 40 | "2 4 2.0", 41 | "2 5 2.0", 42 | "3 5 1.0", 43 | "3 6 5.0", 44 | "5 6 3.0", 45 | "2 1 10.0", 46 | "4 1 1.0", 47 | "3 2 3.0", 48 | "4 2 2.0", 49 | "5 2 2.0", 50 | "5 3 1.0", 51 | "6 3 5.0", 52 | "6 5 3.0" 53 | }; 54 | 55 | GiraphConfiguration conf = new GiraphConfiguration(); 56 | conf.setComputationClass(SemimetricTriangles.PropagateId.class); 57 | conf.setMasterComputeClass(SemimetricTriangles.SemimetricMasterCompute.class); 58 | conf.setEdgeInputFormatClass(LongDoubleTextEdgeInputFormat.class); 59 | conf.setVertexOutputFormatClass(AdjacencyListTextVertexOutputFormat.class); 60 | conf.setOutEdgesClass(HashMapEdges.class); 61 | 62 | Iterable results; 63 | try { 64 | results = InternalVertexRunner.run(conf, null, graph); 65 | } catch (Exception e) { 66 | e.printStackTrace(); 67 | fail("Exception occurred"); 68 | return; 69 | } 70 | List res = new LinkedList(); 71 | for (String string : results) { 72 | res.add(string); 73 | System.out.println(string); 74 | 75 | String[] output = string.split("[\t ]"); 76 | if (Integer.parseInt(output[0]) == 1) { 77 | // check that semi-metric edge (1, 2) has been removed 78 | assertEquals(output.length, 4); 79 | assertEquals(4, Integer.parseInt(output[2])); 80 | } 81 | if (Integer.parseInt(output[0]) == 3) { 82 | // check that semi-metric edge (3, 6) has been removed 83 | assertEquals(output.length, 6); 84 | assertEquals(2, Integer.parseInt(output[2])); 85 | } 86 | } 87 | } 88 | } -------------------------------------------------------------------------------- /src/test/java/ml/grafos/okapi/graphs/maxbmatching/MaxBMatchingTest.java: -------------------------------------------------------------------------------- 1 | package ml.grafos.okapi.graphs.maxbmatching; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import java.util.Map; 6 | 7 | import ml.grafos.okapi.graphs.maxbmatching.MBMEdgeValue.State; 8 | 9 | import org.apache.giraph.conf.GiraphConfiguration; 10 | import org.apache.giraph.edge.HashMapEdges; 11 | import org.apache.giraph.utils.InternalVertexRunner; 12 | import org.apache.log4j.Logger; 13 | import org.junit.Test; 14 | import org.python.google.common.collect.Maps; 15 | 16 | import com.google.common.collect.HashBasedTable; 17 | import com.google.common.collect.Table; 18 | import com.google.common.primitives.Longs; 19 | 20 | public class MaxBMatchingTest { 21 | private static final Logger LOG = Logger.getLogger(MaxBMatchingTest.class); 22 | 23 | @Test 24 | public void test1() throws Exception { 25 | String[] graph = new String[] { 26 | "1\t1\t2\t3.0\t3\t1.0", 27 | "2\t2\t1\t3.0\t4\t1.0\t5\t1.0", 28 | "3\t1\t1\t1.0\t5\t3.0", 29 | "4\t1\t2\t1.0\t5\t2.0", 30 | "5\t3\t2\t1.0\t3\t3.0\t4\t2.0"}; 31 | 32 | /* output, removes 2 edges (1-3) (2-4) 33 | * 5 0 4 2.0 2 1.0 3 3.0 34 | * 2 0 1 3.0 5 1.0 35 | * 1 0 2 3.0 36 | * 3 0 5 3.0 37 | * 4 0 5 2.0 38 | */ 39 | 40 | // run to check results correctness 41 | GiraphConfiguration conf = new GiraphConfiguration(); 42 | conf.setComputationClass(MaxBMatching.class); 43 | conf.setOutEdgesClass(HashMapEdges.class); 44 | conf.setVertexInputFormatClass(MBMTextInputFormat.class); 45 | conf.setVertexOutputFormatClass(MBMTextOutputFormat.class); 46 | 47 | // run internally 48 | Iterable results = InternalVertexRunner.run(conf, graph); 49 | Table> subgraph = parseResults(results, conf); 50 | 51 | assertEquals(5, subgraph.size()); // five vertices 52 | 53 | // vertex 5 54 | assertEquals(1, subgraph.row(5L).size()); // only one capacity 55 | assertEquals(0, subgraph.row(5L).keySet().iterator().next().intValue()); // capacity is zero 56 | assertEquals(3, subgraph.get(5L, 0).size()); // three edges 57 | assertTrue(subgraph.get(5L, 0).keySet().containsAll(Longs.asList(4L, 2L, 3L))); // edges to these vertices 58 | 59 | // vertex 2 60 | assertEquals(0, subgraph.row(2L).keySet().iterator().next().intValue()); // capacity is zero 61 | assertTrue(subgraph.get(2L, 0).keySet().containsAll(Longs.asList(1L, 5L))); // edges to these vertices 62 | assertEquals(3.0, subgraph.get(2L, 0).get(1L).getWeight(), Double.MIN_VALUE); // edge weight to 1 is 3.0 63 | } 64 | 65 | private Table> parseResults(Iterable results, GiraphConfiguration conf) { 66 | Table> graph = HashBasedTable.create(); 67 | for (String result : results) { 68 | if (LOG.isDebugEnabled()) 69 | LOG.debug(result); 70 | final String delimiter = conf.get(MBMTextOutputFormat.LINE_TOKENIZE_VALUE, 71 | MBMTextOutputFormat.LINE_TOKENIZE_VALUE_DEFAULT); 72 | String[] parts = result.split(delimiter); 73 | long vertex = Long.parseLong(parts[0]); 74 | int capacity = Integer.parseInt(parts[1]); 75 | Map edges = Maps.newHashMap(); 76 | 77 | final int numEdgeComponents = 2; 78 | for (int i = 2; i < parts.length; i += numEdgeComponents) { 79 | long targetVID = Long.parseLong(parts[i]); 80 | double weight = Double.parseDouble(parts[i + 1]); // skip state 81 | MBMEdgeValue edgeValue = new MBMEdgeValue(weight, State.INCLUDED); 82 | edges.put(targetVID, edgeValue); 83 | } 84 | graph.put(vertex, capacity, edges); 85 | } 86 | return graph; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Set root logger level to DEBUG and its only appender to A1. 2 | log4j.rootLogger=ERROR, A1 3 | 4 | # A1 is set to be a ConsoleAppender. 5 | log4j.appender.A1=org.apache.log4j.ConsoleAppender 6 | 7 | # A1 uses PatternLayout. 8 | log4j.appender.A1.layout=org.apache.log4j.PatternLayout 9 | log4j.appender.A1.layout.ConversionPattern=%c %x - %m%n 10 | log4j.logger.org.apache=ERROR 11 | log4j.logger.ml.grafos=DEBUG --------------------------------------------------------------------------------