├── run-dataset-20101205T014106 ├── 9-2.log ├── 9-3.log ├── 9-1.log ├── 0-1.log ├── 0-2.log ├── 0-3.log ├── 1-1.log ├── 1-2.log ├── 1-3.log ├── 2-1.log ├── 2-2.log ├── 2-3.log ├── 3-1.log ├── 3-2.log └── 3-3.log ├── run-scaleup-20101206T184139 ├── 3-2.log ├── 3-1.log ├── 1-1.log ├── 1-2.log └── 1-3.log ├── .hgignore ├── benchmark-ec2.xlsx ├── report-fall2010.docx ├── report-fall2010.pptx ├── benchmark-r-cluster.xlsx ├── run-scaleout-20101205T231425 ├── 4-1.log ├── 4-2.log ├── 4-3.log ├── 5-1.log ├── 5-2.log ├── 5-3.log ├── quota-error.txt ├── 2-1.log ├── 2-2.log └── 2-3.log ├── plot-points.plt ├── common-options.sh ├── KMeansHelper.scala ├── Point.scala ├── KMeansDataGenerator.scala ├── perf-scaleout ├── perf-dataset ├── Makefile ├── perf-scaleup ├── perf-cachesize ├── run ├── README ├── LogAnalyzer.scala ├── init-cluster.sh ├── LocalKMeans.scala ├── run-dataset-20101204T225747 ├── 0-1.log ├── 0-2.log ├── 0-3.log ├── 1-1.log ├── 1-2.log ├── 1-3.log ├── 2-1.log ├── 2-2.log ├── 2-3.log ├── 3-2.log ├── 3-3.log └── 3-1.log └── SparkKMeans.scala /run-dataset-20101205T014106/9-2.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /run-dataset-20101205T014106/9-3.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /run-scaleup-20101206T184139/3-2.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.hgignore: -------------------------------------------------------------------------------- 1 | syntax: glob 2 | 3 | build 4 | *~ 5 | *.class -------------------------------------------------------------------------------- /benchmark-ec2.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurdave/kmeans-spark/HEAD/benchmark-ec2.xlsx -------------------------------------------------------------------------------- /report-fall2010.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurdave/kmeans-spark/HEAD/report-fall2010.docx -------------------------------------------------------------------------------- /report-fall2010.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurdave/kmeans-spark/HEAD/report-fall2010.pptx -------------------------------------------------------------------------------- /benchmark-r-cluster.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ankurdave/kmeans-spark/HEAD/benchmark-r-cluster.xlsx -------------------------------------------------------------------------------- /run-scaleout-20101205T231425/4-1.log: -------------------------------------------------------------------------------- 1 | ssh: Could not resolve hostname ERROR:: Name or service not known 2 | -------------------------------------------------------------------------------- /run-scaleout-20101205T231425/4-2.log: -------------------------------------------------------------------------------- 1 | ssh: Could not resolve hostname ERROR:: Name or service not known 2 | -------------------------------------------------------------------------------- /run-scaleout-20101205T231425/4-3.log: -------------------------------------------------------------------------------- 1 | ssh: Could not resolve hostname ERROR:: Name or service not known 2 | -------------------------------------------------------------------------------- /run-scaleout-20101205T231425/5-1.log: -------------------------------------------------------------------------------- 1 | ssh: Could not resolve hostname ERROR:: Name or service not known 2 | -------------------------------------------------------------------------------- /run-scaleout-20101205T231425/5-2.log: -------------------------------------------------------------------------------- 1 | ssh: Could not resolve hostname ERROR:: Name or service not known 2 | -------------------------------------------------------------------------------- /run-scaleout-20101205T231425/5-3.log: -------------------------------------------------------------------------------- 1 | ssh: Could not resolve hostname ERROR:: Name or service not known 2 | -------------------------------------------------------------------------------- /plot-points.plt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/gnuplot -persist 2 | 3 | plot "points.dat" using 1:2 with points, \ 4 | "output.dat" using 1:2 with points -------------------------------------------------------------------------------- /common-options.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DATE=$(date +%Y%m%dT%H%M%S) 4 | TRIALS=3 5 | KEYPAIR=Mesos 6 | KEYPAIRFILE=/work/ankurd/keys/Mesos.pem 7 | AWSKEYFILE=/work/ankurd/keys/env.sh 8 | ZONE=us-east-1d 9 | CLUSTERNAME=${PERF_CLUSTERNAME:-KMeans} 10 | -------------------------------------------------------------------------------- /run-dataset-20101205T014106/9-1.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1205 05:16:25.177633 6076 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/05 05:16:25 INFO MesosScheduler: Registered as framework ID 201012050516-0-0053 4 | -------------------------------------------------------------------------------- /KMeansHelper.scala: -------------------------------------------------------------------------------- 1 | object KMeansHelper { 2 | /** 3 | * Finds the closest centroid to the given point. 4 | */ 5 | def closestCentroid(centroids: Seq[Point], point: Point) = { 6 | centroids.reduceLeft((a, b) => if ((point distance a) < (point distance b)) a else b) 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /Point.scala: -------------------------------------------------------------------------------- 1 | @serializable case class Point(val x: Double, val y: Double) { 2 | def + (that: Point) = new Point(this.x + that.x, this.y + that.y) 3 | def - (that: Point) = this + (-that) 4 | def unary_- () = new Point(-this.x, -this.y) 5 | def / (d: Double) = new Point(this.x / d, this.y / d) 6 | def magnitude = math.sqrt(x * x + y * y) 7 | def distance(that: Point) = (that - this).magnitude 8 | override def toString = format("(%.2f,%.2f)", x, y) 9 | } 10 | 11 | object Point { 12 | def random() = { 13 | new Point( 14 | math.random * 50, 15 | math.random * 50) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /KMeansDataGenerator.scala: -------------------------------------------------------------------------------- 1 | import scala.util.Random 2 | 3 | object KMeansDataGenerator { 4 | def main(args: Array[String]) { 5 | if (args.length < 3) { 6 | System.err.println("Usage: KMeansDataGenerator ") 7 | System.exit(-1) 8 | } 9 | 10 | val numClusters = args(0).toInt 11 | val desiredCentroids = Array.fill(numClusters) { Point.random } 12 | desiredCentroids.foreach(p => printf("#%f\t%f\n", p.x, p.y)) 13 | 14 | val numPoints = args(1).toInt 15 | val spread = args(2).toDouble 16 | val random = new Random() 17 | for (i <- 1 to numPoints) { 18 | val point = desiredCentroids(random.nextInt(desiredCentroids.length)) + 19 | new Point(random.nextGaussian() * spread, random.nextGaussian() * spread) 20 | printf("%f\t%f\n", point.x, point.y) 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /perf-scaleout: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source common-options.sh 4 | 5 | MANIPULATIONS=5 6 | LOGDIR=run-scaleout-$DATE 7 | INSTANCETYPE=m1.large 8 | MEMORY=6500m 9 | NUMPOINTS=100000000 10 | 11 | echo "Starting experiment..." 12 | for manip in $(seq 1 $MANIPULATIONS) 13 | do 14 | let NUMSLAVES=2**$manip 15 | echo "Number of slaves: $NUMSLAVES" 16 | 17 | source init-cluster.sh 18 | 19 | echo "Starting performance run..." 20 | for trial in $(seq 1 $TRIALS) 21 | do 22 | echo "Trial $trial..." 23 | ssh -i $KEYPAIRFILE root@$MASTER \ 24 | "source ~/.profile; \ 25 | cd kmeans-spark; \ 26 | ./run SparkKMeans $POINTFILE 3 $MESOSMASTER 7,40 45,35 20,8" \ 27 | 2>&1 | tee $LOGDIR/$manip-$trial.log 28 | done 29 | 30 | echo "Ending performance run..." 31 | echo y | mesos-ec2 --key-pair=$KEYPAIR --identity-file=$KEYPAIRFILE destroy $CLUSTERNAME 32 | done 33 | -------------------------------------------------------------------------------- /perf-dataset: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source common-options.sh 4 | 5 | MANIPULATIONS=5 6 | LOGDIR=run-dataset-$DATE 7 | INSTANCETYPE=m1.large 8 | MEMORY=6g 9 | NUMSLAVES=2 10 | BASENUMPOINTS=10000000 11 | NUMPOINTS=$BASENUMPOINTS 12 | 13 | source init-cluster.sh 14 | 15 | echo "Starting experiment..." 16 | for manip in $(seq 0 $(expr $MANIPULATIONS - 1)) 17 | do 18 | export manip 19 | POINTFILEREPEATED=$(perl -e 'print join(",", ($ENV{"POINTFILE"}) x (2 ** $ENV{"manip"}))') 20 | 21 | for trial in $(seq 1 $TRIALS) 22 | do 23 | echo "Trial $trial..." 24 | ssh -i $KEYPAIRFILE root@$MASTER \ 25 | "source ~/.profile; \ 26 | cd kmeans-spark; \ 27 | ./run SparkKMeans $POINTFILEREPEATED 3 $MESOSMASTER 7,40 45,35 20,8" \ 28 | 2>&1 | tee $LOGDIR/$manip-$trial.log 29 | done 30 | done 31 | 32 | echo "Ending experiment..." 33 | echo y | mesos-ec2 --key-pair=$KEYPAIR --identity-file=$KEYPAIRFILE destroy $CLUSTERNAME 34 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | export SPARK_HOME=/root/spark 2 | 3 | SCALA_SOURCES = *.scala 4 | CLASSPATH = build/spark.jar:build/spark-dep.jar 5 | 6 | ifeq ($(USE_FSC),1) 7 | COMPILER_NAME = fsc 8 | else 9 | COMPILER_NAME = scalac 10 | endif 11 | 12 | ifeq ($(SCALA_HOME),) 13 | COMPILER = $(COMPILER_NAME) 14 | else 15 | COMPILER = $(SCALA_HOME)/bin/$(COMPILER_NAME) 16 | endif 17 | 18 | 19 | all: jar 20 | 21 | jar: build/kmeans-spark.jar 22 | 23 | build/kmeans-spark.jar: build/spark.jar $(SCALA_SOURCES) 24 | mkdir -p build/classes 25 | $(COMPILER) -d build/classes -classpath build/classes:$(CLASSPATH) $(SCALA_SOURCES) 26 | jar cf build/kmeans-spark.jar -C build/classes . 27 | 28 | build/spark.jar: 29 | make -C $(SPARK_HOME) jar 30 | mkdir -p build 31 | cp $(SPARK_HOME)/build/spark.jar $(SPARK_HOME)/build/spark-dep.jar build/ 32 | 33 | 34 | default: all 35 | 36 | clean: 37 | rm -rf build 38 | 39 | .phony: default all clean jar 40 | -------------------------------------------------------------------------------- /perf-scaleup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source common-options.sh 4 | 5 | MAXINSTANCETYPEINDEX=3 6 | INSTANCETYPES[0]='m1.large' 7 | INSTANCETYPES[1]='m1.xlarge' 8 | INSTANCETYPES[2]='m2.xlarge' 9 | INSTANCETYPES[3]='c1.xlarge' 10 | MEMORYS[0]='6500m' 11 | MEMORYS[1]='14g' 12 | MEMORYS[2]='16g' 13 | MEMORYS[3]='6g' 14 | LOGDIR=run-scaleup-$DATE 15 | NUMPOINTS=100000000 16 | NUMSLAVES=2 17 | 18 | echo "Starting experiment..." 19 | for manip in $(seq 0 $MAXINSTANCETYPEINDEX) 20 | do 21 | INSTANCETYPE=${INSTANCETYPES[$manip]} 22 | MEMORY=${MEMORYS[$manip]} 23 | echo "Using instance type $INSTANCETYPE with $MEMORY memory" 24 | 25 | source init-cluster.sh 26 | 27 | echo "Starting performance run..." 28 | for trial in $(seq 1 $TRIALS) 29 | do 30 | echo "Trial $trial..." 31 | ssh -i $KEYPAIRFILE root@$MASTER \ 32 | "source ~/.profile; \ 33 | cd kmeans-spark; \ 34 | ./run SparkKMeans $POINTFILE 3 $MESOSMASTER 7,40 45,35 20,8" \ 35 | 2>&1 | tee $LOGDIR/$manip-$trial.log 36 | done 37 | 38 | echo "Ending performance run..." 39 | echo y | mesos-ec2 --key-pair=$KEYPAIR --identity-file=$KEYPAIRFILE destroy $CLUSTERNAME 40 | done 41 | -------------------------------------------------------------------------------- /perf-cachesize: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source common-options.sh 4 | 5 | MAXCACHETYPEINDEX=2 6 | CACHETYPES[0]='BoundedMemoryCache' 7 | CACHETYPES[1]='WeakReferenceCache' 8 | CACHETYPES[2]='SoftReferenceCache' 9 | LOGDIR=run-cachesize-$DATE 10 | INSTANCETYPE=m1.large 11 | MEMORY=6500m 12 | NUMPOINTS=100000000 13 | NUMSLAVES=2 14 | 15 | source init-cluster.sh 16 | 17 | echo "Starting experiment..." 18 | for manip in $(seq 0 $MAXCACHETYPEINDEX) 19 | do 20 | CACHETYPE=${CACHETYPES[$manip]} 21 | echo "Using cache type $CACHETYPE" 22 | ssh -i $KEYPAIRFILE root@$MASTER \ 23 | "echo 'export SPARK_JAVA_OPTS=\"-Dspark.cache.class=spark.$CACHETYPE\"' >> /root/spark/conf/spark-env.sh" 24 | 25 | echo "Starting performance run..." 26 | for trial in $(seq 1 $TRIALS) 27 | do 28 | echo "Trial $trial..." 29 | ssh -i $KEYPAIRFILE root@$MASTER \ 30 | "source ~/.profile; \ 31 | cd kmeans-spark; \ 32 | ./run SparkKMeans $POINTFILE 3 $MESOSMASTER 7,40 45,35 20,8" 33 | 2>&1 | tee $LOGDIR/$manip-$trial.log 34 | done 35 | 36 | echo "Ending performance run..." 37 | echo y | mesos-ec2 --key-pair=$KEYPAIR --identity-file=$KEYPAIRFILE destroy $CLUSTERNAME 38 | done 39 | -------------------------------------------------------------------------------- /run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | FWDIR=/root/spark 4 | 5 | # Export this as SPARK_HOME 6 | export SPARK_HOME="$FWDIR" 7 | 8 | # Load environment variables from conf/spark-env.sh, if it exists 9 | if [ -e $FWDIR/conf/spark-env.sh ] ; then 10 | . $FWDIR/conf/spark-env.sh 11 | fi 12 | 13 | MESOS_CLASSPATH="" 14 | MESOS_LIBRARY_PATH="" 15 | 16 | if [ "x$MESOS_HOME" != "x" ] ; then 17 | MESOS_CLASSPATH="$MESOS_HOME/lib/java/mesos.jar" 18 | MESOS_LIBRARY_PATH="$MESOS_HOME/lib/java" 19 | fi 20 | 21 | if [ "x$SPARK_MEM" == "x" ] ; then 22 | SPARK_MEM="300m" 23 | fi 24 | 25 | # Set JAVA_OPTS to be able to load native libraries and to set heap size 26 | JAVA_OPTS="$SPARK_JAVA_OPTS" 27 | JAVA_OPTS+=" -Djava.library.path=$SPARK_LIBRARY_PATH:$FWDIR/third_party:$FWDIR/src/native:$MESOS_LIBRARY_PATH" 28 | JAVA_OPTS+=" -Xms$SPARK_MEM -Xmx$SPARK_MEM" 29 | # Load extra JAVA_OPTS from conf/java-opts, if it exists 30 | if [ -e $FWDIR/conf/java-opts ] ; then 31 | JAVA_OPTS+=" `cat $FWDIR/conf/java-opts`" 32 | fi 33 | export JAVA_OPTS 34 | 35 | # Build up classpath 36 | CLASSPATH="build/kmeans-spark.jar:build/spark.jar:build/spark-dep.jar" 37 | export CLASSPATH # Needed for spark-shell 38 | 39 | if [ -n "$SCALA_HOME" ]; then 40 | SCALA=${SCALA_HOME}/bin/scala 41 | else 42 | SCALA=scala 43 | fi 44 | 45 | exec $SCALA -cp $CLASSPATH $@ 46 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | kmeans-spark - a simple implementation of k-means clustering on the Spark cluster computing framework 2 | 3 | COMPILING 4 | ========= 5 | 6 | Edit the Makefile and run to set SPARK_HOME and FWDIR appropriately. Then run 7 | make 8 | 9 | This builds Spark along with kmeans-spark in a pretty hacky fashion. At some point it would be good to move to sbt. 10 | 11 | RUNNING ON SPARK 12 | ================ 13 | 14 | Use the following command: 15 | ./run SparkKMeans [,, ...] 16 | 17 | pointsFile - the location of the input dataset. Any path that SparkContext.textFile understands is fine, including hdfs paths. This file should be formatted with one 2D point per line using the following format string: "%f\t%f\n".format(point.x, point.y). Lines beginning with # will be ignored. 18 | 19 | numClusters - an integer representing the number of clusters to use (the value of k). 20 | 21 | host - the Mesos host to run on, or "local" to run locally. 22 | startCentroid1{X,Y} - optionally, the initial centroid values. 23 | 24 | GENERATING POINTS 25 | ================= 26 | 27 | To generate random points distributed randomly in clusters, use the following command: 28 | ./run KMeansDataGenerator 29 | This will generate the points data and send it to standard output. It will also print the generated centroids at the beginning of the file, preceded by # so they are ignored by SparkKMeans. 30 | 31 | numClusters - the number of clusters to generate. 32 | 33 | numPoints - the number of random points to generate. Each point is generated close to the centroid of a random cluster, randomly perturbed according to a Gaussian distribution. 34 | 35 | pointSpread - the Gaussian spread of points around centroids. -------------------------------------------------------------------------------- /LogAnalyzer.scala: -------------------------------------------------------------------------------- 1 | import java.io.File 2 | import scala.io.Source 3 | import scala.util.matching.Regex 4 | import scala.collection.immutable.SortedMap 5 | 6 | object LogAnalyzer { 7 | def main(args: Array[String]) { 8 | if (args.length < 1) { 9 | System.err.println("Usage: LogAnalyzer ") 10 | System.exit(-1) 11 | } 12 | 13 | val logdir = args(0) 14 | 15 | val LogFileName = """(\d+)-(\d+)\.log""".r 16 | val TaskTimeEntry = """.*Tasks finished in (\d+.\d+) s""".r 17 | 18 | val times = 19 | for { 20 | logfile <- new File(logdir).listFiles 21 | if !logfile.isDirectory 22 | if (logfile.getName match { case LogFileName(_, _) => true; case _ => false }) 23 | LogFileName(manip, trial) = logfile.getName 24 | src = Source.fromFile(logfile) 25 | line <- src.getLines.filter(_.matches(TaskTimeEntry.toString)).drop(1).toTraversable 26 | TaskTimeEntry(timeStr) = line 27 | } yield (manip.toInt, trial.toInt) -> timeStr.toDouble 28 | 29 | val timesCollected = 30 | mergePairsIntoMap((a: List[Double], b: List[Double]) => a ::: b, times.map({case ((manip, trial), time) => manip -> List(time)})) 31 | 32 | for ((manip, times) <- SortedMap[Int, List[Double]]() ++ timesCollected) { 33 | println("%d, %s".format(manip, times.mkString(", "))) 34 | } 35 | } 36 | 37 | /** 38 | * Merges a list of Maps into a single Map, using the mergeValues function to resolve key conflicts. 39 | */ 40 | def mergePairsIntoMap[A, B](mergeValues: (B, B) => B, pairs: Seq[Pair[A, B]]): Map[A, B] = { 41 | pairs.foldLeft(Map[A, B]()) { 42 | (soFar, pair) => pair match { 43 | case (key, value) => { 44 | soFar + ( 45 | if (soFar.contains(key)) 46 | key -> mergeValues(soFar(key), value) 47 | else 48 | key -> value 49 | ) 50 | } 51 | } 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /init-cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mkdir -p $LOGDIR 4 | 5 | source $AWSKEYFILE 6 | 7 | echo "Launching cluster..." 8 | mesos-ec2 \ 9 | --slaves=$NUMSLAVES \ 10 | --key-pair=$KEYPAIR \ 11 | --identity-file=$KEYPAIRFILE \ 12 | --instance-type=$INSTANCETYPE \ 13 | --zone=$ZONE launch $CLUSTERNAME 14 | 15 | MASTER=$(mesos-ec2 -k $KEYPAIR -i $KEYPAIRFILE get-master $CLUSTERNAME | tail -n 1) 16 | MESOSMASTER=$(curl http://$MASTER:8080 2>/dev/null | perl -ne 'print $1 if /^PID: (.*)
/') 17 | echo "Master public hostname: $MASTER" 18 | echo "Master Mesos PID: $MESOSMASTER" 19 | 20 | echo "Configuring cluster: Updating Spark from Git..." 21 | ssh -i $KEYPAIRFILE root@$MASTER \ 22 | "cd /root/spark; \ 23 | git pull git://github.com/mesos/spark.git master; \ 24 | make clean" 25 | echo \ 26 | "export MESOS_HOME=/root/mesos 27 | export SCALA_HOME=/root/scala-2.8.0.final 28 | export SPARK_CLASSPATH=/root/kmeans-spark/build/kmeans-spark.jar 29 | export SPARK_MEM=$MEMORY 30 | export SPARK_JAVA_OPTS=\"-Dspark.cache.class=spark.BoundedMemoryCache\"" > /tmp/$CLUSTERNAME-spark-env.sh 31 | scp -i $KEYPAIRFILE /tmp/$CLUSTERNAME-spark-env.sh root@$MASTER:/root/spark/conf/spark-env.sh 32 | 33 | echo "Configuring cluster: Setting up kmeans-spark..." 34 | ssh -i $KEYPAIRFILE root@$MASTER \ 35 | "apt-get -y install mercurial; \ 36 | cd /root; \ 37 | hg clone http://ankurdave.com/hg/kmeans-spark; \ 38 | cd kmeans-spark; \ 39 | source ~/.profile; \ 40 | make" 41 | 42 | echo "Configuring cluster: Copying modified files to slaves..." 43 | ssh -i $KEYPAIRFILE root@$MASTER \ 44 | "cd /root; \ 45 | mesos-ec2/copy-dir spark; \ 46 | mesos-ec2/copy-dir kmeans-spark" 47 | 48 | echo "Generating point file..." 49 | ssh -i $KEYPAIRFILE root@$MASTER \ 50 | "source ~/.profile; \ 51 | cd kmeans-spark; \ 52 | ./run KMeansDataGenerator 3 $NUMPOINTS 1 > pts.dat; \ 53 | ../hadoop-0.20.2/bin/hadoop fs -put pts.dat /" 54 | export POINTFILE=hdfs://$MASTER:9000/pts.dat 55 | -------------------------------------------------------------------------------- /LocalKMeans.scala: -------------------------------------------------------------------------------- 1 | import scala.io.Source 2 | 3 | object LocalKMeans { 4 | def main(args: Array[String]) { 5 | if (args.length < 2) { 6 | System.err.println("Usage: LocalKMeans ") 7 | System.exit(-1) 8 | } 9 | 10 | // Parse the points from a file into an array 11 | val points = Source.fromFile(args(0)).getLines.toSeq.filter(line => !line.matches("^\\s*#.*")).map( 12 | line => { 13 | val parts = line.split("\t").map(_.toDouble) 14 | new Point(parts(0), parts(1)) 15 | } 16 | ).toArray 17 | System.err.println("Read " + points.length + " points.") 18 | 19 | // Initialize k random centroids 20 | val centroids = Array.fill(args(1).toInt) { Point.random } 21 | 22 | // Start the local run 23 | val resultCentroids = kmeans(points, centroids, 0.1) 24 | 25 | System.err.println("Final centroids: ") 26 | println(resultCentroids.map(centroid => "%3f\t%3f\n".format(centroid.x, centroid.y)).mkString) 27 | } 28 | 29 | def kmeans(points: Seq[Point], centroids: Seq[Point], epsilon: Double): Seq[Point] = { 30 | // Assign points to centroids 31 | val clusters = points.groupBy(KMeansHelper.closestCentroid(centroids, _)) 32 | 33 | // Recalculate centroids as the average of the points in their cluster 34 | // (or leave them alone if they don't have any points in their cluster) 35 | val newCentroids = centroids.map(oldCentroid => { 36 | clusters.get(oldCentroid) match { 37 | case Some(pointsInCluster) => pointsInCluster.reduceLeft(_ + _) / pointsInCluster.length 38 | case None => oldCentroid 39 | }}) 40 | 41 | // Calculate the centroid movement for the stopping condition 42 | val movement = (centroids zip newCentroids).map({ case (a, b) => a distance b }) 43 | 44 | System.err.println("Centroids changed by\n" + 45 | "\t " + movement.map(d => "%3f".format(d)).mkString("(", ", ", ")") + "\n" + 46 | "\tto " + newCentroids.mkString("(", ", ", ")")) 47 | 48 | // Iterate if movement exceeds threshold 49 | if (movement.exists(_ > epsilon)) 50 | kmeans(points, newCentroids, epsilon) 51 | else 52 | return newCentroids 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /run-dataset-20101204T225747/0-1.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1204 22:57:48.434339 3762 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/04 22:57:48 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 4 | 10/12/04 22:57:48 INFO MesosScheduler: Registered as framework ID 201012042257-0-0011 5 | 10/12/04 22:57:49 INFO FileInputFormat: Total input paths to process : 1 6 | 10/12/04 22:57:49 INFO SparkContext: Running 2 tasks in parallel 7 | 10/12/04 22:57:49 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/04 22:57:49 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/04 22:58:11 INFO SimpleJob: Finished TID 1 (progress: 1/2) 10 | 10/12/04 22:58:14 INFO SimpleJob: Finished TID 0 (progress: 2/2) 11 | 10/12/04 22:58:14 INFO SparkContext: Tasks finished in 25.288100098 s 12 | Read 10000000 points. 13 | 10/12/04 22:58:14 INFO SparkContext: Running 2 tasks in parallel 14 | 10/12/04 22:58:14 INFO SimpleJob: Starting task 1:0 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 15 | 10/12/04 22:58:14 INFO SimpleJob: Starting task 1:1 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 16 | 10/12/04 22:58:18 INFO SimpleJob: Finished TID 3 (progress: 1/2) 17 | 10/12/04 22:58:19 INFO SimpleJob: Finished TID 2 (progress: 2/2) 18 | 10/12/04 22:58:19 INFO SparkContext: Tasks finished in 5.443556283 s 19 | Centroids changed by 20 | (1.020119, 0.433392, 2.183411) 21 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 22 | 10/12/04 22:58:19 INFO SparkContext: Running 2 tasks in parallel 23 | 10/12/04 22:58:19 INFO SimpleJob: Starting task 2:0 as TID 4 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 24 | 10/12/04 22:58:19 INFO SimpleJob: Starting task 2:1 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 25 | 10/12/04 22:58:23 INFO SimpleJob: Finished TID 5 (progress: 1/2) 26 | 10/12/04 22:58:23 INFO SimpleJob: Finished TID 4 (progress: 2/2) 27 | 10/12/04 22:58:23 INFO SparkContext: Tasks finished in 4.026354641 s 28 | Centroids changed by 29 | (0.000000, 0.000000, 0.000000) 30 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 31 | Final centroids: 32 | 7.776982 40.661016 33 | 44.751415 35.355014 34 | 17.872132 7.510652 35 | 36 | -------------------------------------------------------------------------------- /run-dataset-20101204T225747/0-2.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1204 22:58:25.451781 3803 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/04 22:58:25 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 4 | 10/12/04 22:58:25 INFO MesosScheduler: Registered as framework ID 201012042258-0-0012 5 | 10/12/04 22:58:25 INFO FileInputFormat: Total input paths to process : 1 6 | 10/12/04 22:58:26 INFO SparkContext: Running 2 tasks in parallel 7 | 10/12/04 22:58:26 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/04 22:58:26 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/04 22:58:47 INFO SimpleJob: Finished TID 1 (progress: 1/2) 10 | 10/12/04 22:58:51 INFO SimpleJob: Finished TID 0 (progress: 2/2) 11 | 10/12/04 22:58:51 INFO SparkContext: Tasks finished in 25.198913577 s 12 | Read 10000000 points. 13 | 10/12/04 22:58:51 INFO SparkContext: Running 2 tasks in parallel 14 | 10/12/04 22:58:51 INFO SimpleJob: Starting task 1:0 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 15 | 10/12/04 22:58:51 INFO SimpleJob: Starting task 1:1 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 16 | 10/12/04 22:58:56 INFO SimpleJob: Finished TID 3 (progress: 1/2) 17 | 10/12/04 22:58:56 INFO SimpleJob: Finished TID 2 (progress: 2/2) 18 | 10/12/04 22:58:56 INFO SparkContext: Tasks finished in 5.61359425 s 19 | Centroids changed by 20 | (1.020119, 0.433392, 2.183411) 21 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 22 | 10/12/04 22:58:56 INFO SparkContext: Running 2 tasks in parallel 23 | 10/12/04 22:58:56 INFO SimpleJob: Starting task 2:0 as TID 4 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 24 | 10/12/04 22:58:56 INFO SimpleJob: Starting task 2:1 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 25 | 10/12/04 22:59:00 INFO SimpleJob: Finished TID 5 (progress: 1/2) 26 | 10/12/04 22:59:01 INFO SimpleJob: Finished TID 4 (progress: 2/2) 27 | 10/12/04 22:59:01 INFO SparkContext: Tasks finished in 4.142518959 s 28 | Centroids changed by 29 | (0.000000, 0.000000, 0.000000) 30 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 31 | Final centroids: 32 | 7.776982 40.661016 33 | 44.751415 35.355014 34 | 17.872132 7.510652 35 | 36 | -------------------------------------------------------------------------------- /run-dataset-20101204T225747/0-3.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1204 22:59:02.725721 3844 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/04 22:59:02 INFO MesosScheduler: Registered as framework ID 201012042259-0-0013 4 | 10/12/04 22:59:02 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/04 22:59:03 INFO FileInputFormat: Total input paths to process : 1 6 | 10/12/04 22:59:03 INFO SparkContext: Running 2 tasks in parallel 7 | 10/12/04 22:59:03 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/04 22:59:03 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/04 22:59:24 INFO SimpleJob: Finished TID 1 (progress: 1/2) 10 | 10/12/04 22:59:28 INFO SimpleJob: Finished TID 0 (progress: 2/2) 11 | 10/12/04 22:59:28 INFO SparkContext: Tasks finished in 25.052851999 s 12 | Read 10000000 points. 13 | 10/12/04 22:59:28 INFO SparkContext: Running 2 tasks in parallel 14 | 10/12/04 22:59:28 INFO SimpleJob: Starting task 1:0 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 15 | 10/12/04 22:59:28 INFO SimpleJob: Starting task 1:1 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 16 | 10/12/04 22:59:33 INFO SimpleJob: Finished TID 3 (progress: 1/2) 17 | 10/12/04 22:59:33 INFO SimpleJob: Finished TID 2 (progress: 2/2) 18 | 10/12/04 22:59:33 INFO SparkContext: Tasks finished in 5.313238789 s 19 | Centroids changed by 20 | (1.020119, 0.433392, 2.183411) 21 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 22 | 10/12/04 22:59:33 INFO SparkContext: Running 2 tasks in parallel 23 | 10/12/04 22:59:33 INFO SimpleJob: Starting task 2:0 as TID 4 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 24 | 10/12/04 22:59:33 INFO SimpleJob: Starting task 2:1 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 25 | 10/12/04 22:59:37 INFO SimpleJob: Finished TID 5 (progress: 1/2) 26 | 10/12/04 22:59:37 INFO SimpleJob: Finished TID 4 (progress: 2/2) 27 | 10/12/04 22:59:37 INFO SparkContext: Tasks finished in 3.911241284 s 28 | Centroids changed by 29 | (0.000000, 0.000000, 0.000000) 30 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 31 | Final centroids: 32 | 7.776982 40.661016 33 | 44.751415 35.355014 34 | 17.872132 7.510652 35 | 36 | -------------------------------------------------------------------------------- /run-dataset-20101205T014106/0-1.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1205 01:41:07.899662 4716 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/05 01:41:07 INFO MesosScheduler: Registered as framework ID 201012050141-0-0026 4 | 10/12/05 01:41:07 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/05 01:41:08 INFO FileInputFormat: Total input paths to process : 1 6 | 10/12/05 01:41:08 INFO SparkContext: Running 2 tasks in parallel 7 | 10/12/05 01:41:08 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/05 01:41:08 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/05 01:41:30 INFO SimpleJob: Finished TID 1 (progress: 1/2) 10 | 10/12/05 01:41:34 INFO SimpleJob: Finished TID 0 (progress: 2/2) 11 | 10/12/05 01:41:34 INFO SparkContext: Tasks finished in 26.458131471 s 12 | Read 10000000 points. 13 | 10/12/05 01:41:34 INFO SparkContext: Running 2 tasks in parallel 14 | 10/12/05 01:41:34 INFO SimpleJob: Starting task 1:0 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 15 | 10/12/05 01:41:34 INFO SimpleJob: Starting task 1:1 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 16 | 10/12/05 01:41:39 INFO SimpleJob: Finished TID 3 (progress: 1/2) 17 | 10/12/05 01:41:40 INFO SimpleJob: Finished TID 2 (progress: 2/2) 18 | 10/12/05 01:41:40 INFO SparkContext: Tasks finished in 5.437511996 s 19 | Centroids changed by 20 | (1.020119, 0.433392, 2.183411) 21 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 22 | 10/12/05 01:41:40 INFO SparkContext: Running 2 tasks in parallel 23 | 10/12/05 01:41:40 INFO SimpleJob: Starting task 2:0 as TID 4 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 24 | 10/12/05 01:41:40 INFO SimpleJob: Starting task 2:1 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 25 | 10/12/05 01:41:43 INFO SimpleJob: Finished TID 5 (progress: 1/2) 26 | 10/12/05 01:41:44 INFO SimpleJob: Finished TID 4 (progress: 2/2) 27 | 10/12/05 01:41:44 INFO SparkContext: Tasks finished in 3.884024155 s 28 | Centroids changed by 29 | (0.000000, 0.000000, 0.000000) 30 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 31 | Final centroids: 32 | 7.776982 40.661016 33 | 44.751415 35.355014 34 | 17.872132 7.510652 35 | 36 | -------------------------------------------------------------------------------- /run-dataset-20101205T014106/0-2.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1205 01:41:45.929488 4758 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/05 01:41:45 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 4 | 10/12/05 01:41:45 INFO MesosScheduler: Registered as framework ID 201012050141-0-0027 5 | 10/12/05 01:41:46 INFO FileInputFormat: Total input paths to process : 1 6 | 10/12/05 01:41:46 INFO SparkContext: Running 2 tasks in parallel 7 | 10/12/05 01:41:46 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/05 01:41:46 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/05 01:42:07 INFO SimpleJob: Finished TID 1 (progress: 1/2) 10 | 10/12/05 01:42:12 INFO SimpleJob: Finished TID 0 (progress: 2/2) 11 | 10/12/05 01:42:12 INFO SparkContext: Tasks finished in 26.221168028 s 12 | Read 10000000 points. 13 | 10/12/05 01:42:12 INFO SparkContext: Running 2 tasks in parallel 14 | 10/12/05 01:42:12 INFO SimpleJob: Starting task 1:0 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 15 | 10/12/05 01:42:12 INFO SimpleJob: Starting task 1:1 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 16 | 10/12/05 01:42:17 INFO SimpleJob: Finished TID 3 (progress: 1/2) 17 | 10/12/05 01:42:18 INFO SimpleJob: Finished TID 2 (progress: 2/2) 18 | 10/12/05 01:42:18 INFO SparkContext: Tasks finished in 5.490562012 s 19 | Centroids changed by 20 | (1.020119, 0.433392, 2.183411) 21 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 22 | 10/12/05 01:42:18 INFO SparkContext: Running 2 tasks in parallel 23 | 10/12/05 01:42:18 INFO SimpleJob: Starting task 2:0 as TID 4 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 24 | 10/12/05 01:42:18 INFO SimpleJob: Starting task 2:1 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 25 | 10/12/05 01:42:21 INFO SimpleJob: Finished TID 5 (progress: 1/2) 26 | 10/12/05 01:42:22 INFO SimpleJob: Finished TID 4 (progress: 2/2) 27 | 10/12/05 01:42:22 INFO SparkContext: Tasks finished in 4.14572333 s 28 | Centroids changed by 29 | (0.000000, 0.000000, 0.000000) 30 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 31 | Final centroids: 32 | 7.776982 40.661016 33 | 44.751415 35.355014 34 | 17.872132 7.510652 35 | 36 | -------------------------------------------------------------------------------- /run-dataset-20101205T014106/0-3.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1205 01:42:24.066999 4799 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/05 01:42:24 INFO MesosScheduler: Registered as framework ID 201012050142-0-0028 4 | 10/12/05 01:42:24 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/05 01:42:24 INFO FileInputFormat: Total input paths to process : 1 6 | 10/12/05 01:42:24 INFO SparkContext: Running 2 tasks in parallel 7 | 10/12/05 01:42:24 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/05 01:42:24 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/05 01:42:46 INFO SimpleJob: Finished TID 1 (progress: 1/2) 10 | 10/12/05 01:42:51 INFO SimpleJob: Finished TID 0 (progress: 2/2) 11 | 10/12/05 01:42:51 INFO SparkContext: Tasks finished in 26.491413211 s 12 | Read 10000000 points. 13 | 10/12/05 01:42:51 INFO SparkContext: Running 2 tasks in parallel 14 | 10/12/05 01:42:51 INFO SimpleJob: Starting task 1:0 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 15 | 10/12/05 01:42:51 INFO SimpleJob: Starting task 1:1 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 16 | 10/12/05 01:42:56 INFO SimpleJob: Finished TID 3 (progress: 1/2) 17 | 10/12/05 01:42:56 INFO SimpleJob: Finished TID 2 (progress: 2/2) 18 | 10/12/05 01:42:56 INFO SparkContext: Tasks finished in 5.505318545 s 19 | Centroids changed by 20 | (1.020119, 0.433392, 2.183411) 21 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 22 | 10/12/05 01:42:56 INFO SparkContext: Running 2 tasks in parallel 23 | 10/12/05 01:42:56 INFO SimpleJob: Starting task 2:0 as TID 4 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 24 | 10/12/05 01:42:56 INFO SimpleJob: Starting task 2:1 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 25 | 10/12/05 01:43:00 INFO SimpleJob: Finished TID 4 (progress: 1/2) 26 | 10/12/05 01:43:02 INFO SimpleJob: Finished TID 5 (progress: 2/2) 27 | 10/12/05 01:43:02 INFO SparkContext: Tasks finished in 5.433287425 s 28 | Centroids changed by 29 | (0.000000, 0.000000, 0.000000) 30 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 31 | Final centroids: 32 | 7.776982 40.661016 33 | 44.751415 35.355014 34 | 17.872132 7.510652 35 | 36 | -------------------------------------------------------------------------------- /SparkKMeans.scala: -------------------------------------------------------------------------------- 1 | import spark._ 2 | import spark.SparkContext._ 3 | 4 | object SparkKMeans { 5 | def main(args: Array[String]) { 6 | if (args.length < 2) { 7 | System.err.println("Usage: SparkKMeans [, ...]") 8 | System.exit(-1) 9 | } 10 | 11 | val pointsFile = args(0) 12 | val k = args(1).toInt 13 | val host = args(2) 14 | val startCentroids = 15 | (args 16 | .slice(3, 3 + 2 * k) 17 | .map(p => { 18 | val ps = p.split(',') 19 | Point(ps(0).toDouble, ps(1).toDouble) 20 | })) 21 | 22 | val sc = new SparkContext(host, "SparkKMeans") 23 | 24 | // Parse the points from a file into an RDD 25 | val points = sc.textFile(pointsFile).filter(line => !line.matches("^\\s*#.*")).map( 26 | line => { 27 | val parts = line.split("\t").map(_.toDouble) 28 | new Point(parts(0), parts(1)) 29 | } 30 | ).cache 31 | System.err.println("Read " + points.count() + " points.") 32 | 33 | // Use the given centroids, or initialize k random ones 34 | val centroids = ( 35 | if (startCentroids.length == k) 36 | startCentroids 37 | else 38 | Array.fill(k) { Point.random } 39 | ) 40 | 41 | // Start the Spark run 42 | val resultCentroids = kmeans(points, centroids, 0.1, sc) 43 | 44 | System.err.println("Final centroids: ") 45 | println(resultCentroids.map(centroid => "%3f\t%3f\n".format(centroid.x, centroid.y)).mkString) 46 | } 47 | 48 | def kmeans(points: spark.RDD[Point], centroids: Seq[Point], epsilon: Double, sc: SparkContext): Seq[Point] = { 49 | // Assign points into clusters based on their closest centroids, 50 | // and take the average of all points in each cluster 51 | val clusters = 52 | (points 53 | .map(point => KMeansHelper.closestCentroid(centroids, point) -> (point, 1)) 54 | .reduceByKeyToDriver({ 55 | case ((ptA, numA), (ptB, numB)) => (ptA + ptB, numA + numB) 56 | }) 57 | .map({ 58 | case (centroid, (ptSum, numPts)) => centroid -> ptSum / numPts 59 | })) 60 | 61 | // Recalculate centroids based on their clusters 62 | // (or leave them alone if they don't have any points in their cluster) 63 | val newCentroids = centroids.map(oldCentroid => { 64 | clusters.get(oldCentroid) match { 65 | case Some(newCentroid) => newCentroid 66 | case None => oldCentroid 67 | } 68 | }) 69 | 70 | // Calculate the centroid movement for the stopping condition 71 | val movement = (centroids zip newCentroids).map({ case (a, b) => a distance b }) 72 | 73 | System.err.println("Centroids changed by\n" + 74 | "\t " + movement.map(d => "%3f".format(d)).mkString("(", ", ", ")") + "\n" + 75 | "\tto " + newCentroids.mkString("(", ", ", ")")) 76 | 77 | // Iterate if movement exceeds threshold 78 | if (movement.exists(_ > epsilon)) 79 | kmeans(points, newCentroids, epsilon, sc) 80 | else 81 | return newCentroids 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /run-dataset-20101204T225747/1-1.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1204 22:59:39.305950 3887 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/04 22:59:39 INFO MesosScheduler: Registered as framework ID 201012042259-0-0014 4 | 10/12/04 22:59:39 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/04 22:59:39 INFO FileInputFormat: Total input paths to process : 2 6 | 10/12/04 22:59:39 INFO SparkContext: Running 4 tasks in parallel 7 | 10/12/04 22:59:39 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/04 22:59:39 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/04 22:59:39 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/04 22:59:39 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/04 22:59:55 INFO SimpleJob: Finished TID 1 (progress: 1/4) 12 | 10/12/04 22:59:55 INFO SimpleJob: Finished TID 3 (progress: 2/4) 13 | 10/12/04 23:00:14 INFO SimpleJob: Finished TID 0 (progress: 3/4) 14 | 10/12/04 23:00:14 INFO SimpleJob: Finished TID 2 (progress: 4/4) 15 | 10/12/04 23:00:14 INFO SparkContext: Tasks finished in 35.073621258 s 16 | Read 20000000 points. 17 | 10/12/04 23:00:14 INFO SparkContext: Running 4 tasks in parallel 18 | 10/12/04 23:00:14 INFO SimpleJob: Starting task 1:0 as TID 4 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/04 23:00:15 INFO SimpleJob: Starting task 1:1 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 20 | 10/12/04 23:00:15 INFO SimpleJob: Starting task 1:2 as TID 6 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 21 | 10/12/04 23:00:15 INFO SimpleJob: Starting task 1:3 as TID 7 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 22 | 10/12/04 23:00:20 INFO SimpleJob: Finished TID 7 (progress: 1/4) 23 | 10/12/04 23:00:20 INFO SimpleJob: Finished TID 5 (progress: 2/4) 24 | 10/12/04 23:00:23 INFO SimpleJob: Finished TID 6 (progress: 3/4) 25 | 10/12/04 23:00:23 INFO SimpleJob: Finished TID 4 (progress: 4/4) 26 | 10/12/04 23:00:23 INFO SparkContext: Tasks finished in 8.653859306 s 27 | Centroids changed by 28 | (1.020119, 0.433392, 2.183411) 29 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 30 | 10/12/04 23:00:23 INFO SparkContext: Running 4 tasks in parallel 31 | 10/12/04 23:00:23 INFO SimpleJob: Starting task 2:0 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 32 | 10/12/04 23:00:23 INFO SimpleJob: Starting task 2:1 as TID 9 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 33 | 10/12/04 23:00:23 INFO SimpleJob: Starting task 2:2 as TID 10 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 34 | 10/12/04 23:00:23 INFO SimpleJob: Starting task 2:3 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 35 | 10/12/04 23:00:28 INFO SimpleJob: Finished TID 9 (progress: 1/4) 36 | 10/12/04 23:00:28 INFO SimpleJob: Finished TID 11 (progress: 2/4) 37 | 10/12/04 23:00:31 INFO SimpleJob: Finished TID 8 (progress: 3/4) 38 | 10/12/04 23:00:32 INFO SimpleJob: Finished TID 10 (progress: 4/4) 39 | 10/12/04 23:00:32 INFO SparkContext: Tasks finished in 8.361218107 s 40 | Centroids changed by 41 | (0.000000, 0.000000, 0.000000) 42 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 43 | Final centroids: 44 | 7.776982 40.661016 45 | 44.751415 35.355014 46 | 17.872132 7.510652 47 | 48 | -------------------------------------------------------------------------------- /run-dataset-20101204T225747/1-2.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1204 23:00:33.638340 3928 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/04 23:00:33 INFO MesosScheduler: Registered as framework ID 201012042300-0-0015 4 | 10/12/04 23:00:33 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/04 23:00:34 INFO FileInputFormat: Total input paths to process : 2 6 | 10/12/04 23:00:34 INFO SparkContext: Running 4 tasks in parallel 7 | 10/12/04 23:00:34 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/04 23:00:34 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/04 23:00:34 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/04 23:00:34 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/04 23:00:49 INFO SimpleJob: Finished TID 1 (progress: 1/4) 12 | 10/12/04 23:00:49 INFO SimpleJob: Finished TID 3 (progress: 2/4) 13 | 10/12/04 23:01:09 INFO SimpleJob: Finished TID 0 (progress: 3/4) 14 | 10/12/04 23:01:09 INFO SimpleJob: Finished TID 2 (progress: 4/4) 15 | 10/12/04 23:01:09 INFO SparkContext: Tasks finished in 35.206983013 s 16 | Read 20000000 points. 17 | 10/12/04 23:01:09 INFO SparkContext: Running 4 tasks in parallel 18 | 10/12/04 23:01:09 INFO SimpleJob: Starting task 1:0 as TID 4 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/04 23:01:09 INFO SimpleJob: Starting task 1:1 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 20 | 10/12/04 23:01:09 INFO SimpleJob: Starting task 1:2 as TID 6 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 21 | 10/12/04 23:01:09 INFO SimpleJob: Starting task 1:3 as TID 7 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 22 | 10/12/04 23:01:15 INFO SimpleJob: Finished TID 7 (progress: 1/4) 23 | 10/12/04 23:01:15 INFO SimpleJob: Finished TID 5 (progress: 2/4) 24 | 10/12/04 23:01:18 INFO SimpleJob: Finished TID 4 (progress: 3/4) 25 | 10/12/04 23:01:18 INFO SimpleJob: Finished TID 6 (progress: 4/4) 26 | 10/12/04 23:01:18 INFO SparkContext: Tasks finished in 8.566587941 s 27 | Centroids changed by 28 | (1.020119, 0.433392, 2.183411) 29 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 30 | 10/12/04 23:01:18 INFO SparkContext: Running 4 tasks in parallel 31 | 10/12/04 23:01:18 INFO SimpleJob: Starting task 2:0 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 32 | 10/12/04 23:01:18 INFO SimpleJob: Starting task 2:1 as TID 9 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 33 | 10/12/04 23:01:18 INFO SimpleJob: Starting task 2:2 as TID 10 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 34 | 10/12/04 23:01:18 INFO SimpleJob: Starting task 2:3 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 35 | 10/12/04 23:01:21 INFO SimpleJob: Finished TID 11 (progress: 1/4) 36 | 10/12/04 23:01:21 INFO SimpleJob: Finished TID 9 (progress: 2/4) 37 | 10/12/04 23:01:26 INFO SimpleJob: Finished TID 8 (progress: 3/4) 38 | 10/12/04 23:01:26 INFO SimpleJob: Finished TID 10 (progress: 4/4) 39 | 10/12/04 23:01:26 INFO SparkContext: Tasks finished in 8.033784565 s 40 | Centroids changed by 41 | (0.000000, 0.000000, 0.000000) 42 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 43 | Final centroids: 44 | 7.776982 40.661016 45 | 44.751415 35.355014 46 | 17.872132 7.510652 47 | 48 | -------------------------------------------------------------------------------- /run-dataset-20101204T225747/1-3.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1204 23:01:27.780500 3969 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/04 23:01:27 INFO MesosScheduler: Registered as framework ID 201012042301-0-0016 4 | 10/12/04 23:01:27 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/04 23:01:28 INFO FileInputFormat: Total input paths to process : 2 6 | 10/12/04 23:01:28 INFO SparkContext: Running 4 tasks in parallel 7 | 10/12/04 23:01:28 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/04 23:01:28 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/04 23:01:28 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/04 23:01:28 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/04 23:01:43 INFO SimpleJob: Finished TID 3 (progress: 1/4) 12 | 10/12/04 23:01:43 INFO SimpleJob: Finished TID 1 (progress: 2/4) 13 | 10/12/04 23:02:03 INFO SimpleJob: Finished TID 2 (progress: 3/4) 14 | 10/12/04 23:02:03 INFO SimpleJob: Finished TID 0 (progress: 4/4) 15 | 10/12/04 23:02:03 INFO SparkContext: Tasks finished in 34.998028501 s 16 | Read 20000000 points. 17 | 10/12/04 23:02:03 INFO SparkContext: Running 4 tasks in parallel 18 | 10/12/04 23:02:03 INFO SimpleJob: Starting task 1:0 as TID 4 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/04 23:02:03 INFO SimpleJob: Starting task 1:1 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 20 | 10/12/04 23:02:03 INFO SimpleJob: Starting task 1:2 as TID 6 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 21 | 10/12/04 23:02:03 INFO SimpleJob: Starting task 1:3 as TID 7 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 22 | 10/12/04 23:02:09 INFO SimpleJob: Finished TID 7 (progress: 1/4) 23 | 10/12/04 23:02:09 INFO SimpleJob: Finished TID 5 (progress: 2/4) 24 | 10/12/04 23:02:11 INFO SimpleJob: Finished TID 4 (progress: 3/4) 25 | 10/12/04 23:02:11 INFO SimpleJob: Finished TID 6 (progress: 4/4) 26 | 10/12/04 23:02:11 INFO SparkContext: Tasks finished in 8.510977126 s 27 | Centroids changed by 28 | (1.020119, 0.433392, 2.183411) 29 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 30 | 10/12/04 23:02:11 INFO SparkContext: Running 4 tasks in parallel 31 | 10/12/04 23:02:11 INFO SimpleJob: Starting task 2:0 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 32 | 10/12/04 23:02:11 INFO SimpleJob: Starting task 2:1 as TID 9 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 33 | 10/12/04 23:02:11 INFO SimpleJob: Starting task 2:2 as TID 10 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 34 | 10/12/04 23:02:11 INFO SimpleJob: Starting task 2:3 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 35 | 10/12/04 23:02:15 INFO SimpleJob: Finished TID 11 (progress: 1/4) 36 | 10/12/04 23:02:15 INFO SimpleJob: Finished TID 9 (progress: 2/4) 37 | 10/12/04 23:02:19 INFO SimpleJob: Finished TID 8 (progress: 3/4) 38 | 10/12/04 23:02:19 INFO SimpleJob: Finished TID 10 (progress: 4/4) 39 | 10/12/04 23:02:19 INFO SparkContext: Tasks finished in 7.993196257 s 40 | Centroids changed by 41 | (0.000000, 0.000000, 0.000000) 42 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 43 | Final centroids: 44 | 7.776982 40.661016 45 | 44.751415 35.355014 46 | 17.872132 7.510652 47 | 48 | -------------------------------------------------------------------------------- /run-dataset-20101205T014106/1-1.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1205 01:43:03.728092 4843 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/05 01:43:03 INFO MesosScheduler: Registered as framework ID 201012050143-0-0029 4 | 10/12/05 01:43:03 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/05 01:43:04 INFO FileInputFormat: Total input paths to process : 2 6 | 10/12/05 01:43:04 INFO SparkContext: Running 4 tasks in parallel 7 | 10/12/05 01:43:04 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/05 01:43:04 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/05 01:43:04 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/05 01:43:04 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/05 01:43:18 INFO SimpleJob: Finished TID 3 (progress: 1/4) 12 | 10/12/05 01:43:18 INFO SimpleJob: Finished TID 1 (progress: 2/4) 13 | 10/12/05 01:43:39 INFO SimpleJob: Finished TID 2 (progress: 3/4) 14 | 10/12/05 01:43:39 INFO SimpleJob: Finished TID 0 (progress: 4/4) 15 | 10/12/05 01:43:39 INFO SparkContext: Tasks finished in 35.117827793 s 16 | Read 20000000 points. 17 | 10/12/05 01:43:39 INFO SparkContext: Running 4 tasks in parallel 18 | 10/12/05 01:43:39 INFO SimpleJob: Starting task 1:0 as TID 4 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/05 01:43:39 INFO SimpleJob: Starting task 1:1 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 20 | 10/12/05 01:43:39 INFO SimpleJob: Starting task 1:2 as TID 6 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 21 | 10/12/05 01:43:39 INFO SimpleJob: Starting task 1:3 as TID 7 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 22 | 10/12/05 01:43:44 INFO SimpleJob: Finished TID 5 (progress: 1/4) 23 | 10/12/05 01:43:45 INFO SimpleJob: Finished TID 7 (progress: 2/4) 24 | 10/12/05 01:43:46 INFO SimpleJob: Finished TID 6 (progress: 3/4) 25 | 10/12/05 01:44:09 INFO SimpleJob: Finished TID 4 (progress: 4/4) 26 | 10/12/05 01:44:09 INFO SparkContext: Tasks finished in 30.369976542 s 27 | Centroids changed by 28 | (1.020119, 0.433392, 2.183411) 29 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 30 | 10/12/05 01:44:09 INFO SparkContext: Running 4 tasks in parallel 31 | 10/12/05 01:44:09 INFO SimpleJob: Starting task 2:0 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 32 | 10/12/05 01:44:09 INFO SimpleJob: Starting task 2:1 as TID 9 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 33 | 10/12/05 01:44:09 INFO SimpleJob: Starting task 2:2 as TID 10 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 34 | 10/12/05 01:44:09 INFO SimpleJob: Starting task 2:3 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 35 | 10/12/05 01:44:13 INFO SimpleJob: Finished TID 9 (progress: 1/4) 36 | 10/12/05 01:44:13 INFO SimpleJob: Finished TID 11 (progress: 2/4) 37 | 10/12/05 01:44:17 INFO SimpleJob: Finished TID 8 (progress: 3/4) 38 | 10/12/05 01:44:17 INFO SimpleJob: Finished TID 10 (progress: 4/4) 39 | 10/12/05 01:44:17 INFO SparkContext: Tasks finished in 8.147326311 s 40 | Centroids changed by 41 | (0.000000, 0.000000, 0.000000) 42 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 43 | Final centroids: 44 | 7.776982 40.661016 45 | 44.751415 35.355014 46 | 17.872132 7.510652 47 | 48 | -------------------------------------------------------------------------------- /run-dataset-20101205T014106/1-2.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1205 01:44:19.563797 4884 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/05 01:44:19 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 4 | 10/12/05 01:44:19 INFO MesosScheduler: Registered as framework ID 201012050144-0-0030 5 | 10/12/05 01:44:20 INFO FileInputFormat: Total input paths to process : 2 6 | 10/12/05 01:44:20 INFO SparkContext: Running 4 tasks in parallel 7 | 10/12/05 01:44:20 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/05 01:44:20 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/05 01:44:20 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/05 01:44:20 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/05 01:44:34 INFO SimpleJob: Finished TID 3 (progress: 1/4) 12 | 10/12/05 01:44:34 INFO SimpleJob: Finished TID 1 (progress: 2/4) 13 | 10/12/05 01:44:55 INFO SimpleJob: Finished TID 2 (progress: 3/4) 14 | 10/12/05 01:44:55 INFO SimpleJob: Finished TID 0 (progress: 4/4) 15 | 10/12/05 01:44:55 INFO SparkContext: Tasks finished in 35.334421248 s 16 | Read 20000000 points. 17 | 10/12/05 01:44:55 INFO SparkContext: Running 4 tasks in parallel 18 | 10/12/05 01:44:55 INFO SimpleJob: Starting task 1:0 as TID 4 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/05 01:44:55 INFO SimpleJob: Starting task 1:1 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 20 | 10/12/05 01:44:55 INFO SimpleJob: Starting task 1:2 as TID 6 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 21 | 10/12/05 01:44:55 INFO SimpleJob: Starting task 1:3 as TID 7 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 22 | 10/12/05 01:45:00 INFO SimpleJob: Finished TID 5 (progress: 1/4) 23 | 10/12/05 01:45:03 INFO SimpleJob: Finished TID 4 (progress: 2/4) 24 | 10/12/05 01:45:03 INFO SimpleJob: Finished TID 6 (progress: 3/4) 25 | 10/12/05 01:45:08 INFO SimpleJob: Finished TID 7 (progress: 4/4) 26 | 10/12/05 01:45:08 INFO SparkContext: Tasks finished in 13.073602111 s 27 | Centroids changed by 28 | (1.020119, 0.433392, 2.183411) 29 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 30 | 10/12/05 01:45:08 INFO SparkContext: Running 4 tasks in parallel 31 | 10/12/05 01:45:08 INFO SimpleJob: Starting task 2:0 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 32 | 10/12/05 01:45:08 INFO SimpleJob: Starting task 2:1 as TID 9 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 33 | 10/12/05 01:45:08 INFO SimpleJob: Starting task 2:2 as TID 10 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 34 | 10/12/05 01:45:08 INFO SimpleJob: Starting task 2:3 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 35 | 10/12/05 01:45:12 INFO SimpleJob: Finished TID 9 (progress: 1/4) 36 | 10/12/05 01:45:12 INFO SimpleJob: Finished TID 11 (progress: 2/4) 37 | 10/12/05 01:45:16 INFO SimpleJob: Finished TID 8 (progress: 3/4) 38 | 10/12/05 01:45:16 INFO SimpleJob: Finished TID 10 (progress: 4/4) 39 | 10/12/05 01:45:16 INFO SparkContext: Tasks finished in 7.949746417 s 40 | Centroids changed by 41 | (0.000000, 0.000000, 0.000000) 42 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 43 | Final centroids: 44 | 7.776982 40.661016 45 | 44.751415 35.355014 46 | 17.872132 7.510652 47 | 48 | -------------------------------------------------------------------------------- /run-dataset-20101205T014106/1-3.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1205 01:45:18.118898 4929 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/05 01:45:18 INFO MesosScheduler: Registered as framework ID 201012050145-0-0031 4 | 10/12/05 01:45:18 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/05 01:45:18 INFO FileInputFormat: Total input paths to process : 2 6 | 10/12/05 01:45:18 INFO SparkContext: Running 4 tasks in parallel 7 | 10/12/05 01:45:18 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/05 01:45:18 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/05 01:45:18 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/05 01:45:18 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/05 01:45:33 INFO SimpleJob: Finished TID 1 (progress: 1/4) 12 | 10/12/05 01:45:33 INFO SimpleJob: Finished TID 3 (progress: 2/4) 13 | 10/12/05 01:45:55 INFO SimpleJob: Finished TID 2 (progress: 3/4) 14 | 10/12/05 01:45:55 INFO SimpleJob: Finished TID 0 (progress: 4/4) 15 | 10/12/05 01:45:55 INFO SparkContext: Tasks finished in 36.93196927 s 16 | Read 20000000 points. 17 | 10/12/05 01:45:55 INFO SparkContext: Running 4 tasks in parallel 18 | 10/12/05 01:45:55 INFO SimpleJob: Starting task 1:0 as TID 4 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/05 01:45:55 INFO SimpleJob: Starting task 1:1 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 20 | 10/12/05 01:45:55 INFO SimpleJob: Starting task 1:2 as TID 6 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 21 | 10/12/05 01:45:55 INFO SimpleJob: Starting task 1:3 as TID 7 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 22 | 10/12/05 01:46:01 INFO SimpleJob: Finished TID 5 (progress: 1/4) 23 | 10/12/05 01:46:01 INFO SimpleJob: Finished TID 7 (progress: 2/4) 24 | 10/12/05 01:46:04 INFO SimpleJob: Finished TID 4 (progress: 3/4) 25 | 10/12/05 01:46:04 INFO SimpleJob: Finished TID 6 (progress: 4/4) 26 | 10/12/05 01:46:04 INFO SparkContext: Tasks finished in 8.658107797 s 27 | Centroids changed by 28 | (1.020119, 0.433392, 2.183411) 29 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 30 | 10/12/05 01:46:04 INFO SparkContext: Running 4 tasks in parallel 31 | 10/12/05 01:46:04 INFO SimpleJob: Starting task 2:0 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 32 | 10/12/05 01:46:04 INFO SimpleJob: Starting task 2:1 as TID 9 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 33 | 10/12/05 01:46:04 INFO SimpleJob: Starting task 2:2 as TID 10 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 34 | 10/12/05 01:46:04 INFO SimpleJob: Starting task 2:3 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 35 | 10/12/05 01:46:07 INFO SimpleJob: Finished TID 9 (progress: 1/4) 36 | 10/12/05 01:46:07 INFO SimpleJob: Finished TID 11 (progress: 2/4) 37 | 10/12/05 01:46:12 INFO SimpleJob: Finished TID 8 (progress: 3/4) 38 | 10/12/05 01:46:12 INFO SimpleJob: Finished TID 10 (progress: 4/4) 39 | 10/12/05 01:46:12 INFO SparkContext: Tasks finished in 7.891924799 s 40 | Centroids changed by 41 | (0.000000, 0.000000, 0.000000) 42 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 43 | Final centroids: 44 | 7.776982 40.661016 45 | 44.751415 35.355014 46 | 17.872132 7.510652 47 | 48 | -------------------------------------------------------------------------------- /run-scaleout-20101205T231425/quota-error.txt: -------------------------------------------------------------------------------- 1 | Number of slaves: 16 2 | Launching cluster... 3 | Setting up security groups... 4 | Checking for running cluster... 5 | Launching instances... 6 | ERROR:boto:400 Bad Request 7 | ERROR:boto: 8 | InstanceLimitExceededYour quota allows for 11 more running instance(s). You requested at least 1682c2f035-1bb5-49e8-abee-f0397b31d26e 9 | Traceback (most recent call last): 10 | File "./mesos_ec2.py", line 477, in 11 | main() 12 | File "./mesos_ec2.py", line 384, in main 13 | conn, opts, cluster_name) 14 | File "./mesos_ec2.py", line 193, in launch_cluster 15 | block_device_map = block_map) 16 | File "/work/ankurd/mesos/third_party/boto-2.0b2/boto/ec2/image.py", line 207, in run 17 | placement_group) 18 | File "/work/ankurd/mesos/third_party/boto-2.0b2/boto/ec2/connection.py", line 502, in run_instances 19 | return self.get_object('RunInstances', params, Reservation, verb='POST') 20 | File "/work/ankurd/mesos/third_party/boto-2.0b2/boto/connection.py", line 648, in get_object 21 | raise self.ResponseError(response.status, response.reason, body) 22 | boto.exception.EC2ResponseError: EC2ResponseError: 400 Bad Request 23 | 24 | InstanceLimitExceededYour quota allows for 11 more running instance(s). You requested at least 1682c2f035-1bb5-49e8-abee-f0397b31d26e 25 | Master public hostname: ERROR: Could not find any existing cluster 26 | Master Mesos PID: 27 | Generating point file... 28 | ssh: Could not resolve hostname ERROR:: Name or service not known 29 | Starting performance run... 30 | Trial 1 31 | ssh: Could not resolve hostname ERROR:: Name or service not known 32 | Trial 2 33 | ssh: Could not resolve hostname ERROR:: Name or service not known 34 | Trial 3 35 | ssh: Could not resolve hostname ERROR:: Name or service not known 36 | Ending performance run... 37 | Are you sure you want to destroy the cluster KMeans? 38 | ALL DATA ON ALL NODES WILL BE LOST!! 39 | Destroy cluster KMeans (y/N): Searching for existing cluster KMeans... 40 | ERROR: Could not find any existing cluster 41 | Number of slaves: 32 42 | Launching cluster... 43 | Setting up security groups... 44 | Checking for running cluster... 45 | Launching instances... 46 | ERROR:boto:400 Bad Request 47 | ERROR:boto: 48 | InstanceLimitExceededYour quota allows for 13 more running instance(s). You requested at least 327339a25e-26e1-41c5-8a41-e6c0b215be2f 49 | Traceback (most recent call last): 50 | File "./mesos_ec2.py", line 477, in 51 | main() 52 | File "./mesos_ec2.py", line 384, in main 53 | conn, opts, cluster_name) 54 | File "./mesos_ec2.py", line 193, in launch_cluster 55 | block_device_map = block_map) 56 | File "/work/ankurd/mesos/third_party/boto-2.0b2/boto/ec2/image.py", line 207, in run 57 | placement_group) 58 | File "/work/ankurd/mesos/third_party/boto-2.0b2/boto/ec2/connection.py", line 502, in run_instances 59 | return self.get_object('RunInstances', params, Reservation, verb='POST') 60 | File "/work/ankurd/mesos/third_party/boto-2.0b2/boto/connection.py", line 648, in get_object 61 | raise self.ResponseError(response.status, response.reason, body) 62 | boto.exception.EC2ResponseError: EC2ResponseError: 400 Bad Request 63 | 64 | InstanceLimitExceededYour quota allows for 13 more running instance(s). You requested at least 327339a25e-26e1-41c5-8a41-e6c0b215be2f 65 | Master public hostname: ERROR: Could not find any existing cluster 66 | Master Mesos PID: 67 | Generating point file... 68 | ssh: Could not resolve hostname ERROR:: Name or service not known 69 | Starting performance run... 70 | Trial 1 71 | ssh: Could not resolve hostname ERROR:: Name or service not known 72 | Trial 2 73 | ssh: Could not resolve hostname ERROR:: Name or service not known 74 | Trial 3 75 | ssh: Could not resolve hostname ERROR:: Name or service not known 76 | Ending performance run... 77 | Are you sure you want to destroy the cluster KMeans? 78 | ALL DATA ON ALL NODES WILL BE LOST!! 79 | Destroy cluster KMeans (y/N): Searching for existing cluster KMeans... 80 | ERROR: Could not find any existing cluster 81 | -------------------------------------------------------------------------------- /run-scaleup-20101206T184139/3-1.log: -------------------------------------------------------------------------------- 1 | stdin: is not a tty 2 | WARNING: Logging before InitGoogleLogging() is written to STDERR 3 | I1207 05:33:07.875705 3155 sched.cpp:181] New master at 1@10.242.34.207:5050 with ID:0 4 | 10/12/07 05:33:07 INFO spark.BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/07 05:33:07 INFO spark.MesosScheduler: Registered as framework ID 201012070533-0-0000 6 | 10/12/07 05:33:08 INFO mapred.FileInputFormat: Total input paths to process : 1 7 | 10/12/07 05:33:08 INFO spark.SparkContext: Running 15 tasks in parallel 8 | 10/12/07 05:33:08 INFO spark.SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-98-197-93.ec2.internal (preferred) 9 | 10/12/07 05:33:08 INFO spark.SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 10 | 10/12/07 05:33:08 INFO spark.SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-98-197-93.ec2.internal (preferred) 11 | 10/12/07 05:33:08 INFO spark.SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 12 | 10/12/07 05:33:08 INFO spark.SimpleJob: Starting task 0:4 as TID 4 on slave 0-0: ip-10-98-197-93.ec2.internal (preferred) 13 | 10/12/07 05:33:08 INFO spark.SimpleJob: Starting task 0:5 as TID 5 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 14 | 10/12/07 05:33:08 INFO spark.SimpleJob: Starting task 0:6 as TID 6 on slave 0-0: ip-10-98-197-93.ec2.internal (preferred) 15 | 10/12/07 05:33:08 INFO spark.SimpleJob: Starting task 0:7 as TID 7 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 16 | 10/12/07 05:33:08 INFO spark.SimpleJob: Starting task 0:8 as TID 8 on slave 0-0: ip-10-98-197-93.ec2.internal (preferred) 17 | 10/12/07 05:33:08 INFO spark.SimpleJob: Starting task 0:9 as TID 9 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 18 | 10/12/07 05:33:08 INFO spark.SimpleJob: Starting task 0:10 as TID 10 on slave 0-0: ip-10-98-197-93.ec2.internal (preferred) 19 | 10/12/07 05:33:08 INFO spark.SimpleJob: Starting task 0:11 as TID 11 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 20 | 10/12/07 05:33:08 INFO spark.SimpleJob: Starting task 0:12 as TID 12 on slave 0-0: ip-10-98-197-93.ec2.internal (preferred) 21 | 10/12/07 05:33:08 INFO spark.SimpleJob: Starting task 0:13 as TID 13 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 22 | 10/12/07 05:33:08 INFO spark.SimpleJob: Starting task 0:14 as TID 14 on slave 0-0: ip-10-98-197-93.ec2.internal (preferred) 23 | 10/12/07 05:34:37 INFO spark.SimpleJob: Lost TID 7 (task 0:7) 24 | 10/12/07 05:34:37 INFO spark.SimpleJob: Lost TID 9 (task 0:9) 25 | 10/12/07 05:34:37 INFO spark.SimpleJob: Lost TID 11 (task 0:11) 26 | 10/12/07 05:34:37 INFO spark.SimpleJob: Lost TID 13 (task 0:13) 27 | 10/12/07 05:34:37 INFO spark.SimpleJob: Lost TID 5 (task 0:5) 28 | 10/12/07 05:34:37 INFO spark.SimpleJob: Lost TID 1 (task 0:1) 29 | 10/12/07 05:34:37 INFO spark.SimpleJob: Lost TID 3 (task 0:3) 30 | 10/12/07 05:34:37 INFO spark.SimpleJob: Starting task 0:3 as TID 15 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 31 | 10/12/07 05:34:37 INFO spark.SimpleJob: Starting task 0:1 as TID 16 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 32 | 10/12/07 05:34:37 INFO spark.SimpleJob: Starting task 0:5 as TID 17 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 33 | 10/12/07 05:34:37 INFO spark.SimpleJob: Starting task 0:13 as TID 18 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 34 | 10/12/07 05:34:37 INFO spark.SimpleJob: Starting task 0:11 as TID 19 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 35 | 10/12/07 05:34:37 INFO spark.SimpleJob: Starting task 0:9 as TID 20 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 36 | 10/12/07 05:34:37 INFO spark.SimpleJob: Starting task 0:7 as TID 21 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 37 | 10/12/07 05:35:15 INFO spark.SimpleJob: Finished TID 14 (progress: 1/15) 38 | 10/12/07 05:35:52 INFO spark.SimpleJob: Lost TID 18 (task 0:13) 39 | 10/12/07 05:35:52 INFO spark.SimpleJob: Lost TID 19 (task 0:11) 40 | 10/12/07 05:35:52 INFO spark.SimpleJob: Lost TID 21 (task 0:7) 41 | 10/12/07 05:35:52 INFO spark.SimpleJob: Lost TID 15 (task 0:3) 42 | 10/12/07 05:35:52 INFO spark.SimpleJob: Lost TID 17 (task 0:5) 43 | 10/12/07 05:35:52 INFO spark.SimpleJob: Lost TID 16 (task 0:1) 44 | 10/12/07 05:35:52 INFO spark.SimpleJob: Lost TID 20 (task 0:9) 45 | 10/12/07 05:35:53 INFO spark.SimpleJob: Starting task 0:9 as TID 22 on slave 0-0: ip-10-98-197-93.ec2.internal (non-preferred) 46 | 10/12/07 05:35:53 INFO spark.SimpleJob: Starting task 0:1 as TID 23 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 47 | 10/12/07 05:35:53 INFO spark.SimpleJob: Starting task 0:5 as TID 24 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 48 | 10/12/07 05:35:53 INFO spark.SimpleJob: Starting task 0:3 as TID 25 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 49 | 10/12/07 05:35:53 INFO spark.SimpleJob: Starting task 0:7 as TID 26 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 50 | 10/12/07 05:35:53 INFO spark.SimpleJob: Starting task 0:11 as TID 27 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 51 | 10/12/07 05:35:53 INFO spark.SimpleJob: Starting task 0:13 as TID 28 on slave 0-1: ip-10-242-34-191.ec2.internal (preferred) 52 | 10/12/07 05:37:07 INFO spark.SimpleJob: Finished TID 6 (progress: 2/15) 53 | 10/12/07 05:37:23 INFO spark.SimpleJob: Finished TID 2 (progress: 3/15) 54 | 10/12/07 05:37:23 INFO spark.SimpleJob: Finished TID 8 (progress: 4/15) 55 | 10/12/07 05:37:24 INFO spark.SimpleJob: Finished TID 0 (progress: 5/15) 56 | 10/12/07 05:37:24 INFO spark.SimpleJob: Finished TID 10 (progress: 6/15) 57 | 10/12/07 05:37:25 INFO spark.SimpleJob: Finished TID 4 (progress: 7/15) 58 | -------------------------------------------------------------------------------- /run-dataset-20101204T225747/2-1.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1204 23:02:21.503859 4012 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/04 23:02:21 INFO MesosScheduler: Registered as framework ID 201012042302-0-0017 4 | 10/12/04 23:02:21 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/04 23:02:22 INFO FileInputFormat: Total input paths to process : 4 6 | 10/12/04 23:02:22 INFO SparkContext: Running 8 tasks in parallel 7 | 10/12/04 23:02:22 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/04 23:02:22 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/04 23:02:22 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/04 23:02:22 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/04 23:02:38 INFO SimpleJob: Finished TID 1 (progress: 1/8) 12 | 10/12/04 23:02:38 INFO SimpleJob: Starting task 0:4 as TID 4 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 13 | 10/12/04 23:02:38 INFO SimpleJob: Finished TID 3 (progress: 2/8) 14 | 10/12/04 23:02:38 INFO SimpleJob: Starting task 0:5 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 15 | 10/12/04 23:02:38 INFO SimpleJob: Finished TID 5 (progress: 3/8) 16 | 10/12/04 23:02:38 INFO SimpleJob: Starting task 0:6 as TID 6 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 17 | 10/12/04 23:02:57 INFO SimpleJob: Finished TID 2 (progress: 4/8) 18 | 10/12/04 23:02:57 INFO SimpleJob: Starting task 0:7 as TID 7 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/04 23:02:57 INFO SimpleJob: Finished TID 0 (progress: 5/8) 20 | 10/12/04 23:03:06 INFO SimpleJob: Finished TID 6 (progress: 6/8) 21 | 10/12/04 23:03:06 INFO SimpleJob: Finished TID 4 (progress: 7/8) 22 | 10/12/04 23:03:09 INFO SimpleJob: Finished TID 7 (progress: 8/8) 23 | 10/12/04 23:03:09 INFO SparkContext: Tasks finished in 47.754237296 s 24 | Read 40000000 points. 25 | 10/12/04 23:03:09 INFO SparkContext: Running 8 tasks in parallel 26 | 10/12/04 23:03:09 INFO SimpleJob: Starting task 1:0 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 27 | 10/12/04 23:03:09 INFO SimpleJob: Starting task 1:1 as TID 9 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 28 | 10/12/04 23:03:09 INFO SimpleJob: Starting task 1:2 as TID 10 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 29 | 10/12/04 23:03:09 INFO SimpleJob: Starting task 1:3 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 30 | 10/12/04 23:03:14 INFO SimpleJob: Finished TID 9 (progress: 1/8) 31 | 10/12/04 23:03:14 INFO SimpleJob: Starting task 1:4 as TID 12 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 32 | 10/12/04 23:03:14 INFO SimpleJob: Finished TID 11 (progress: 2/8) 33 | 10/12/04 23:03:14 INFO SimpleJob: Starting task 1:5 as TID 13 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 34 | 10/12/04 23:03:17 INFO SimpleJob: Finished TID 10 (progress: 3/8) 35 | 10/12/04 23:03:17 INFO SimpleJob: Starting task 1:7 as TID 14 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 36 | 10/12/04 23:03:17 INFO SimpleJob: Finished TID 8 (progress: 4/8) 37 | 10/12/04 23:03:19 INFO SimpleJob: Finished TID 13 (progress: 5/8) 38 | 10/12/04 23:03:19 INFO SimpleJob: Starting task 1:6 as TID 15 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 39 | 10/12/04 23:03:20 INFO SimpleJob: Finished TID 14 (progress: 6/8) 40 | 10/12/04 23:03:24 INFO SimpleJob: Finished TID 12 (progress: 7/8) 41 | 10/12/04 23:03:26 INFO SimpleJob: Finished TID 15 (progress: 8/8) 42 | 10/12/04 23:03:26 INFO SparkContext: Tasks finished in 16.693034592 s 43 | Centroids changed by 44 | (1.020119, 0.433392, 2.183411) 45 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 46 | 10/12/04 23:03:26 INFO SparkContext: Running 8 tasks in parallel 47 | 10/12/04 23:03:26 INFO SimpleJob: Starting task 2:0 as TID 16 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 48 | 10/12/04 23:03:26 INFO SimpleJob: Starting task 2:1 as TID 17 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 49 | 10/12/04 23:03:26 INFO SimpleJob: Starting task 2:2 as TID 18 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 50 | 10/12/04 23:03:26 INFO SimpleJob: Starting task 2:3 as TID 19 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 51 | 10/12/04 23:03:31 INFO SimpleJob: Finished TID 17 (progress: 1/8) 52 | 10/12/04 23:03:31 INFO SimpleJob: Starting task 2:4 as TID 20 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 53 | 10/12/04 23:03:31 INFO SimpleJob: Finished TID 19 (progress: 2/8) 54 | 10/12/04 23:03:31 INFO SimpleJob: Starting task 2:5 as TID 21 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 55 | 10/12/04 23:03:34 INFO SimpleJob: Finished TID 16 (progress: 3/8) 56 | 10/12/04 23:03:34 INFO SimpleJob: Starting task 2:7 as TID 22 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 57 | 10/12/04 23:03:34 INFO SimpleJob: Finished TID 18 (progress: 4/8) 58 | 10/12/04 23:03:35 INFO SimpleJob: Finished TID 21 (progress: 5/8) 59 | 10/12/04 23:03:35 INFO SimpleJob: Starting task 2:6 as TID 23 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 60 | 10/12/04 23:03:36 INFO SimpleJob: Finished TID 22 (progress: 6/8) 61 | 10/12/04 23:03:41 INFO SimpleJob: Finished TID 20 (progress: 7/8) 62 | 10/12/04 23:03:43 INFO SimpleJob: Finished TID 23 (progress: 8/8) 63 | 10/12/04 23:03:43 INFO SparkContext: Tasks finished in 17.058480659 s 64 | Centroids changed by 65 | (0.000000, 0.000000, 0.000000) 66 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 67 | Final centroids: 68 | 7.776982 40.661016 69 | 44.751415 35.355014 70 | 17.872132 7.510652 71 | 72 | -------------------------------------------------------------------------------- /run-dataset-20101204T225747/2-2.log: -------------------------------------------------------------------------------- 1 | 10/12/04 23:03:45 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 2 | WARNING: Logging before InitGoogleLogging() is written to STDERR 3 | I1204 23:03:45.361492 4054 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 4 | 10/12/04 23:03:45 INFO MesosScheduler: Registered as framework ID 201012042303-0-0018 5 | 10/12/04 23:03:45 INFO FileInputFormat: Total input paths to process : 4 6 | 10/12/04 23:03:45 INFO SparkContext: Running 8 tasks in parallel 7 | 10/12/04 23:03:45 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/04 23:03:45 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/04 23:03:45 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/04 23:03:45 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/04 23:04:00 INFO SimpleJob: Finished TID 3 (progress: 1/8) 12 | 10/12/04 23:04:00 INFO SimpleJob: Starting task 0:4 as TID 4 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 13 | 10/12/04 23:04:00 INFO SimpleJob: Finished TID 1 (progress: 2/8) 14 | 10/12/04 23:04:00 INFO SimpleJob: Starting task 0:5 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 15 | 10/12/04 23:04:01 INFO SimpleJob: Finished TID 5 (progress: 3/8) 16 | 10/12/04 23:04:01 INFO SimpleJob: Starting task 0:6 as TID 6 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 17 | 10/12/04 23:04:21 INFO SimpleJob: Finished TID 2 (progress: 4/8) 18 | 10/12/04 23:04:21 INFO SimpleJob: Starting task 0:7 as TID 7 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/04 23:04:21 INFO SimpleJob: Finished TID 0 (progress: 5/8) 20 | 10/12/04 23:04:26 INFO SimpleJob: Finished TID 6 (progress: 6/8) 21 | 10/12/04 23:04:26 INFO SimpleJob: Finished TID 4 (progress: 7/8) 22 | 10/12/04 23:04:33 INFO SimpleJob: Finished TID 7 (progress: 8/8) 23 | 10/12/04 23:04:33 INFO SparkContext: Tasks finished in 47.245761986 s 24 | Read 40000000 points. 25 | 10/12/04 23:04:33 INFO SparkContext: Running 8 tasks in parallel 26 | 10/12/04 23:04:33 INFO SimpleJob: Starting task 1:0 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 27 | 10/12/04 23:04:33 INFO SimpleJob: Starting task 1:1 as TID 9 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 28 | 10/12/04 23:04:33 INFO SimpleJob: Starting task 1:2 as TID 10 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 29 | 10/12/04 23:04:33 INFO SimpleJob: Starting task 1:3 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 30 | 10/12/04 23:04:37 INFO SimpleJob: Finished TID 11 (progress: 1/8) 31 | 10/12/04 23:04:37 INFO SimpleJob: Starting task 1:4 as TID 12 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 32 | 10/12/04 23:04:37 INFO SimpleJob: Finished TID 9 (progress: 2/8) 33 | 10/12/04 23:04:37 INFO SimpleJob: Starting task 1:5 as TID 13 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 34 | 10/12/04 23:04:41 INFO SimpleJob: Finished TID 13 (progress: 3/8) 35 | 10/12/04 23:04:41 INFO SimpleJob: Starting task 1:6 as TID 14 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 36 | 10/12/04 23:04:41 INFO SimpleJob: Finished TID 8 (progress: 4/8) 37 | 10/12/04 23:04:41 INFO SimpleJob: Starting task 1:7 as TID 15 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 38 | 10/12/04 23:04:41 INFO SimpleJob: Finished TID 10 (progress: 5/8) 39 | 10/12/04 23:04:43 INFO SimpleJob: Finished TID 15 (progress: 6/8) 40 | 10/12/04 23:04:46 INFO SimpleJob: Finished TID 12 (progress: 7/8) 41 | 10/12/04 23:04:48 INFO SimpleJob: Finished TID 14 (progress: 8/8) 42 | 10/12/04 23:04:48 INFO SparkContext: Tasks finished in 15.168912906 s 43 | Centroids changed by 44 | (1.020119, 0.433392, 2.183411) 45 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 46 | 10/12/04 23:04:48 INFO SparkContext: Running 8 tasks in parallel 47 | 10/12/04 23:04:48 INFO SimpleJob: Starting task 2:0 as TID 16 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 48 | 10/12/04 23:04:48 INFO SimpleJob: Starting task 2:1 as TID 17 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 49 | 10/12/04 23:04:48 INFO SimpleJob: Starting task 2:2 as TID 18 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 50 | 10/12/04 23:04:48 INFO SimpleJob: Starting task 2:3 as TID 19 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 51 | 10/12/04 23:04:52 INFO SimpleJob: Finished TID 17 (progress: 1/8) 52 | 10/12/04 23:04:52 INFO SimpleJob: Starting task 2:4 as TID 20 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 53 | 10/12/04 23:04:52 INFO SimpleJob: Finished TID 19 (progress: 2/8) 54 | 10/12/04 23:04:52 INFO SimpleJob: Starting task 2:5 as TID 21 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 55 | 10/12/04 23:04:56 INFO SimpleJob: Finished TID 21 (progress: 3/8) 56 | 10/12/04 23:04:56 INFO SimpleJob: Starting task 2:6 as TID 22 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 57 | 10/12/04 23:04:56 INFO SimpleJob: Finished TID 16 (progress: 4/8) 58 | 10/12/04 23:04:56 INFO SimpleJob: Starting task 2:7 as TID 23 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 59 | 10/12/04 23:04:56 INFO SimpleJob: Finished TID 18 (progress: 5/8) 60 | 10/12/04 23:04:58 INFO SimpleJob: Finished TID 23 (progress: 6/8) 61 | 10/12/04 23:05:01 INFO SimpleJob: Finished TID 20 (progress: 7/8) 62 | 10/12/04 23:05:03 INFO SimpleJob: Finished TID 22 (progress: 8/8) 63 | 10/12/04 23:05:03 INFO SparkContext: Tasks finished in 14.989145182 s 64 | Centroids changed by 65 | (0.000000, 0.000000, 0.000000) 66 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 67 | Final centroids: 68 | 7.776982 40.661016 69 | 44.751415 35.355014 70 | 17.872132 7.510652 71 | 72 | -------------------------------------------------------------------------------- /run-dataset-20101204T225747/2-3.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1204 23:05:05.038697 4099 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/04 23:05:05 INFO MesosScheduler: Registered as framework ID 201012042305-0-0019 4 | 10/12/04 23:05:05 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/04 23:05:05 INFO FileInputFormat: Total input paths to process : 4 6 | 10/12/04 23:05:05 INFO SparkContext: Running 8 tasks in parallel 7 | 10/12/04 23:05:05 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/04 23:05:05 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/04 23:05:05 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/04 23:05:05 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/04 23:05:20 INFO SimpleJob: Finished TID 3 (progress: 1/8) 12 | 10/12/04 23:05:20 INFO SimpleJob: Starting task 0:4 as TID 4 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 13 | 10/12/04 23:05:20 INFO SimpleJob: Finished TID 1 (progress: 2/8) 14 | 10/12/04 23:05:20 INFO SimpleJob: Starting task 0:5 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 15 | 10/12/04 23:05:21 INFO SimpleJob: Finished TID 5 (progress: 3/8) 16 | 10/12/04 23:05:21 INFO SimpleJob: Starting task 0:6 as TID 6 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 17 | 10/12/04 23:05:43 INFO SimpleJob: Finished TID 0 (progress: 4/8) 18 | 10/12/04 23:05:43 INFO SimpleJob: Starting task 0:7 as TID 7 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/04 23:05:43 INFO SimpleJob: Finished TID 2 (progress: 5/8) 20 | 10/12/04 23:05:46 INFO SimpleJob: Finished TID 4 (progress: 6/8) 21 | 10/12/04 23:05:46 INFO SimpleJob: Finished TID 6 (progress: 7/8) 22 | 10/12/04 23:05:56 INFO SimpleJob: Finished TID 7 (progress: 8/8) 23 | 10/12/04 23:05:56 INFO SparkContext: Tasks finished in 50.920427713 s 24 | Read 40000000 points. 25 | 10/12/04 23:05:56 INFO SparkContext: Running 8 tasks in parallel 26 | 10/12/04 23:05:56 INFO SimpleJob: Starting task 1:0 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 27 | 10/12/04 23:05:56 INFO SimpleJob: Starting task 1:1 as TID 9 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 28 | 10/12/04 23:05:56 INFO SimpleJob: Starting task 1:2 as TID 10 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 29 | 10/12/04 23:05:56 INFO SimpleJob: Starting task 1:3 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 30 | 10/12/04 23:06:01 INFO SimpleJob: Finished TID 11 (progress: 1/8) 31 | 10/12/04 23:06:01 INFO SimpleJob: Starting task 1:4 as TID 12 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 32 | 10/12/04 23:06:01 INFO SimpleJob: Finished TID 9 (progress: 2/8) 33 | 10/12/04 23:06:01 INFO SimpleJob: Starting task 1:5 as TID 13 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 34 | 10/12/04 23:06:04 INFO SimpleJob: Finished TID 8 (progress: 3/8) 35 | 10/12/04 23:06:04 INFO SimpleJob: Starting task 1:7 as TID 14 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 36 | 10/12/04 23:06:05 INFO SimpleJob: Finished TID 10 (progress: 4/8) 37 | 10/12/04 23:06:05 INFO SimpleJob: Finished TID 13 (progress: 5/8) 38 | 10/12/04 23:06:05 INFO SimpleJob: Starting task 1:6 as TID 15 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 39 | 10/12/04 23:06:07 INFO SimpleJob: Finished TID 14 (progress: 6/8) 40 | 10/12/04 23:06:10 INFO SimpleJob: Finished TID 12 (progress: 7/8) 41 | 10/12/04 23:06:13 INFO SimpleJob: Finished TID 15 (progress: 8/8) 42 | 10/12/04 23:06:13 INFO SparkContext: Tasks finished in 16.524680117 s 43 | Centroids changed by 44 | (1.020119, 0.433392, 2.183411) 45 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 46 | 10/12/04 23:06:13 INFO SparkContext: Running 8 tasks in parallel 47 | 10/12/04 23:06:13 INFO SimpleJob: Starting task 2:0 as TID 16 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 48 | 10/12/04 23:06:13 INFO SimpleJob: Starting task 2:1 as TID 17 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 49 | 10/12/04 23:06:13 INFO SimpleJob: Starting task 2:2 as TID 18 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 50 | 10/12/04 23:06:13 INFO SimpleJob: Starting task 2:3 as TID 19 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 51 | 10/12/04 23:06:17 INFO SimpleJob: Finished TID 19 (progress: 1/8) 52 | 10/12/04 23:06:17 INFO SimpleJob: Starting task 2:4 as TID 20 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 53 | 10/12/04 23:06:17 INFO SimpleJob: Finished TID 17 (progress: 2/8) 54 | 10/12/04 23:06:17 INFO SimpleJob: Starting task 2:5 as TID 21 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 55 | 10/12/04 23:06:21 INFO SimpleJob: Finished TID 16 (progress: 3/8) 56 | 10/12/04 23:06:21 INFO SimpleJob: Starting task 2:7 as TID 22 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 57 | 10/12/04 23:06:21 INFO SimpleJob: Finished TID 18 (progress: 4/8) 58 | 10/12/04 23:06:22 INFO SimpleJob: Finished TID 21 (progress: 5/8) 59 | 10/12/04 23:06:22 INFO SimpleJob: Starting task 2:6 as TID 23 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 60 | 10/12/04 23:06:23 INFO SimpleJob: Finished TID 22 (progress: 6/8) 61 | 10/12/04 23:06:27 INFO SimpleJob: Finished TID 20 (progress: 7/8) 62 | 10/12/04 23:06:30 INFO SimpleJob: Finished TID 23 (progress: 8/8) 63 | 10/12/04 23:06:30 INFO SparkContext: Tasks finished in 16.900329118 s 64 | Centroids changed by 65 | (0.000000, 0.000000, 0.000000) 66 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 67 | Final centroids: 68 | 7.776982 40.661016 69 | 44.751415 35.355014 70 | 17.872132 7.510652 71 | 72 | -------------------------------------------------------------------------------- /run-dataset-20101205T014106/2-1.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1205 01:46:13.905045 4972 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/05 01:46:13 INFO MesosScheduler: Registered as framework ID 201012050146-0-0032 4 | 10/12/05 01:46:13 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/05 01:46:14 INFO FileInputFormat: Total input paths to process : 4 6 | 10/12/05 01:46:14 INFO SparkContext: Running 8 tasks in parallel 7 | 10/12/05 01:46:14 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/05 01:46:14 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/05 01:46:14 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/05 01:46:14 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/05 01:46:29 INFO SimpleJob: Finished TID 3 (progress: 1/8) 12 | 10/12/05 01:46:29 INFO SimpleJob: Starting task 0:4 as TID 4 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 13 | 10/12/05 01:46:29 INFO SimpleJob: Finished TID 1 (progress: 2/8) 14 | 10/12/05 01:46:29 INFO SimpleJob: Starting task 0:5 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 15 | 10/12/05 01:46:30 INFO SimpleJob: Finished TID 5 (progress: 3/8) 16 | 10/12/05 01:46:30 INFO SimpleJob: Starting task 0:6 as TID 6 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 17 | 10/12/05 01:46:48 INFO SimpleJob: Finished TID 2 (progress: 4/8) 18 | 10/12/05 01:46:48 INFO SimpleJob: Starting task 0:7 as TID 7 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/05 01:46:48 INFO SimpleJob: Finished TID 0 (progress: 5/8) 20 | 10/12/05 01:46:55 INFO SimpleJob: Finished TID 4 (progress: 6/8) 21 | 10/12/05 01:46:55 INFO SimpleJob: Finished TID 6 (progress: 7/8) 22 | 10/12/05 01:47:01 INFO SimpleJob: Finished TID 7 (progress: 8/8) 23 | 10/12/05 01:47:01 INFO SparkContext: Tasks finished in 46.754233261 s 24 | Read 40000000 points. 25 | 10/12/05 01:47:01 INFO SparkContext: Running 8 tasks in parallel 26 | 10/12/05 01:47:01 INFO SimpleJob: Starting task 1:0 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 27 | 10/12/05 01:47:01 INFO SimpleJob: Starting task 1:1 as TID 9 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 28 | 10/12/05 01:47:01 INFO SimpleJob: Starting task 1:2 as TID 10 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 29 | 10/12/05 01:47:01 INFO SimpleJob: Starting task 1:3 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 30 | 10/12/05 01:47:05 INFO SimpleJob: Finished TID 9 (progress: 1/8) 31 | 10/12/05 01:47:05 INFO SimpleJob: Starting task 1:4 as TID 12 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 32 | 10/12/05 01:47:06 INFO SimpleJob: Finished TID 11 (progress: 2/8) 33 | 10/12/05 01:47:06 INFO SimpleJob: Starting task 1:5 as TID 13 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 34 | 10/12/05 01:47:09 INFO SimpleJob: Finished TID 10 (progress: 3/8) 35 | 10/12/05 01:47:09 INFO SimpleJob: Starting task 1:7 as TID 14 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 36 | 10/12/05 01:47:09 INFO SimpleJob: Finished TID 8 (progress: 4/8) 37 | 10/12/05 01:47:11 INFO SimpleJob: Finished TID 13 (progress: 5/8) 38 | 10/12/05 01:47:11 INFO SimpleJob: Starting task 1:6 as TID 15 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 39 | 10/12/05 01:47:11 INFO SimpleJob: Finished TID 14 (progress: 6/8) 40 | 10/12/05 01:47:15 INFO SimpleJob: Finished TID 12 (progress: 7/8) 41 | 10/12/05 01:47:17 INFO SimpleJob: Finished TID 15 (progress: 8/8) 42 | 10/12/05 01:47:17 INFO SparkContext: Tasks finished in 16.681410464 s 43 | Centroids changed by 44 | (1.020119, 0.433392, 2.183411) 45 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 46 | 10/12/05 01:47:17 INFO SparkContext: Running 8 tasks in parallel 47 | 10/12/05 01:47:17 INFO SimpleJob: Starting task 2:0 as TID 16 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 48 | 10/12/05 01:47:17 INFO SimpleJob: Starting task 2:1 as TID 17 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 49 | 10/12/05 01:47:17 INFO SimpleJob: Starting task 2:2 as TID 18 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 50 | 10/12/05 01:47:17 INFO SimpleJob: Starting task 2:3 as TID 19 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 51 | 10/12/05 01:47:21 INFO SimpleJob: Finished TID 19 (progress: 1/8) 52 | 10/12/05 01:47:21 INFO SimpleJob: Starting task 2:4 as TID 20 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 53 | 10/12/05 01:47:21 INFO SimpleJob: Finished TID 17 (progress: 2/8) 54 | 10/12/05 01:47:22 INFO SimpleJob: Starting task 2:5 as TID 21 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 55 | 10/12/05 01:47:25 INFO SimpleJob: Finished TID 18 (progress: 3/8) 56 | 10/12/05 01:47:25 INFO SimpleJob: Starting task 2:7 as TID 22 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 57 | 10/12/05 01:47:26 INFO SimpleJob: Finished TID 16 (progress: 4/8) 58 | 10/12/05 01:47:26 INFO SimpleJob: Finished TID 21 (progress: 5/8) 59 | 10/12/05 01:47:26 INFO SimpleJob: Starting task 2:6 as TID 23 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 60 | 10/12/05 01:47:28 INFO SimpleJob: Finished TID 22 (progress: 6/8) 61 | 10/12/05 01:47:31 INFO SimpleJob: Finished TID 20 (progress: 7/8) 62 | 10/12/05 01:47:34 INFO SimpleJob: Finished TID 23 (progress: 8/8) 63 | 10/12/05 01:47:34 INFO SparkContext: Tasks finished in 16.234956396 s 64 | Centroids changed by 65 | (0.000000, 0.000000, 0.000000) 66 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 67 | Final centroids: 68 | 7.776982 40.661016 69 | 44.751415 35.355014 70 | 17.872132 7.510652 71 | 72 | -------------------------------------------------------------------------------- /run-dataset-20101205T014106/2-2.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1205 01:47:35.789819 5013 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/05 01:47:35 INFO MesosScheduler: Registered as framework ID 201012050147-0-0033 4 | 10/12/05 01:47:35 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/05 01:47:36 INFO FileInputFormat: Total input paths to process : 4 6 | 10/12/05 01:47:36 INFO SparkContext: Running 8 tasks in parallel 7 | 10/12/05 01:47:36 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/05 01:47:36 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/05 01:47:36 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/05 01:47:36 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/05 01:47:51 INFO SimpleJob: Finished TID 3 (progress: 1/8) 12 | 10/12/05 01:47:51 INFO SimpleJob: Starting task 0:4 as TID 4 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 13 | 10/12/05 01:47:51 INFO SimpleJob: Finished TID 1 (progress: 2/8) 14 | 10/12/05 01:47:51 INFO SimpleJob: Starting task 0:5 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 15 | 10/12/05 01:47:52 INFO SimpleJob: Finished TID 5 (progress: 3/8) 16 | 10/12/05 01:47:52 INFO SimpleJob: Starting task 0:6 as TID 6 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 17 | 10/12/05 01:48:13 INFO SimpleJob: Finished TID 0 (progress: 4/8) 18 | 10/12/05 01:48:13 INFO SimpleJob: Starting task 0:7 as TID 7 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/05 01:48:13 INFO SimpleJob: Finished TID 2 (progress: 5/8) 20 | 10/12/05 01:48:17 INFO SimpleJob: Finished TID 6 (progress: 6/8) 21 | 10/12/05 01:48:17 INFO SimpleJob: Finished TID 4 (progress: 7/8) 22 | 10/12/05 01:48:26 INFO SimpleJob: Finished TID 7 (progress: 8/8) 23 | 10/12/05 01:48:26 INFO SparkContext: Tasks finished in 49.766566579 s 24 | Read 40000000 points. 25 | 10/12/05 01:48:26 INFO SparkContext: Running 8 tasks in parallel 26 | 10/12/05 01:48:26 INFO SimpleJob: Starting task 1:0 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 27 | 10/12/05 01:48:26 INFO SimpleJob: Starting task 1:1 as TID 9 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 28 | 10/12/05 01:48:26 INFO SimpleJob: Starting task 1:2 as TID 10 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 29 | 10/12/05 01:48:26 INFO SimpleJob: Starting task 1:3 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 30 | 10/12/05 01:48:30 INFO SimpleJob: Finished TID 9 (progress: 1/8) 31 | 10/12/05 01:48:30 INFO SimpleJob: Starting task 1:4 as TID 12 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 32 | 10/12/05 01:48:30 INFO SimpleJob: Finished TID 11 (progress: 2/8) 33 | 10/12/05 01:48:30 INFO SimpleJob: Starting task 1:5 as TID 13 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 34 | 10/12/05 01:48:34 INFO SimpleJob: Finished TID 13 (progress: 3/8) 35 | 10/12/05 01:48:34 INFO SimpleJob: Starting task 1:6 as TID 14 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 36 | 10/12/05 01:48:34 INFO SimpleJob: Finished TID 10 (progress: 4/8) 37 | 10/12/05 01:48:34 INFO SimpleJob: Starting task 1:7 as TID 15 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 38 | 10/12/05 01:48:34 INFO SimpleJob: Finished TID 8 (progress: 5/8) 39 | 10/12/05 01:48:37 INFO SimpleJob: Finished TID 15 (progress: 6/8) 40 | 10/12/05 01:48:38 INFO SimpleJob: Finished TID 12 (progress: 7/8) 41 | 10/12/05 01:48:40 INFO SimpleJob: Finished TID 14 (progress: 8/8) 42 | 10/12/05 01:48:40 INFO SparkContext: Tasks finished in 14.576628547 s 43 | Centroids changed by 44 | (1.020119, 0.433392, 2.183411) 45 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 46 | 10/12/05 01:48:40 INFO SparkContext: Running 8 tasks in parallel 47 | 10/12/05 01:48:40 INFO SimpleJob: Starting task 2:0 as TID 16 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 48 | 10/12/05 01:48:40 INFO SimpleJob: Starting task 2:1 as TID 17 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 49 | 10/12/05 01:48:40 INFO SimpleJob: Starting task 2:2 as TID 18 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 50 | 10/12/05 01:48:40 INFO SimpleJob: Starting task 2:3 as TID 19 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 51 | 10/12/05 01:48:44 INFO SimpleJob: Finished TID 17 (progress: 1/8) 52 | 10/12/05 01:48:44 INFO SimpleJob: Starting task 2:4 as TID 20 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 53 | 10/12/05 01:48:45 INFO SimpleJob: Finished TID 19 (progress: 2/8) 54 | 10/12/05 01:48:45 INFO SimpleJob: Starting task 2:5 as TID 21 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 55 | 10/12/05 01:48:48 INFO SimpleJob: Finished TID 16 (progress: 3/8) 56 | 10/12/05 01:48:48 INFO SimpleJob: Starting task 2:7 as TID 22 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 57 | 10/12/05 01:48:48 INFO SimpleJob: Finished TID 18 (progress: 4/8) 58 | 10/12/05 01:48:49 INFO SimpleJob: Finished TID 21 (progress: 5/8) 59 | 10/12/05 01:48:49 INFO SimpleJob: Starting task 2:6 as TID 23 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 60 | 10/12/05 01:48:51 INFO SimpleJob: Finished TID 22 (progress: 6/8) 61 | 10/12/05 01:48:53 INFO SimpleJob: Finished TID 20 (progress: 7/8) 62 | 10/12/05 01:48:55 INFO SimpleJob: Finished TID 23 (progress: 8/8) 63 | 10/12/05 01:48:55 INFO SparkContext: Tasks finished in 14.785649045 s 64 | Centroids changed by 65 | (0.000000, 0.000000, 0.000000) 66 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 67 | Final centroids: 68 | 7.776982 40.661016 69 | 44.751415 35.355014 70 | 17.872132 7.510652 71 | 72 | -------------------------------------------------------------------------------- /run-dataset-20101205T014106/2-3.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1205 01:48:57.147780 5055 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/05 01:48:57 INFO MesosScheduler: Registered as framework ID 201012050148-0-0034 4 | 10/12/05 01:48:57 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/05 01:48:57 INFO FileInputFormat: Total input paths to process : 4 6 | 10/12/05 01:48:57 INFO SparkContext: Running 8 tasks in parallel 7 | 10/12/05 01:48:57 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/05 01:48:57 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/05 01:48:57 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/05 01:48:57 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/05 01:49:12 INFO SimpleJob: Finished TID 3 (progress: 1/8) 12 | 10/12/05 01:49:12 INFO SimpleJob: Starting task 0:4 as TID 4 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 13 | 10/12/05 01:49:12 INFO SimpleJob: Finished TID 1 (progress: 2/8) 14 | 10/12/05 01:49:12 INFO SimpleJob: Starting task 0:5 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 15 | 10/12/05 01:49:13 INFO SimpleJob: Finished TID 5 (progress: 3/8) 16 | 10/12/05 01:49:13 INFO SimpleJob: Starting task 0:6 as TID 6 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 17 | 10/12/05 01:49:33 INFO SimpleJob: Finished TID 0 (progress: 4/8) 18 | 10/12/05 01:49:33 INFO SimpleJob: Starting task 0:7 as TID 7 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/05 01:49:33 INFO SimpleJob: Finished TID 2 (progress: 5/8) 20 | 10/12/05 01:49:38 INFO SimpleJob: Finished TID 4 (progress: 6/8) 21 | 10/12/05 01:49:38 INFO SimpleJob: Finished TID 6 (progress: 7/8) 22 | 10/12/05 01:49:45 INFO SimpleJob: Finished TID 7 (progress: 8/8) 23 | 10/12/05 01:49:45 INFO SparkContext: Tasks finished in 47.877277636 s 24 | Read 40000000 points. 25 | 10/12/05 01:49:45 INFO SparkContext: Running 8 tasks in parallel 26 | 10/12/05 01:49:45 INFO SimpleJob: Starting task 1:0 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 27 | 10/12/05 01:49:45 INFO SimpleJob: Starting task 1:1 as TID 9 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 28 | 10/12/05 01:49:45 INFO SimpleJob: Starting task 1:2 as TID 10 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 29 | 10/12/05 01:49:45 INFO SimpleJob: Starting task 1:3 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 30 | 10/12/05 01:49:50 INFO SimpleJob: Finished TID 11 (progress: 1/8) 31 | 10/12/05 01:49:50 INFO SimpleJob: Starting task 1:4 as TID 12 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 32 | 10/12/05 01:49:50 INFO SimpleJob: Finished TID 9 (progress: 2/8) 33 | 10/12/05 01:49:50 INFO SimpleJob: Starting task 1:5 as TID 13 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 34 | 10/12/05 01:49:53 INFO SimpleJob: Finished TID 8 (progress: 3/8) 35 | 10/12/05 01:49:53 INFO SimpleJob: Starting task 1:7 as TID 14 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 36 | 10/12/05 01:49:54 INFO SimpleJob: Finished TID 10 (progress: 4/8) 37 | 10/12/05 01:49:54 INFO SimpleJob: Finished TID 13 (progress: 5/8) 38 | 10/12/05 01:49:54 INFO SimpleJob: Starting task 1:6 as TID 15 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 39 | 10/12/05 01:49:56 INFO SimpleJob: Finished TID 14 (progress: 6/8) 40 | 10/12/05 01:49:58 INFO SimpleJob: Finished TID 12 (progress: 7/8) 41 | 10/12/05 01:50:00 INFO SimpleJob: Finished TID 15 (progress: 8/8) 42 | 10/12/05 01:50:00 INFO SparkContext: Tasks finished in 14.852692258 s 43 | Centroids changed by 44 | (1.020119, 0.433392, 2.183411) 45 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 46 | 10/12/05 01:50:00 INFO SparkContext: Running 8 tasks in parallel 47 | 10/12/05 01:50:00 INFO SimpleJob: Starting task 2:0 as TID 16 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 48 | 10/12/05 01:50:00 INFO SimpleJob: Starting task 2:1 as TID 17 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 49 | 10/12/05 01:50:00 INFO SimpleJob: Starting task 2:2 as TID 18 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 50 | 10/12/05 01:50:00 INFO SimpleJob: Starting task 2:3 as TID 19 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 51 | 10/12/05 01:50:04 INFO SimpleJob: Finished TID 19 (progress: 1/8) 52 | 10/12/05 01:50:04 INFO SimpleJob: Starting task 2:4 as TID 20 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 53 | 10/12/05 01:50:04 INFO SimpleJob: Finished TID 17 (progress: 2/8) 54 | 10/12/05 01:50:04 INFO SimpleJob: Starting task 2:5 as TID 21 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 55 | 10/12/05 01:50:08 INFO SimpleJob: Finished TID 21 (progress: 3/8) 56 | 10/12/05 01:50:08 INFO SimpleJob: Starting task 2:6 as TID 22 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 57 | 10/12/05 01:50:08 INFO SimpleJob: Finished TID 16 (progress: 4/8) 58 | 10/12/05 01:50:08 INFO SimpleJob: Starting task 2:7 as TID 23 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 59 | 10/12/05 01:50:08 INFO SimpleJob: Finished TID 18 (progress: 5/8) 60 | 10/12/05 01:50:11 INFO SimpleJob: Finished TID 23 (progress: 6/8) 61 | 10/12/05 01:50:12 INFO SimpleJob: Finished TID 20 (progress: 7/8) 62 | 10/12/05 01:50:14 INFO SimpleJob: Finished TID 22 (progress: 8/8) 63 | 10/12/05 01:50:14 INFO SparkContext: Tasks finished in 14.376864794 s 64 | Centroids changed by 65 | (0.000000, 0.000000, 0.000000) 66 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 67 | Final centroids: 68 | 7.776982 40.661016 69 | 44.751415 35.355014 70 | 17.872132 7.510652 71 | 72 | -------------------------------------------------------------------------------- /run-scaleup-20101206T184139/1-1.log: -------------------------------------------------------------------------------- 1 | stdin: is not a tty 2 | WARNING: Logging before InitGoogleLogging() is written to STDERR 3 | I1207 04:12:16.565948 2964 sched.cpp:181] New master at 1@10.101.18.77:5050 with ID:0 4 | 10/12/07 04:12:16 INFO spark.MesosScheduler: Registered as framework ID 201012070412-0-0000 5 | 10/12/07 04:12:16 INFO spark.BoundedMemoryCache: BoundedMemoryCache.maxBytes = 7203028992 6 | 10/12/07 04:12:17 INFO mapred.FileInputFormat: Total input paths to process : 1 7 | 10/12/07 04:12:17 INFO spark.SparkContext: Running 15 tasks in parallel 8 | 10/12/07 04:12:17 INFO spark.SimpleJob: Starting task 0:0 as TID 0 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 9 | 10/12/07 04:12:17 INFO spark.SimpleJob: Starting task 0:1 as TID 1 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 10 | 10/12/07 04:12:17 INFO spark.SimpleJob: Starting task 0:2 as TID 2 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 11 | 10/12/07 04:12:17 INFO spark.SimpleJob: Starting task 0:3 as TID 3 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 12 | 10/12/07 04:12:17 INFO spark.SimpleJob: Starting task 0:4 as TID 4 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 13 | 10/12/07 04:12:17 INFO spark.SimpleJob: Starting task 0:5 as TID 5 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 14 | 10/12/07 04:12:17 INFO spark.SimpleJob: Starting task 0:6 as TID 6 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 15 | 10/12/07 04:12:17 INFO spark.SimpleJob: Starting task 0:7 as TID 7 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 16 | 10/12/07 04:13:00 INFO spark.SimpleJob: Finished TID 1 (progress: 1/15) 17 | 10/12/07 04:13:00 INFO spark.SimpleJob: Starting task 0:8 as TID 8 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 18 | 10/12/07 04:13:01 INFO spark.SimpleJob: Finished TID 7 (progress: 2/15) 19 | 10/12/07 04:13:01 INFO spark.SimpleJob: Starting task 0:9 as TID 9 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 20 | 10/12/07 04:13:01 INFO spark.SimpleJob: Finished TID 3 (progress: 3/15) 21 | 10/12/07 04:13:01 INFO spark.SimpleJob: Starting task 0:10 as TID 10 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 22 | 10/12/07 04:13:02 INFO spark.SimpleJob: Finished TID 5 (progress: 4/15) 23 | 10/12/07 04:13:02 INFO spark.SimpleJob: Starting task 0:11 as TID 11 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 24 | 10/12/07 04:13:04 INFO spark.SimpleJob: Finished TID 4 (progress: 5/15) 25 | 10/12/07 04:13:04 INFO spark.SimpleJob: Starting task 0:12 as TID 12 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 26 | 10/12/07 04:13:04 INFO spark.SimpleJob: Finished TID 2 (progress: 6/15) 27 | 10/12/07 04:13:04 INFO spark.SimpleJob: Starting task 0:13 as TID 13 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 28 | 10/12/07 04:13:04 INFO spark.SimpleJob: Finished TID 6 (progress: 7/15) 29 | 10/12/07 04:13:04 INFO spark.SimpleJob: Starting task 0:14 as TID 14 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 30 | 10/12/07 04:13:04 INFO spark.SimpleJob: Finished TID 0 (progress: 8/15) 31 | 10/12/07 04:13:16 INFO spark.SimpleJob: Finished TID 14 (progress: 9/15) 32 | 10/12/07 04:13:31 INFO spark.SimpleJob: Finished TID 12 (progress: 10/15) 33 | 10/12/07 04:13:32 INFO spark.SimpleJob: Finished TID 13 (progress: 11/15) 34 | 10/12/07 04:13:33 INFO spark.SimpleJob: Finished TID 9 (progress: 12/15) 35 | 10/12/07 04:13:34 INFO spark.SimpleJob: Finished TID 8 (progress: 13/15) 36 | 10/12/07 04:13:34 INFO spark.SimpleJob: Finished TID 11 (progress: 14/15) 37 | 10/12/07 04:13:34 INFO spark.SimpleJob: Finished TID 10 (progress: 15/15) 38 | 10/12/07 04:13:34 INFO spark.SparkContext: Tasks finished in 77.282055666 s 39 | Read 100000000 points. 40 | 10/12/07 04:13:34 INFO spark.SparkContext: Running 15 tasks in parallel 41 | 10/12/07 04:13:34 INFO spark.SimpleJob: Starting task 1:0 as TID 15 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 42 | 10/12/07 04:13:34 INFO spark.SimpleJob: Starting task 1:1 as TID 16 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 43 | 10/12/07 04:13:34 INFO spark.SimpleJob: Starting task 1:2 as TID 17 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 44 | 10/12/07 04:13:34 INFO spark.SimpleJob: Starting task 1:3 as TID 18 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 45 | 10/12/07 04:13:34 INFO spark.SimpleJob: Starting task 1:4 as TID 19 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 46 | 10/12/07 04:13:34 INFO spark.SimpleJob: Starting task 1:5 as TID 20 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 47 | 10/12/07 04:13:34 INFO spark.SimpleJob: Starting task 1:6 as TID 21 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 48 | 10/12/07 04:13:34 INFO spark.SimpleJob: Starting task 1:7 as TID 22 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 49 | 10/12/07 04:13:40 INFO spark.SimpleJob: Finished TID 17 (progress: 1/15) 50 | 10/12/07 04:13:40 INFO spark.SimpleJob: Starting task 1:12 as TID 23 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 51 | 10/12/07 04:13:40 INFO spark.SimpleJob: Finished TID 19 (progress: 2/15) 52 | 10/12/07 04:13:40 INFO spark.SimpleJob: Starting task 1:13 as TID 24 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 53 | 10/12/07 04:13:42 INFO spark.SimpleJob: Finished TID 16 (progress: 3/15) 54 | 10/12/07 04:13:42 INFO spark.SimpleJob: Starting task 1:8 as TID 25 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 55 | 10/12/07 04:13:42 INFO spark.SimpleJob: Finished TID 21 (progress: 4/15) 56 | 10/12/07 04:13:42 INFO spark.SimpleJob: Starting task 1:14 as TID 26 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 57 | 10/12/07 04:13:42 INFO spark.SimpleJob: Finished TID 15 (progress: 5/15) 58 | 10/12/07 04:13:42 INFO spark.SimpleJob: Finished TID 20 (progress: 6/15) 59 | 10/12/07 04:13:42 INFO spark.SimpleJob: Starting task 1:9 as TID 27 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 60 | 10/12/07 04:13:42 INFO spark.SimpleJob: Finished TID 18 (progress: 7/15) 61 | 10/12/07 04:13:42 INFO spark.SimpleJob: Starting task 1:10 as TID 28 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 62 | 10/12/07 04:13:42 INFO spark.SimpleJob: Finished TID 22 (progress: 8/15) 63 | 10/12/07 04:13:42 INFO spark.SimpleJob: Starting task 1:11 as TID 29 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 64 | 10/12/07 04:13:44 INFO spark.SimpleJob: Finished TID 26 (progress: 9/15) 65 | 10/12/07 04:13:45 INFO spark.SimpleJob: Finished TID 24 (progress: 10/15) 66 | 10/12/07 04:13:46 INFO spark.SimpleJob: Finished TID 23 (progress: 11/15) 67 | 10/12/07 04:13:49 INFO spark.SimpleJob: Finished TID 25 (progress: 12/15) 68 | 10/12/07 04:13:49 INFO spark.SimpleJob: Finished TID 29 (progress: 13/15) 69 | 10/12/07 04:13:49 INFO spark.SimpleJob: Finished TID 28 (progress: 14/15) 70 | 10/12/07 04:13:49 INFO spark.SimpleJob: Finished TID 27 (progress: 15/15) 71 | 10/12/07 04:13:49 INFO spark.SparkContext: Tasks finished in 15.121590053 s 72 | Centroids changed by 73 | (0.000000, 0.000000, 3.974912) 74 | to ((7.00,40.00), (45.00,35.00), (23.82,6.89)) 75 | 10/12/07 04:13:49 INFO spark.SparkContext: Running 15 tasks in parallel 76 | 10/12/07 04:13:49 INFO spark.SimpleJob: Starting task 2:0 as TID 30 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 77 | 10/12/07 04:13:49 INFO spark.SimpleJob: Starting task 2:1 as TID 31 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 78 | 10/12/07 04:13:49 INFO spark.SimpleJob: Starting task 2:2 as TID 32 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 79 | 10/12/07 04:13:49 INFO spark.SimpleJob: Starting task 2:3 as TID 33 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 80 | 10/12/07 04:13:49 INFO spark.SimpleJob: Starting task 2:4 as TID 34 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 81 | 10/12/07 04:13:49 INFO spark.SimpleJob: Starting task 2:5 as TID 35 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 82 | 10/12/07 04:13:49 INFO spark.SimpleJob: Starting task 2:6 as TID 36 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 83 | 10/12/07 04:13:49 INFO spark.SimpleJob: Starting task 2:7 as TID 37 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 84 | 10/12/07 04:13:56 INFO spark.SimpleJob: Finished TID 30 (progress: 1/15) 85 | 10/12/07 04:13:56 INFO spark.SimpleJob: Starting task 2:12 as TID 38 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 86 | 10/12/07 04:13:56 INFO spark.SimpleJob: Finished TID 32 (progress: 2/15) 87 | 10/12/07 04:13:56 INFO spark.SimpleJob: Starting task 2:13 as TID 39 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 88 | 10/12/07 04:13:56 INFO spark.SimpleJob: Finished TID 34 (progress: 3/15) 89 | 10/12/07 04:13:56 INFO spark.SimpleJob: Starting task 2:14 as TID 40 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 90 | 10/12/07 04:13:56 INFO spark.SimpleJob: Finished TID 36 (progress: 4/15) 91 | 10/12/07 04:13:57 INFO spark.SimpleJob: Finished TID 35 (progress: 5/15) 92 | 10/12/07 04:13:57 INFO spark.SimpleJob: Starting task 2:8 as TID 41 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 93 | 10/12/07 04:13:57 INFO spark.SimpleJob: Finished TID 31 (progress: 6/15) 94 | 10/12/07 04:13:57 INFO spark.SimpleJob: Starting task 2:9 as TID 42 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 95 | 10/12/07 04:13:57 INFO spark.SimpleJob: Finished TID 37 (progress: 7/15) 96 | 10/12/07 04:13:57 INFO spark.SimpleJob: Starting task 2:10 as TID 43 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 97 | 10/12/07 04:13:58 INFO spark.SimpleJob: Finished TID 33 (progress: 8/15) 98 | 10/12/07 04:13:58 INFO spark.SimpleJob: Starting task 2:11 as TID 44 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 99 | 10/12/07 04:13:59 INFO spark.SimpleJob: Finished TID 40 (progress: 9/15) 100 | 10/12/07 04:14:01 INFO spark.SimpleJob: Finished TID 38 (progress: 10/15) 101 | 10/12/07 04:14:01 INFO spark.SimpleJob: Finished TID 39 (progress: 11/15) 102 | 10/12/07 04:14:04 INFO spark.SimpleJob: Finished TID 41 (progress: 12/15) 103 | 10/12/07 04:14:04 INFO spark.SimpleJob: Finished TID 43 (progress: 13/15) 104 | 10/12/07 04:14:05 INFO spark.SimpleJob: Finished TID 42 (progress: 14/15) 105 | 10/12/07 04:14:05 INFO spark.SimpleJob: Finished TID 44 (progress: 15/15) 106 | 10/12/07 04:14:05 INFO spark.SparkContext: Tasks finished in 16.098885003 s 107 | Centroids changed by 108 | (0.000000, 0.000000, 0.000000) 109 | to ((7.00,40.00), (45.00,35.00), (23.82,6.89))7.000000 40.000000 110 | 45.000000 35.000000 111 | 23.815831 6.886736 112 | 113 | 114 | Final centroids: 115 | -------------------------------------------------------------------------------- /run-scaleup-20101206T184139/1-2.log: -------------------------------------------------------------------------------- 1 | stdin: is not a tty 2 | WARNING: Logging before InitGoogleLogging() is written to STDERR 3 | I1207 04:14:08.798882 3067 sched.cpp:181] New master at 1@10.101.18.77:5050 with ID:0 4 | 10/12/07 04:14:08 INFO spark.MesosScheduler: Registered as framework ID 201012070414-0-0001 5 | 10/12/07 04:14:08 INFO spark.BoundedMemoryCache: BoundedMemoryCache.maxBytes = 7203028992 6 | 10/12/07 04:14:09 INFO mapred.FileInputFormat: Total input paths to process : 1 7 | 10/12/07 04:14:09 INFO spark.SparkContext: Running 15 tasks in parallel 8 | 10/12/07 04:14:09 INFO spark.SimpleJob: Starting task 0:0 as TID 0 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 9 | 10/12/07 04:14:09 INFO spark.SimpleJob: Starting task 0:1 as TID 1 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 10 | 10/12/07 04:14:09 INFO spark.SimpleJob: Starting task 0:2 as TID 2 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 11 | 10/12/07 04:14:09 INFO spark.SimpleJob: Starting task 0:3 as TID 3 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 12 | 10/12/07 04:14:09 INFO spark.SimpleJob: Starting task 0:4 as TID 4 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 13 | 10/12/07 04:14:09 INFO spark.SimpleJob: Starting task 0:5 as TID 5 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 14 | 10/12/07 04:14:09 INFO spark.SimpleJob: Starting task 0:6 as TID 6 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 15 | 10/12/07 04:14:09 INFO spark.SimpleJob: Starting task 0:7 as TID 7 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 16 | 10/12/07 04:14:44 INFO spark.SimpleJob: Finished TID 3 (progress: 1/15) 17 | 10/12/07 04:14:44 INFO spark.SimpleJob: Starting task 0:8 as TID 8 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 18 | 10/12/07 04:14:44 INFO spark.SimpleJob: Finished TID 5 (progress: 2/15) 19 | 10/12/07 04:14:44 INFO spark.SimpleJob: Starting task 0:9 as TID 9 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 20 | 10/12/07 04:14:44 INFO spark.SimpleJob: Finished TID 7 (progress: 3/15) 21 | 10/12/07 04:14:44 INFO spark.SimpleJob: Starting task 0:10 as TID 10 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 22 | 10/12/07 04:14:44 INFO spark.SimpleJob: Finished TID 1 (progress: 4/15) 23 | 10/12/07 04:14:44 INFO spark.SimpleJob: Starting task 0:11 as TID 11 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 24 | 10/12/07 04:14:44 INFO spark.SimpleJob: Finished TID 0 (progress: 5/15) 25 | 10/12/07 04:14:44 INFO spark.SimpleJob: Starting task 0:12 as TID 12 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 26 | 10/12/07 04:14:44 INFO spark.SimpleJob: Finished TID 6 (progress: 6/15) 27 | 10/12/07 04:14:44 INFO spark.SimpleJob: Starting task 0:13 as TID 13 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 28 | 10/12/07 04:14:44 INFO spark.SimpleJob: Finished TID 4 (progress: 7/15) 29 | 10/12/07 04:14:44 INFO spark.SimpleJob: Starting task 0:14 as TID 14 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 30 | 10/12/07 04:14:44 INFO spark.SimpleJob: Finished TID 2 (progress: 8/15) 31 | 10/12/07 04:14:56 INFO spark.SimpleJob: Finished TID 14 (progress: 9/15) 32 | 10/12/07 04:15:09 INFO spark.SimpleJob: Finished TID 12 (progress: 10/15) 33 | 10/12/07 04:15:11 INFO spark.SimpleJob: Finished TID 13 (progress: 11/15) 34 | 10/12/07 04:15:15 INFO spark.SimpleJob: Finished TID 11 (progress: 12/15) 35 | 10/12/07 04:15:16 INFO spark.SimpleJob: Finished TID 8 (progress: 13/15) 36 | 10/12/07 04:15:16 INFO spark.SimpleJob: Finished TID 9 (progress: 14/15) 37 | 10/12/07 04:15:17 INFO spark.SimpleJob: Finished TID 10 (progress: 15/15) 38 | 10/12/07 04:15:17 INFO spark.SparkContext: Tasks finished in 67.725547257 s 39 | Read 100000000 points. 40 | 10/12/07 04:15:17 INFO spark.SparkContext: Running 15 tasks in parallel 41 | 10/12/07 04:15:17 INFO spark.SimpleJob: Starting task 1:0 as TID 15 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 42 | 10/12/07 04:15:17 INFO spark.SimpleJob: Starting task 1:1 as TID 16 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 43 | 10/12/07 04:15:17 INFO spark.SimpleJob: Starting task 1:2 as TID 17 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 44 | 10/12/07 04:15:17 INFO spark.SimpleJob: Starting task 1:3 as TID 18 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 45 | 10/12/07 04:15:17 INFO spark.SimpleJob: Starting task 1:4 as TID 19 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 46 | 10/12/07 04:15:17 INFO spark.SimpleJob: Starting task 1:5 as TID 20 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 47 | 10/12/07 04:15:17 INFO spark.SimpleJob: Starting task 1:6 as TID 21 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 48 | 10/12/07 04:15:17 INFO spark.SimpleJob: Starting task 1:7 as TID 22 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 49 | 10/12/07 04:15:23 INFO spark.SimpleJob: Finished TID 15 (progress: 1/15) 50 | 10/12/07 04:15:23 INFO spark.SimpleJob: Starting task 1:12 as TID 23 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 51 | 10/12/07 04:15:23 INFO spark.SimpleJob: Finished TID 19 (progress: 2/15) 52 | 10/12/07 04:15:23 INFO spark.SimpleJob: Starting task 1:13 as TID 24 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 53 | 10/12/07 04:15:23 INFO spark.SimpleJob: Finished TID 17 (progress: 3/15) 54 | 10/12/07 04:15:23 INFO spark.SimpleJob: Starting task 1:14 as TID 25 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 55 | 10/12/07 04:15:23 INFO spark.SimpleJob: Finished TID 21 (progress: 4/15) 56 | 10/12/07 04:15:24 INFO spark.SimpleJob: Finished TID 18 (progress: 5/15) 57 | 10/12/07 04:15:24 INFO spark.SimpleJob: Starting task 1:8 as TID 26 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 58 | 10/12/07 04:15:24 INFO spark.SimpleJob: Finished TID 16 (progress: 6/15) 59 | 10/12/07 04:15:24 INFO spark.SimpleJob: Starting task 1:9 as TID 27 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 60 | 10/12/07 04:15:24 INFO spark.SimpleJob: Finished TID 22 (progress: 7/15) 61 | 10/12/07 04:15:24 INFO spark.SimpleJob: Starting task 1:10 as TID 28 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 62 | 10/12/07 04:15:24 INFO spark.SimpleJob: Finished TID 20 (progress: 8/15) 63 | 10/12/07 04:15:24 INFO spark.SimpleJob: Starting task 1:11 as TID 29 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 64 | 10/12/07 04:15:25 INFO spark.SimpleJob: Finished TID 25 (progress: 9/15) 65 | 10/12/07 04:15:28 INFO spark.SimpleJob: Finished TID 23 (progress: 10/15) 66 | 10/12/07 04:15:28 INFO spark.SimpleJob: Finished TID 24 (progress: 11/15) 67 | 10/12/07 04:15:31 INFO spark.SimpleJob: Finished TID 26 (progress: 12/15) 68 | 10/12/07 04:15:31 INFO spark.SimpleJob: Finished TID 27 (progress: 13/15) 69 | 10/12/07 04:15:31 INFO spark.SimpleJob: Finished TID 28 (progress: 14/15) 70 | 10/12/07 04:15:32 INFO spark.SimpleJob: Finished TID 29 (progress: 15/15) 71 | 10/12/07 04:15:32 INFO spark.SparkContext: Tasks finished in 15.069806022 s 72 | Centroids changed by 73 | (0.000000, 0.000000, 3.974912) 74 | to ((7.00,40.00), (45.00,35.00), (23.82,6.89)) 75 | 10/12/07 04:15:32 INFO spark.SparkContext: Running 15 tasks in parallel 76 | 10/12/07 04:15:32 INFO spark.SimpleJob: Starting task 2:0 as TID 30 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 77 | 10/12/07 04:15:32 INFO spark.SimpleJob: Starting task 2:1 as TID 31 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 78 | 10/12/07 04:15:32 INFO spark.SimpleJob: Starting task 2:2 as TID 32 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 79 | 10/12/07 04:15:32 INFO spark.SimpleJob: Starting task 2:3 as TID 33 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 80 | 10/12/07 04:15:32 INFO spark.SimpleJob: Starting task 2:4 as TID 34 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 81 | 10/12/07 04:15:32 INFO spark.SimpleJob: Starting task 2:5 as TID 35 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 82 | 10/12/07 04:15:32 INFO spark.SimpleJob: Starting task 2:6 as TID 36 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 83 | 10/12/07 04:15:32 INFO spark.SimpleJob: Starting task 2:7 as TID 37 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 84 | 10/12/07 04:15:38 INFO spark.SimpleJob: Finished TID 34 (progress: 1/15) 85 | 10/12/07 04:15:38 INFO spark.SimpleJob: Starting task 2:12 as TID 38 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 86 | 10/12/07 04:15:38 INFO spark.SimpleJob: Finished TID 36 (progress: 2/15) 87 | 10/12/07 04:15:38 INFO spark.SimpleJob: Starting task 2:13 as TID 39 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 88 | 10/12/07 04:15:39 INFO spark.SimpleJob: Finished TID 32 (progress: 3/15) 89 | 10/12/07 04:15:39 INFO spark.SimpleJob: Starting task 2:14 as TID 40 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 90 | 10/12/07 04:15:39 INFO spark.SimpleJob: Finished TID 30 (progress: 4/15) 91 | 10/12/07 04:15:39 INFO spark.SimpleJob: Finished TID 33 (progress: 5/15) 92 | 10/12/07 04:15:39 INFO spark.SimpleJob: Starting task 2:8 as TID 41 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 93 | 10/12/07 04:15:39 INFO spark.SimpleJob: Finished TID 35 (progress: 6/15) 94 | 10/12/07 04:15:39 INFO spark.SimpleJob: Starting task 2:9 as TID 42 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 95 | 10/12/07 04:15:39 INFO spark.SimpleJob: Finished TID 31 (progress: 7/15) 96 | 10/12/07 04:15:39 INFO spark.SimpleJob: Starting task 2:10 as TID 43 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 97 | 10/12/07 04:15:39 INFO spark.SimpleJob: Finished TID 37 (progress: 8/15) 98 | 10/12/07 04:15:39 INFO spark.SimpleJob: Starting task 2:11 as TID 44 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 99 | 10/12/07 04:15:41 INFO spark.SimpleJob: Finished TID 40 (progress: 9/15) 100 | 10/12/07 04:15:44 INFO spark.SimpleJob: Finished TID 38 (progress: 10/15) 101 | 10/12/07 04:15:44 INFO spark.SimpleJob: Finished TID 39 (progress: 11/15) 102 | 10/12/07 04:15:47 INFO spark.SimpleJob: Finished TID 42 (progress: 12/15) 103 | 10/12/07 04:15:47 INFO spark.SimpleJob: Finished TID 41 (progress: 13/15) 104 | 10/12/07 04:15:47 INFO spark.SimpleJob: Finished TID 43 (progress: 14/15) 105 | 10/12/07 04:15:47 INFO spark.SimpleJob: Finished TID 44 (progress: 15/15) 106 | 10/12/07 04:15:47 INFO spark.SparkContext: Tasks finished in 15.40925161 s 107 | Centroids changed by 108 | (0.000000, 0.000000, 0.000000) 109 | to ((7.00,40.00), (45.00,35.00), (23.82,6.89)) 110 | Final centroids: 7.000000 40.000000 111 | 45.000000 35.000000 112 | 23.815831 6.886736 113 | 114 | 115 | -------------------------------------------------------------------------------- /run-scaleup-20101206T184139/1-3.log: -------------------------------------------------------------------------------- 1 | stdin: is not a tty 2 | WARNING: Logging before InitGoogleLogging() is written to STDERR 3 | I1207 04:15:50.732949 3172 sched.cpp:181] New master at 1@10.101.18.77:5050 with ID:0 4 | 10/12/07 04:15:50 INFO spark.MesosScheduler: Registered as framework ID 201012070415-0-0002 5 | 10/12/07 04:15:50 INFO spark.BoundedMemoryCache: BoundedMemoryCache.maxBytes = 7203028992 6 | 10/12/07 04:15:51 INFO mapred.FileInputFormat: Total input paths to process : 1 7 | 10/12/07 04:15:51 INFO spark.SparkContext: Running 15 tasks in parallel 8 | 10/12/07 04:15:51 INFO spark.SimpleJob: Starting task 0:0 as TID 0 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 9 | 10/12/07 04:15:51 INFO spark.SimpleJob: Starting task 0:1 as TID 1 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 10 | 10/12/07 04:15:51 INFO spark.SimpleJob: Starting task 0:2 as TID 2 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 11 | 10/12/07 04:15:51 INFO spark.SimpleJob: Starting task 0:3 as TID 3 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 12 | 10/12/07 04:15:51 INFO spark.SimpleJob: Starting task 0:4 as TID 4 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 13 | 10/12/07 04:15:51 INFO spark.SimpleJob: Starting task 0:5 as TID 5 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 14 | 10/12/07 04:15:51 INFO spark.SimpleJob: Starting task 0:6 as TID 6 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 15 | 10/12/07 04:15:51 INFO spark.SimpleJob: Starting task 0:7 as TID 7 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 16 | 10/12/07 04:16:26 INFO spark.SimpleJob: Finished TID 2 (progress: 1/15) 17 | 10/12/07 04:16:26 INFO spark.SimpleJob: Starting task 0:8 as TID 8 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 18 | 10/12/07 04:16:26 INFO spark.SimpleJob: Finished TID 0 (progress: 2/15) 19 | 10/12/07 04:16:26 INFO spark.SimpleJob: Starting task 0:9 as TID 9 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 20 | 10/12/07 04:16:26 INFO spark.SimpleJob: Finished TID 6 (progress: 3/15) 21 | 10/12/07 04:16:26 INFO spark.SimpleJob: Starting task 0:10 as TID 10 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 22 | 10/12/07 04:16:27 INFO spark.SimpleJob: Finished TID 1 (progress: 4/15) 23 | 10/12/07 04:16:27 INFO spark.SimpleJob: Starting task 0:11 as TID 11 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 24 | 10/12/07 04:16:27 INFO spark.SimpleJob: Finished TID 4 (progress: 5/15) 25 | 10/12/07 04:16:27 INFO spark.SimpleJob: Starting task 0:12 as TID 12 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 26 | 10/12/07 04:16:27 INFO spark.SimpleJob: Finished TID 3 (progress: 6/15) 27 | 10/12/07 04:16:27 INFO spark.SimpleJob: Starting task 0:13 as TID 13 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 28 | 10/12/07 04:16:27 INFO spark.SimpleJob: Finished TID 7 (progress: 7/15) 29 | 10/12/07 04:16:27 INFO spark.SimpleJob: Starting task 0:14 as TID 14 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 30 | 10/12/07 04:16:27 INFO spark.SimpleJob: Finished TID 5 (progress: 8/15) 31 | 10/12/07 04:16:38 INFO spark.SimpleJob: Finished TID 14 (progress: 9/15) 32 | 10/12/07 04:16:52 INFO spark.SimpleJob: Finished TID 11 (progress: 10/15) 33 | 10/12/07 04:16:52 INFO spark.SimpleJob: Finished TID 13 (progress: 11/15) 34 | 10/12/07 04:17:01 INFO spark.SimpleJob: Finished TID 9 (progress: 12/15) 35 | 10/12/07 04:17:02 INFO spark.SimpleJob: Finished TID 8 (progress: 13/15) 36 | 10/12/07 04:17:02 INFO spark.SimpleJob: Finished TID 12 (progress: 14/15) 37 | 10/12/07 04:17:02 INFO spark.SimpleJob: Finished TID 10 (progress: 15/15) 38 | 10/12/07 04:17:02 INFO spark.SparkContext: Tasks finished in 71.756551937 s 39 | Read 100000000 points. 40 | 10/12/07 04:17:03 INFO spark.SparkContext: Running 15 tasks in parallel 41 | 10/12/07 04:17:03 INFO spark.SimpleJob: Starting task 1:0 as TID 15 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 42 | 10/12/07 04:17:03 INFO spark.SimpleJob: Starting task 1:1 as TID 16 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 43 | 10/12/07 04:17:03 INFO spark.SimpleJob: Starting task 1:2 as TID 17 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 44 | 10/12/07 04:17:03 INFO spark.SimpleJob: Starting task 1:3 as TID 18 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 45 | 10/12/07 04:17:03 INFO spark.SimpleJob: Starting task 1:4 as TID 19 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 46 | 10/12/07 04:17:03 INFO spark.SimpleJob: Starting task 1:5 as TID 20 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 47 | 10/12/07 04:17:03 INFO spark.SimpleJob: Starting task 1:6 as TID 21 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 48 | 10/12/07 04:17:03 INFO spark.SimpleJob: Starting task 1:7 as TID 22 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 49 | 10/12/07 04:17:09 INFO spark.SimpleJob: Finished TID 22 (progress: 1/15) 50 | 10/12/07 04:17:09 INFO spark.SimpleJob: Starting task 1:11 as TID 23 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 51 | 10/12/07 04:17:09 INFO spark.SimpleJob: Finished TID 16 (progress: 2/15) 52 | 10/12/07 04:17:09 INFO spark.SimpleJob: Starting task 1:13 as TID 24 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 53 | 10/12/07 04:17:09 INFO spark.SimpleJob: Finished TID 20 (progress: 3/15) 54 | 10/12/07 04:17:09 INFO spark.SimpleJob: Starting task 1:14 as TID 25 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 55 | 10/12/07 04:17:09 INFO spark.SimpleJob: Finished TID 18 (progress: 4/15) 56 | 10/12/07 04:17:11 INFO spark.SimpleJob: Finished TID 17 (progress: 5/15) 57 | 10/12/07 04:17:11 INFO spark.SimpleJob: Starting task 1:8 as TID 26 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 58 | 10/12/07 04:17:11 INFO spark.SimpleJob: Finished TID 21 (progress: 6/15) 59 | 10/12/07 04:17:11 INFO spark.SimpleJob: Starting task 1:9 as TID 27 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 60 | 10/12/07 04:17:11 INFO spark.SimpleJob: Finished TID 25 (progress: 7/15) 61 | 10/12/07 04:17:12 INFO spark.SimpleJob: Finished TID 19 (progress: 8/15) 62 | 10/12/07 04:17:12 INFO spark.SimpleJob: Starting task 1:10 as TID 28 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 63 | 10/12/07 04:17:12 INFO spark.SimpleJob: Finished TID 15 (progress: 9/15) 64 | 10/12/07 04:17:12 INFO spark.SimpleJob: Starting task 1:12 as TID 29 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 65 | 10/12/07 04:17:13 INFO spark.SimpleJob: Finished TID 23 (progress: 10/15) 66 | 10/12/07 04:17:14 INFO spark.SimpleJob: Finished TID 24 (progress: 11/15) 67 | 10/12/07 04:17:19 INFO spark.SimpleJob: Finished TID 26 (progress: 12/15) 68 | 10/12/07 04:17:20 INFO spark.SimpleJob: Finished TID 28 (progress: 13/15) 69 | 10/12/07 04:17:20 INFO spark.SimpleJob: Finished TID 27 (progress: 14/15) 70 | 10/12/07 04:17:20 INFO spark.SimpleJob: Finished TID 29 (progress: 15/15) 71 | 10/12/07 04:17:20 INFO spark.SparkContext: Tasks finished in 17.765848752 s 72 | Centroids changed by 73 | (0.000000, 0.000000, 3.974912) 74 | to ((7.00,40.00), (45.00,35.00), (23.82,6.89)) 75 | 10/12/07 04:17:20 INFO spark.SparkContext: Running 15 tasks in parallel 76 | 10/12/07 04:17:20 INFO spark.SimpleJob: Starting task 2:0 as TID 30 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 77 | 10/12/07 04:17:20 INFO spark.SimpleJob: Starting task 2:1 as TID 31 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 78 | 10/12/07 04:17:20 INFO spark.SimpleJob: Starting task 2:2 as TID 32 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 79 | 10/12/07 04:17:20 INFO spark.SimpleJob: Starting task 2:3 as TID 33 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 80 | 10/12/07 04:17:20 INFO spark.SimpleJob: Starting task 2:4 as TID 34 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 81 | 10/12/07 04:17:20 INFO spark.SimpleJob: Starting task 2:5 as TID 35 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 82 | 10/12/07 04:17:20 INFO spark.SimpleJob: Starting task 2:6 as TID 36 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 83 | 10/12/07 04:17:20 INFO spark.SimpleJob: Starting task 2:7 as TID 37 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 84 | 10/12/07 04:17:27 INFO spark.SimpleJob: Finished TID 33 (progress: 1/15) 85 | 10/12/07 04:17:27 INFO spark.SimpleJob: Starting task 2:11 as TID 38 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 86 | 10/12/07 04:17:27 INFO spark.SimpleJob: Finished TID 37 (progress: 2/15) 87 | 10/12/07 04:17:27 INFO spark.SimpleJob: Starting task 2:13 as TID 39 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 88 | 10/12/07 04:17:28 INFO spark.SimpleJob: Finished TID 35 (progress: 3/15) 89 | 10/12/07 04:17:28 INFO spark.SimpleJob: Starting task 2:14 as TID 40 on slave 0-0: ip-10-101-18-231.ec2.internal (preferred) 90 | 10/12/07 04:17:28 INFO spark.SimpleJob: Finished TID 31 (progress: 4/15) 91 | 10/12/07 04:17:28 INFO spark.SimpleJob: Finished TID 30 (progress: 5/15) 92 | 10/12/07 04:17:28 INFO spark.SimpleJob: Starting task 2:8 as TID 41 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 93 | 10/12/07 04:17:29 INFO spark.SimpleJob: Finished TID 34 (progress: 6/15) 94 | 10/12/07 04:17:29 INFO spark.SimpleJob: Starting task 2:9 as TID 42 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 95 | 10/12/07 04:17:29 INFO spark.SimpleJob: Finished TID 32 (progress: 7/15) 96 | 10/12/07 04:17:29 INFO spark.SimpleJob: Starting task 2:10 as TID 43 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 97 | 10/12/07 04:17:29 INFO spark.SimpleJob: Finished TID 36 (progress: 8/15) 98 | 10/12/07 04:17:29 INFO spark.SimpleJob: Starting task 2:12 as TID 44 on slave 0-1: ip-10-100-226-63.ec2.internal (preferred) 99 | 10/12/07 04:17:30 INFO spark.SimpleJob: Finished TID 40 (progress: 9/15) 100 | 10/12/07 04:17:33 INFO spark.SimpleJob: Finished TID 38 (progress: 10/15) 101 | 10/12/07 04:17:33 INFO spark.SimpleJob: Finished TID 39 (progress: 11/15) 102 | 10/12/07 04:17:36 INFO spark.SimpleJob: Finished TID 41 (progress: 12/15) 103 | 10/12/07 04:17:37 INFO spark.SimpleJob: Finished TID 43 (progress: 13/15) 104 | 10/12/07 04:17:38 INFO spark.SimpleJob: Finished TID 42 (progress: 14/15) 105 | 10/12/07 04:17:38 INFO spark.SimpleJob: Finished TID 44 (progress: 15/15) 106 | 7.000000 40.000000 107 | 45.000000 35.000000 108 | 23.815831 6.886736 109 | 110 | 10/12/07 04:17:38 INFO spark.SparkContext: Tasks finished in 17.781927637 s 111 | Centroids changed by 112 | (0.000000, 0.000000, 0.000000) 113 | to ((7.00,40.00), (45.00,35.00), (23.82,6.89)) 114 | Final centroids: 115 | -------------------------------------------------------------------------------- /run-scaleout-20101205T231425/2-1.log: -------------------------------------------------------------------------------- 1 | stdin: is not a tty 2 | WARNING: Logging before InitGoogleLogging() is written to STDERR 3 | I1206 08:48:41.395102 2644 sched.cpp:181] New master at 1@10.244.130.127:5050 with ID:0 4 | 10/12/06 08:48:41 INFO spark.MesosScheduler: Registered as framework ID 201012060848-0-0000 5 | 10/12/06 08:48:41 INFO spark.BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 6 | 10/12/06 08:48:41 INFO mapred.FileInputFormat: Total input paths to process : 1 7 | 10/12/06 08:48:41 INFO spark.SparkContext: Running 15 tasks in parallel 8 | 10/12/06 08:48:41 INFO spark.SimpleJob: Starting task 0:0 as TID 0 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 9 | 10/12/06 08:48:41 INFO spark.SimpleJob: Starting task 0:2 as TID 1 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 10 | 10/12/06 08:48:41 INFO spark.SimpleJob: Starting task 0:1 as TID 2 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 11 | 10/12/06 08:48:41 INFO spark.SimpleJob: Starting task 0:3 as TID 3 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 12 | 10/12/06 08:48:41 INFO spark.SimpleJob: Starting task 0:4 as TID 4 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 13 | 10/12/06 08:48:41 INFO spark.SimpleJob: Starting task 0:6 as TID 5 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 14 | 10/12/06 08:49:24 INFO spark.SimpleJob: Finished TID 2 (progress: 1/15) 15 | 10/12/06 08:49:24 INFO spark.SimpleJob: Starting task 0:8 as TID 6 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 16 | 10/12/06 08:49:24 INFO spark.SimpleJob: Finished TID 5 (progress: 2/15) 17 | 10/12/06 08:49:24 INFO spark.SimpleJob: Starting task 0:11 as TID 7 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 18 | 10/12/06 08:49:28 INFO spark.SimpleJob: Finished TID 4 (progress: 3/15) 19 | 10/12/06 08:49:28 INFO spark.SimpleJob: Starting task 0:5 as TID 8 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 20 | 10/12/06 08:49:29 INFO spark.SimpleJob: Finished TID 1 (progress: 4/15) 21 | 10/12/06 08:49:29 INFO spark.SimpleJob: Starting task 0:7 as TID 9 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 22 | 10/12/06 08:49:29 INFO spark.SimpleJob: Finished TID 3 (progress: 5/15) 23 | 10/12/06 08:49:29 INFO spark.SimpleJob: Starting task 0:9 as TID 10 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 24 | 10/12/06 08:49:32 INFO spark.SimpleJob: Finished TID 0 (progress: 6/15) 25 | 10/12/06 08:49:32 INFO spark.SimpleJob: Starting task 0:10 as TID 11 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 26 | 10/12/06 08:50:03 INFO spark.SimpleJob: Finished TID 6 (progress: 7/15) 27 | 10/12/06 08:50:03 INFO spark.SimpleJob: Starting task 0:12 as TID 12 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 28 | 10/12/06 08:50:04 INFO spark.SimpleJob: Finished TID 7 (progress: 8/15) 29 | 10/12/06 08:50:04 INFO spark.SimpleJob: Starting task 0:13 as TID 13 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 30 | 10/12/06 08:50:08 INFO spark.SimpleJob: Finished TID 8 (progress: 9/15) 31 | 10/12/06 08:50:08 INFO spark.SimpleJob: Starting task 0:14 as TID 14 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 32 | 10/12/06 08:50:09 INFO spark.SimpleJob: Finished TID 9 (progress: 10/15) 33 | 10/12/06 08:50:13 INFO spark.SimpleJob: Finished TID 10 (progress: 11/15) 34 | 10/12/06 08:50:16 INFO spark.SimpleJob: Finished TID 11 (progress: 12/15) 35 | 10/12/06 08:50:23 INFO spark.SimpleJob: Finished TID 14 (progress: 13/15) 36 | 10/12/06 08:50:57 INFO spark.SimpleJob: Finished TID 12 (progress: 14/15) 37 | 10/12/06 08:50:58 INFO spark.SimpleJob: Finished TID 13 (progress: 15/15) 38 | 10/12/06 08:50:58 INFO spark.SparkContext: Tasks finished in 136.220323941 s 39 | Read 100000000 points. 40 | 10/12/06 08:50:58 INFO spark.SparkContext: Running 15 tasks in parallel 41 | 10/12/06 08:50:58 INFO spark.SimpleJob: Starting task 1:0 as TID 15 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 42 | 10/12/06 08:50:58 INFO spark.SimpleJob: Starting task 1:2 as TID 16 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 43 | 10/12/06 08:50:58 INFO spark.SimpleJob: Starting task 1:1 as TID 17 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 44 | 10/12/06 08:50:58 INFO spark.SimpleJob: Starting task 1:3 as TID 18 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 45 | 10/12/06 08:50:58 INFO spark.SimpleJob: Starting task 1:4 as TID 19 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 46 | 10/12/06 08:50:58 INFO spark.SimpleJob: Starting task 1:6 as TID 20 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 47 | 10/12/06 08:51:06 INFO spark.SimpleJob: Finished TID 17 (progress: 1/15) 48 | 10/12/06 08:51:06 INFO spark.SimpleJob: Starting task 1:8 as TID 21 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 49 | 10/12/06 08:51:06 INFO spark.SimpleJob: Finished TID 20 (progress: 2/15) 50 | 10/12/06 08:51:06 INFO spark.SimpleJob: Starting task 1:11 as TID 22 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 51 | 10/12/06 08:51:08 INFO spark.SimpleJob: Finished TID 19 (progress: 3/15) 52 | 10/12/06 08:51:08 INFO spark.SimpleJob: Starting task 1:5 as TID 23 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 53 | 10/12/06 08:51:08 INFO spark.SimpleJob: Finished TID 16 (progress: 4/15) 54 | 10/12/06 08:51:08 INFO spark.SimpleJob: Starting task 1:7 as TID 24 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 55 | 10/12/06 08:51:10 INFO spark.SimpleJob: Finished TID 18 (progress: 5/15) 56 | 10/12/06 08:51:10 INFO spark.SimpleJob: Starting task 1:9 as TID 25 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 57 | 10/12/06 08:51:11 INFO spark.SimpleJob: Finished TID 15 (progress: 6/15) 58 | 10/12/06 08:51:11 INFO spark.SimpleJob: Starting task 1:10 as TID 26 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 59 | 10/12/06 08:51:14 INFO spark.SimpleJob: Finished TID 22 (progress: 7/15) 60 | 10/12/06 08:51:14 INFO spark.SimpleJob: Starting task 1:12 as TID 27 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 61 | 10/12/06 08:51:14 INFO spark.SimpleJob: Finished TID 21 (progress: 8/15) 62 | 10/12/06 08:51:14 INFO spark.SimpleJob: Starting task 1:13 as TID 28 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 63 | 10/12/06 08:51:18 INFO spark.SimpleJob: Finished TID 24 (progress: 9/15) 64 | 10/12/06 08:51:18 INFO spark.SimpleJob: Starting task 1:14 as TID 29 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 65 | 10/12/06 08:51:18 INFO spark.SimpleJob: Finished TID 23 (progress: 10/15) 66 | 10/12/06 08:51:21 INFO spark.SimpleJob: Finished TID 29 (progress: 11/15) 67 | 10/12/06 08:51:22 INFO spark.SimpleJob: Finished TID 28 (progress: 12/15) 68 | 10/12/06 08:51:22 INFO spark.SimpleJob: Finished TID 27 (progress: 13/15) 69 | 10/12/06 08:51:22 INFO spark.SimpleJob: Finished TID 25 (progress: 14/15) 70 | 10/12/06 08:51:22 INFO spark.SimpleJob: Finished TID 26 (progress: 15/15) 71 | 10/12/06 08:51:22 INFO spark.SparkContext: Tasks finished in 24.670772555 s 72 | Centroids changed by 73 | (2.255668, 26.978290, 27.152732) 74 | to ((6.17,42.10), (48.24,8.22), (47.14,7.02)) 75 | 10/12/06 08:51:22 INFO spark.SparkContext: Running 15 tasks in parallel 76 | 10/12/06 08:51:22 INFO spark.SimpleJob: Starting task 2:0 as TID 30 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 77 | 10/12/06 08:51:22 INFO spark.SimpleJob: Starting task 2:2 as TID 31 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 78 | 10/12/06 08:51:22 INFO spark.SimpleJob: Starting task 2:1 as TID 32 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 79 | 10/12/06 08:51:22 INFO spark.SimpleJob: Starting task 2:3 as TID 33 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 80 | 10/12/06 08:51:22 INFO spark.SimpleJob: Starting task 2:4 as TID 34 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 81 | 10/12/06 08:51:22 INFO spark.SimpleJob: Starting task 2:6 as TID 35 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 82 | 10/12/06 08:51:31 INFO spark.SimpleJob: Finished TID 32 (progress: 1/15) 83 | 10/12/06 08:51:31 INFO spark.SimpleJob: Starting task 2:8 as TID 36 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 84 | 10/12/06 08:51:31 INFO spark.SimpleJob: Finished TID 35 (progress: 2/15) 85 | 10/12/06 08:51:31 INFO spark.SimpleJob: Starting task 2:11 as TID 37 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 86 | 10/12/06 08:51:33 INFO spark.SimpleJob: Finished TID 31 (progress: 3/15) 87 | 10/12/06 08:51:33 INFO spark.SimpleJob: Starting task 2:5 as TID 38 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 88 | 10/12/06 08:51:33 INFO spark.SimpleJob: Finished TID 34 (progress: 4/15) 89 | 10/12/06 08:51:33 INFO spark.SimpleJob: Starting task 2:7 as TID 39 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 90 | 10/12/06 08:51:34 INFO spark.SimpleJob: Finished TID 33 (progress: 5/15) 91 | 10/12/06 08:51:34 INFO spark.SimpleJob: Starting task 2:9 as TID 40 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 92 | 10/12/06 08:51:35 INFO spark.SimpleJob: Finished TID 30 (progress: 6/15) 93 | 10/12/06 08:51:35 INFO spark.SimpleJob: Starting task 2:10 as TID 41 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 94 | 10/12/06 08:51:40 INFO spark.SimpleJob: Finished TID 36 (progress: 7/15) 95 | 10/12/06 08:51:40 INFO spark.SimpleJob: Starting task 2:12 as TID 42 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 96 | 10/12/06 08:51:40 INFO spark.SimpleJob: Finished TID 37 (progress: 8/15) 97 | 10/12/06 08:51:40 INFO spark.SimpleJob: Starting task 2:13 as TID 43 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 98 | 10/12/06 08:51:42 INFO spark.SimpleJob: Finished TID 38 (progress: 9/15) 99 | 10/12/06 08:51:42 INFO spark.SimpleJob: Starting task 2:14 as TID 44 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 100 | 10/12/06 08:51:42 INFO spark.SimpleJob: Finished TID 39 (progress: 10/15) 101 | 10/12/06 08:51:45 INFO spark.SimpleJob: Finished TID 44 (progress: 11/15) 102 | 10/12/06 08:51:46 INFO spark.SimpleJob: Finished TID 40 (progress: 12/15) 103 | 10/12/06 08:51:47 INFO spark.SimpleJob: Finished TID 41 (progress: 13/15) 104 | 10/12/06 08:51:48 INFO spark.SimpleJob: Finished TID 42 (progress: 14/15) 105 | 10/12/06 08:51:48 INFO spark.SimpleJob: Finished TID 43 (progress: 15/15) 106 | 6.173993 42.098988 107 | 48.291269 8.272699 108 | 47.198808 7.093458 109 | 110 | 10/12/06 08:51:48 INFO spark.SparkContext: Tasks finished in 25.749868377 s 111 | Centroids changed by 112 | (0.000000, 0.076094, 0.093685) 113 | to ((6.17,42.10), (48.29,8.27), (47.20,7.09)) 114 | Final centroids: 115 | -------------------------------------------------------------------------------- /run-scaleout-20101205T231425/2-2.log: -------------------------------------------------------------------------------- 1 | stdin: is not a tty 2 | WARNING: Logging before InitGoogleLogging() is written to STDERR 3 | I1206 08:51:51.921068 2746 sched.cpp:181] New master at 1@10.244.130.127:5050 with ID:0 4 | 10/12/06 08:51:51 INFO spark.MesosScheduler: Registered as framework ID 201012060851-0-0001 5 | 10/12/06 08:51:51 INFO spark.BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 6 | 10/12/06 08:51:52 INFO mapred.FileInputFormat: Total input paths to process : 1 7 | 10/12/06 08:51:52 INFO spark.SparkContext: Running 15 tasks in parallel 8 | 10/12/06 08:51:52 INFO spark.SimpleJob: Starting task 0:0 as TID 0 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 9 | 10/12/06 08:51:52 INFO spark.SimpleJob: Starting task 0:2 as TID 1 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 10 | 10/12/06 08:51:52 INFO spark.SimpleJob: Starting task 0:1 as TID 2 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 11 | 10/12/06 08:51:52 INFO spark.SimpleJob: Starting task 0:3 as TID 3 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 12 | 10/12/06 08:51:52 INFO spark.SimpleJob: Starting task 0:4 as TID 4 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 13 | 10/12/06 08:51:52 INFO spark.SimpleJob: Starting task 0:6 as TID 5 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 14 | 10/12/06 08:52:34 INFO spark.SimpleJob: Finished TID 5 (progress: 1/15) 15 | 10/12/06 08:52:34 INFO spark.SimpleJob: Starting task 0:8 as TID 6 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 16 | 10/12/06 08:52:34 INFO spark.SimpleJob: Finished TID 2 (progress: 2/15) 17 | 10/12/06 08:52:34 INFO spark.SimpleJob: Starting task 0:11 as TID 7 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 18 | 10/12/06 08:52:34 INFO spark.SimpleJob: Finished TID 4 (progress: 3/15) 19 | 10/12/06 08:52:34 INFO spark.SimpleJob: Starting task 0:5 as TID 8 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 20 | 10/12/06 08:52:34 INFO spark.SimpleJob: Finished TID 1 (progress: 4/15) 21 | 10/12/06 08:52:34 INFO spark.SimpleJob: Starting task 0:7 as TID 9 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 22 | 10/12/06 08:52:36 INFO spark.SimpleJob: Finished TID 3 (progress: 5/15) 23 | 10/12/06 08:52:36 INFO spark.SimpleJob: Starting task 0:9 as TID 10 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 24 | 10/12/06 08:52:37 INFO spark.SimpleJob: Finished TID 0 (progress: 6/15) 25 | 10/12/06 08:52:37 INFO spark.SimpleJob: Starting task 0:10 as TID 11 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 26 | 10/12/06 08:53:15 INFO spark.SimpleJob: Finished TID 6 (progress: 7/15) 27 | 10/12/06 08:53:15 INFO spark.SimpleJob: Starting task 0:12 as TID 12 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 28 | 10/12/06 08:53:15 INFO spark.SimpleJob: Finished TID 7 (progress: 8/15) 29 | 10/12/06 08:53:15 INFO spark.SimpleJob: Starting task 0:13 as TID 13 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 30 | 10/12/06 08:53:15 INFO spark.SimpleJob: Finished TID 8 (progress: 9/15) 31 | 10/12/06 08:53:15 INFO spark.SimpleJob: Starting task 0:14 as TID 14 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 32 | 10/12/06 08:53:16 INFO spark.SimpleJob: Finished TID 9 (progress: 10/15) 33 | 10/12/06 08:53:22 INFO spark.SimpleJob: Finished TID 11 (progress: 11/15) 34 | 10/12/06 08:53:22 INFO spark.SimpleJob: Finished TID 10 (progress: 12/15) 35 | 10/12/06 08:53:30 INFO spark.SimpleJob: Finished TID 14 (progress: 13/15) 36 | 10/12/06 08:53:54 INFO spark.SimpleJob: Finished TID 13 (progress: 14/15) 37 | 10/12/06 08:53:54 INFO spark.SimpleJob: Finished TID 12 (progress: 15/15) 38 | 10/12/06 08:53:54 INFO spark.SparkContext: Tasks finished in 122.14520988 s 39 | Read 100000000 points. 40 | 10/12/06 08:53:54 INFO spark.SparkContext: Running 15 tasks in parallel 41 | 10/12/06 08:53:54 INFO spark.SimpleJob: Starting task 1:0 as TID 15 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 42 | 10/12/06 08:53:54 INFO spark.SimpleJob: Starting task 1:2 as TID 16 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 43 | 10/12/06 08:53:54 INFO spark.SimpleJob: Starting task 1:1 as TID 17 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 44 | 10/12/06 08:53:54 INFO spark.SimpleJob: Starting task 1:3 as TID 18 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 45 | 10/12/06 08:53:54 INFO spark.SimpleJob: Starting task 1:4 as TID 19 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 46 | 10/12/06 08:53:54 INFO spark.SimpleJob: Starting task 1:6 as TID 20 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 47 | 10/12/06 08:54:05 INFO spark.SimpleJob: Finished TID 17 (progress: 1/15) 48 | 10/12/06 08:54:05 INFO spark.SimpleJob: Starting task 1:8 as TID 21 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 49 | 10/12/06 08:54:05 INFO spark.SimpleJob: Finished TID 20 (progress: 2/15) 50 | 10/12/06 08:54:05 INFO spark.SimpleJob: Starting task 1:11 as TID 22 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 51 | 10/12/06 08:54:05 INFO spark.SimpleJob: Finished TID 15 (progress: 3/15) 52 | 10/12/06 08:54:05 INFO spark.SimpleJob: Starting task 1:9 as TID 23 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 53 | 10/12/06 08:54:06 INFO spark.SimpleJob: Finished TID 18 (progress: 4/15) 54 | 10/12/06 08:54:06 INFO spark.SimpleJob: Starting task 1:10 as TID 24 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 55 | 10/12/06 08:54:07 INFO spark.SimpleJob: Finished TID 19 (progress: 5/15) 56 | 10/12/06 08:54:07 INFO spark.SimpleJob: Starting task 1:5 as TID 25 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 57 | 10/12/06 08:54:07 INFO spark.SimpleJob: Finished TID 16 (progress: 6/15) 58 | 10/12/06 08:54:07 INFO spark.SimpleJob: Starting task 1:7 as TID 26 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 59 | 10/12/06 08:54:16 INFO spark.SimpleJob: Finished TID 21 (progress: 7/15) 60 | 10/12/06 08:54:16 INFO spark.SimpleJob: Starting task 1:12 as TID 27 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 61 | 10/12/06 08:54:16 INFO spark.SimpleJob: Finished TID 22 (progress: 8/15) 62 | 10/12/06 08:54:16 INFO spark.SimpleJob: Starting task 1:13 as TID 28 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 63 | 10/12/06 08:54:16 INFO spark.SimpleJob: Finished TID 23 (progress: 9/15) 64 | 10/12/06 08:54:17 INFO spark.SimpleJob: Finished TID 24 (progress: 10/15) 65 | 10/12/06 08:54:18 INFO spark.SimpleJob: Finished TID 26 (progress: 11/15) 66 | 10/12/06 08:54:18 INFO spark.SimpleJob: Starting task 1:14 as TID 29 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 67 | 10/12/06 08:54:18 INFO spark.SimpleJob: Finished TID 25 (progress: 12/15) 68 | 10/12/06 08:54:21 INFO spark.SimpleJob: Finished TID 29 (progress: 13/15) 69 | 10/12/06 08:54:26 INFO spark.SimpleJob: Finished TID 27 (progress: 14/15) 70 | 10/12/06 08:54:27 INFO spark.SimpleJob: Finished TID 28 (progress: 15/15) 71 | 10/12/06 08:54:27 INFO spark.SparkContext: Tasks finished in 32.641606121 s 72 | Centroids changed by 73 | (2.255668, 26.978290, 27.152732) 74 | to ((6.17,42.10), (48.24,8.22), (47.14,7.02)) 75 | 10/12/06 08:54:27 INFO spark.SparkContext: Running 15 tasks in parallel 76 | 10/12/06 08:54:27 INFO spark.SimpleJob: Starting task 2:0 as TID 30 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 77 | 10/12/06 08:54:27 INFO spark.SimpleJob: Starting task 2:2 as TID 31 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 78 | 10/12/06 08:54:27 INFO spark.SimpleJob: Starting task 2:1 as TID 32 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 79 | 10/12/06 08:54:27 INFO spark.SimpleJob: Starting task 2:3 as TID 33 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 80 | 10/12/06 08:54:27 INFO spark.SimpleJob: Starting task 2:4 as TID 34 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 81 | 10/12/06 08:54:27 INFO spark.SimpleJob: Starting task 2:6 as TID 35 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 82 | 10/12/06 08:54:37 INFO spark.SimpleJob: Finished TID 30 (progress: 1/15) 83 | 10/12/06 08:54:37 INFO spark.SimpleJob: Starting task 2:9 as TID 36 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 84 | 10/12/06 08:54:38 INFO spark.SimpleJob: Finished TID 31 (progress: 2/15) 85 | 10/12/06 08:54:38 INFO spark.SimpleJob: Starting task 2:5 as TID 37 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 86 | 10/12/06 08:54:38 INFO spark.SimpleJob: Finished TID 34 (progress: 3/15) 87 | 10/12/06 08:54:38 INFO spark.SimpleJob: Starting task 2:7 as TID 38 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 88 | 10/12/06 08:54:39 INFO spark.SimpleJob: Finished TID 33 (progress: 4/15) 89 | 10/12/06 08:54:39 INFO spark.SimpleJob: Starting task 2:10 as TID 39 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 90 | 10/12/06 08:54:44 INFO spark.SimpleJob: Finished TID 35 (progress: 5/15) 91 | 10/12/06 08:54:44 INFO spark.SimpleJob: Starting task 2:8 as TID 40 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 92 | 10/12/06 08:54:44 INFO spark.SimpleJob: Finished TID 32 (progress: 6/15) 93 | 10/12/06 08:54:44 INFO spark.SimpleJob: Starting task 2:11 as TID 41 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 94 | 10/12/06 08:54:48 INFO spark.SimpleJob: Finished TID 36 (progress: 7/15) 95 | 10/12/06 08:54:48 INFO spark.SimpleJob: Starting task 2:12 as TID 42 on slave 0-2: ip-10-99-87-160.ec2.internal (non-preferred) 96 | 10/12/06 08:54:49 INFO spark.SimpleJob: Finished TID 37 (progress: 8/15) 97 | 10/12/06 08:54:49 INFO spark.SimpleJob: Starting task 2:14 as TID 43 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 98 | 10/12/06 08:54:50 INFO spark.SimpleJob: Finished TID 39 (progress: 9/15) 99 | 10/12/06 08:54:50 INFO spark.SimpleJob: Finished TID 38 (progress: 10/15) 100 | 10/12/06 08:54:52 INFO spark.SimpleJob: Finished TID 40 (progress: 11/15) 101 | 10/12/06 08:54:52 INFO spark.SimpleJob: Starting task 2:13 as TID 44 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 102 | 10/12/06 08:54:52 INFO spark.SimpleJob: Finished TID 41 (progress: 12/15) 103 | 10/12/06 08:54:53 INFO spark.SimpleJob: Finished TID 43 (progress: 13/15) 104 | 10/12/06 08:54:57 INFO spark.SimpleJob: Finished TID 44 (progress: 14/15) 105 | 10/12/06 08:55:22 INFO spark.SimpleJob: Finished TID 42 (progress: 15/15) 106 | 10/12/06 08:55:22 INFO spark.SparkContext: Tasks finished in 55.621989664 s 107 | Centroids changed by 108 | (0.000000, 0.076094, 0.093685) 109 | to ((6.17,42.10), (48.29,8.27), (47.20,7.09))6.173993 42.098988 110 | 48.291269 8.272699 111 | 47.198808 7.093458 112 | 113 | 114 | Final centroids: 115 | -------------------------------------------------------------------------------- /run-dataset-20101204T225747/3-2.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1204 23:08:15.313992 4184 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/04 23:08:15 INFO MesosScheduler: Registered as framework ID 201012042308-0-0021 4 | 10/12/04 23:08:15 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/04 23:08:15 INFO FileInputFormat: Total input paths to process : 8 6 | 10/12/04 23:08:15 INFO SparkContext: Running 16 tasks in parallel 7 | 10/12/04 23:08:15 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/04 23:08:16 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/04 23:08:16 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/04 23:08:16 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/04 23:08:30 INFO SimpleJob: Finished TID 3 (progress: 1/16) 12 | 10/12/04 23:08:30 INFO SimpleJob: Starting task 0:4 as TID 4 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 13 | 10/12/04 23:08:30 INFO SimpleJob: Finished TID 1 (progress: 2/16) 14 | 10/12/04 23:08:30 INFO SimpleJob: Starting task 0:5 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 15 | 10/12/04 23:08:31 INFO SimpleJob: Finished TID 5 (progress: 3/16) 16 | 10/12/04 23:08:31 INFO SimpleJob: Starting task 0:6 as TID 6 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 17 | 10/12/04 23:08:50 INFO SimpleJob: Finished TID 2 (progress: 4/16) 18 | 10/12/04 23:08:50 INFO SimpleJob: Starting task 0:7 as TID 7 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/04 23:08:50 INFO SimpleJob: Finished TID 0 (progress: 5/16) 20 | 10/12/04 23:08:50 INFO SimpleJob: Starting task 0:8 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 21 | 10/12/04 23:08:51 INFO SimpleJob: Finished TID 8 (progress: 6/16) 22 | 10/12/04 23:08:51 INFO SimpleJob: Starting task 0:9 as TID 9 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 23 | 10/12/04 23:08:55 INFO SimpleJob: Finished TID 4 (progress: 7/16) 24 | 10/12/04 23:08:55 INFO SimpleJob: Starting task 0:10 as TID 10 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 25 | 10/12/04 23:08:55 INFO SimpleJob: Finished TID 6 (progress: 8/16) 26 | 10/12/04 23:08:55 INFO SimpleJob: Starting task 0:11 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 27 | 10/12/04 23:08:56 INFO SimpleJob: Finished TID 11 (progress: 9/16) 28 | 10/12/04 23:08:56 INFO SimpleJob: Starting task 0:12 as TID 12 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 29 | 10/12/04 23:08:56 INFO SimpleJob: Finished TID 10 (progress: 10/16) 30 | 10/12/04 23:08:56 INFO SimpleJob: Starting task 0:13 as TID 13 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 31 | 10/12/04 23:08:56 INFO SimpleJob: Finished TID 13 (progress: 11/16) 32 | 10/12/04 23:08:56 INFO SimpleJob: Starting task 0:14 as TID 14 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 33 | 10/12/04 23:08:56 INFO SimpleJob: Finished TID 12 (progress: 12/16) 34 | 10/12/04 23:08:56 INFO SimpleJob: Starting task 0:15 as TID 15 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 35 | 10/12/04 23:08:57 INFO SimpleJob: Finished TID 15 (progress: 13/16) 36 | 10/12/04 23:08:57 INFO SimpleJob: Finished TID 14 (progress: 14/16) 37 | 10/12/04 23:09:03 INFO SimpleJob: Finished TID 9 (progress: 15/16) 38 | 10/12/04 23:09:03 INFO SimpleJob: Finished TID 7 (progress: 16/16) 39 | 10/12/04 23:09:03 INFO SparkContext: Tasks finished in 47.125822345 s 40 | Read 80000000 points. 41 | 10/12/04 23:09:03 INFO SparkContext: Running 16 tasks in parallel 42 | 10/12/04 23:09:03 INFO SimpleJob: Starting task 1:0 as TID 16 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 43 | 10/12/04 23:09:03 INFO SimpleJob: Starting task 1:1 as TID 17 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 44 | 10/12/04 23:09:03 INFO SimpleJob: Starting task 1:2 as TID 18 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 45 | 10/12/04 23:09:03 INFO SimpleJob: Starting task 1:3 as TID 19 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 46 | 10/12/04 23:09:06 INFO SimpleJob: Finished TID 19 (progress: 1/16) 47 | 10/12/04 23:09:06 INFO SimpleJob: Starting task 1:4 as TID 20 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 48 | 10/12/04 23:09:07 INFO SimpleJob: Finished TID 17 (progress: 2/16) 49 | 10/12/04 23:09:07 INFO SimpleJob: Starting task 1:5 as TID 21 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 50 | 10/12/04 23:09:10 INFO SimpleJob: Finished TID 21 (progress: 3/16) 51 | 10/12/04 23:09:10 INFO SimpleJob: Starting task 1:6 as TID 22 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 52 | 10/12/04 23:09:11 INFO SimpleJob: Finished TID 16 (progress: 4/16) 53 | 10/12/04 23:09:11 INFO SimpleJob: Starting task 1:7 as TID 23 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 54 | 10/12/04 23:09:11 INFO SimpleJob: Finished TID 18 (progress: 5/16) 55 | 10/12/04 23:09:11 INFO SimpleJob: Starting task 1:8 as TID 24 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 56 | 10/12/04 23:09:14 INFO SimpleJob: Finished TID 23 (progress: 6/16) 57 | 10/12/04 23:09:14 INFO SimpleJob: Starting task 1:9 as TID 25 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 58 | 10/12/04 23:09:15 INFO SimpleJob: Finished TID 20 (progress: 7/16) 59 | 10/12/04 23:09:15 INFO SimpleJob: Starting task 1:10 as TID 26 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 60 | 10/12/04 23:09:18 INFO SimpleJob: Finished TID 25 (progress: 8/16) 61 | 10/12/04 23:09:18 INFO SimpleJob: Finished TID 24 (progress: 9/16) 62 | 10/12/04 23:09:19 INFO SimpleJob: Finished TID 22 (progress: 10/16) 63 | 10/12/04 23:09:19 INFO SimpleJob: Starting task 1:11 as TID 27 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 64 | 10/12/04 23:09:22 INFO SimpleJob: Starting task 1:12 as TID 28 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 65 | 10/12/04 23:09:22 INFO SimpleJob: Starting task 1:13 as TID 29 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 66 | 10/12/04 23:09:22 INFO SimpleJob: Finished TID 27 (progress: 11/16) 67 | 10/12/04 23:09:22 INFO SimpleJob: Starting task 1:14 as TID 30 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 68 | 10/12/04 23:09:24 INFO SimpleJob: Finished TID 26 (progress: 12/16) 69 | 10/12/04 23:09:24 INFO SimpleJob: Starting task 1:15 as TID 31 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 70 | 10/12/04 23:09:26 INFO SimpleJob: Finished TID 29 (progress: 13/16) 71 | 10/12/04 23:09:28 INFO SimpleJob: Finished TID 31 (progress: 14/16) 72 | 10/12/04 23:09:29 INFO SimpleJob: Finished TID 28 (progress: 15/16) 73 | 10/12/04 23:09:30 INFO SimpleJob: Finished TID 30 (progress: 16/16) 74 | 10/12/04 23:09:30 INFO SparkContext: Tasks finished in 27.061461036 s 75 | Centroids changed by 76 | (1.020119, 0.433392, 2.183411) 77 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 78 | 10/12/04 23:09:30 INFO SparkContext: Running 16 tasks in parallel 79 | 10/12/04 23:09:30 INFO SimpleJob: Starting task 2:0 as TID 32 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 80 | 10/12/04 23:09:30 INFO SimpleJob: Starting task 2:1 as TID 33 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 81 | 10/12/04 23:09:30 INFO SimpleJob: Starting task 2:2 as TID 34 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 82 | 10/12/04 23:09:30 INFO SimpleJob: Starting task 2:3 as TID 35 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 83 | 10/12/04 23:09:34 INFO SimpleJob: Finished TID 33 (progress: 1/16) 84 | 10/12/04 23:09:34 INFO SimpleJob: Starting task 2:4 as TID 36 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 85 | 10/12/04 23:09:34 INFO SimpleJob: Finished TID 35 (progress: 2/16) 86 | 10/12/04 23:09:34 INFO SimpleJob: Starting task 2:5 as TID 37 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 87 | 10/12/04 23:09:38 INFO SimpleJob: Finished TID 37 (progress: 3/16) 88 | 10/12/04 23:09:38 INFO SimpleJob: Starting task 2:6 as TID 38 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 89 | 10/12/04 23:09:38 INFO SimpleJob: Finished TID 32 (progress: 4/16) 90 | 10/12/04 23:09:38 INFO SimpleJob: Starting task 2:7 as TID 39 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 91 | 10/12/04 23:09:38 INFO SimpleJob: Finished TID 34 (progress: 5/16) 92 | 10/12/04 23:09:38 INFO SimpleJob: Starting task 2:8 as TID 40 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 93 | 10/12/04 23:09:42 INFO SimpleJob: Finished TID 39 (progress: 6/16) 94 | 10/12/04 23:09:42 INFO SimpleJob: Starting task 2:9 as TID 41 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 95 | 10/12/04 23:09:43 INFO SimpleJob: Finished TID 36 (progress: 7/16) 96 | 10/12/04 23:09:43 INFO SimpleJob: Starting task 2:10 as TID 42 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 97 | 10/12/04 23:09:46 INFO SimpleJob: Finished TID 41 (progress: 8/16) 98 | 10/12/04 23:09:46 INFO SimpleJob: Starting task 2:12 as TID 43 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 99 | 10/12/04 23:09:47 INFO SimpleJob: Finished TID 38 (progress: 9/16) 100 | 10/12/04 23:09:47 INFO SimpleJob: Starting task 2:11 as TID 44 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 101 | 10/12/04 23:09:47 INFO SimpleJob: Finished TID 40 (progress: 10/16) 102 | 10/12/04 23:09:47 INFO SimpleJob: Starting task 2:13 as TID 45 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 103 | 10/12/04 23:09:51 INFO SimpleJob: Finished TID 44 (progress: 11/16) 104 | 10/12/04 23:09:51 INFO SimpleJob: Starting task 2:14 as TID 46 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 105 | 10/12/04 23:09:51 INFO SimpleJob: Finished TID 45 (progress: 12/16) 106 | 10/12/04 23:09:52 INFO SimpleJob: Finished TID 42 (progress: 13/16) 107 | 10/12/04 23:09:52 INFO SimpleJob: Starting task 2:15 as TID 47 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 108 | 10/12/04 23:09:54 INFO SimpleJob: Finished TID 43 (progress: 14/16) 109 | 10/12/04 23:09:56 INFO SimpleJob: Finished TID 47 (progress: 15/16) 110 | 10/12/04 23:09:58 INFO SimpleJob: Finished TID 46 (progress: 16/16) 111 | 10/12/04 23:09:58 INFO SparkContext: Tasks finished in 28.41892853 s 112 | Centroids changed by 113 | (0.000000, 0.000000, 0.000000) 114 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 115 | Final centroids: 116 | 7.776982 40.661016 117 | 44.751415 35.355014 118 | 17.872132 7.510652 119 | 120 | -------------------------------------------------------------------------------- /run-dataset-20101204T225747/3-3.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1204 23:10:00.150779 4235 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/04 23:10:00 INFO MesosScheduler: Registered as framework ID 201012042310-0-0022 4 | 10/12/04 23:10:00 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/04 23:10:00 INFO FileInputFormat: Total input paths to process : 8 6 | 10/12/04 23:10:00 INFO SparkContext: Running 16 tasks in parallel 7 | 10/12/04 23:10:00 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/04 23:10:00 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/04 23:10:00 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/04 23:10:00 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/04 23:10:15 INFO SimpleJob: Finished TID 1 (progress: 1/16) 12 | 10/12/04 23:10:15 INFO SimpleJob: Starting task 0:4 as TID 4 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 13 | 10/12/04 23:10:15 INFO SimpleJob: Finished TID 3 (progress: 2/16) 14 | 10/12/04 23:10:15 INFO SimpleJob: Starting task 0:5 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 15 | 10/12/04 23:10:16 INFO SimpleJob: Finished TID 5 (progress: 3/16) 16 | 10/12/04 23:10:16 INFO SimpleJob: Starting task 0:6 as TID 6 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 17 | 10/12/04 23:10:35 INFO SimpleJob: Finished TID 0 (progress: 4/16) 18 | 10/12/04 23:10:35 INFO SimpleJob: Starting task 0:7 as TID 7 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/04 23:10:35 INFO SimpleJob: Finished TID 2 (progress: 5/16) 20 | 10/12/04 23:10:35 INFO SimpleJob: Starting task 0:8 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 21 | 10/12/04 23:10:36 INFO SimpleJob: Finished TID 8 (progress: 6/16) 22 | 10/12/04 23:10:36 INFO SimpleJob: Starting task 0:9 as TID 9 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 23 | 10/12/04 23:10:40 INFO SimpleJob: Finished TID 6 (progress: 7/16) 24 | 10/12/04 23:10:40 INFO SimpleJob: Starting task 0:10 as TID 10 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 25 | 10/12/04 23:10:40 INFO SimpleJob: Finished TID 4 (progress: 8/16) 26 | 10/12/04 23:10:40 INFO SimpleJob: Starting task 0:11 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 27 | 10/12/04 23:10:41 INFO SimpleJob: Finished TID 11 (progress: 9/16) 28 | 10/12/04 23:10:41 INFO SimpleJob: Starting task 0:12 as TID 12 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 29 | 10/12/04 23:10:41 INFO SimpleJob: Finished TID 10 (progress: 10/16) 30 | 10/12/04 23:10:41 INFO SimpleJob: Starting task 0:13 as TID 13 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 31 | 10/12/04 23:10:41 INFO SimpleJob: Finished TID 13 (progress: 11/16) 32 | 10/12/04 23:10:41 INFO SimpleJob: Starting task 0:14 as TID 14 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 33 | 10/12/04 23:10:41 INFO SimpleJob: Finished TID 12 (progress: 12/16) 34 | 10/12/04 23:10:41 INFO SimpleJob: Starting task 0:15 as TID 15 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 35 | 10/12/04 23:10:42 INFO SimpleJob: Finished TID 14 (progress: 13/16) 36 | 10/12/04 23:10:42 INFO SimpleJob: Finished TID 15 (progress: 14/16) 37 | 10/12/04 23:10:48 INFO SimpleJob: Finished TID 9 (progress: 15/16) 38 | 10/12/04 23:10:48 INFO SimpleJob: Finished TID 7 (progress: 16/16) 39 | 10/12/04 23:10:48 INFO SparkContext: Tasks finished in 47.56838133 s 40 | Read 80000000 points. 41 | 10/12/04 23:10:48 INFO SparkContext: Running 16 tasks in parallel 42 | 10/12/04 23:10:48 INFO SimpleJob: Starting task 1:0 as TID 16 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 43 | 10/12/04 23:10:48 INFO SimpleJob: Starting task 1:1 as TID 17 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 44 | 10/12/04 23:10:48 INFO SimpleJob: Starting task 1:2 as TID 18 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 45 | 10/12/04 23:10:48 INFO SimpleJob: Starting task 1:3 as TID 19 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 46 | 10/12/04 23:10:52 INFO SimpleJob: Finished TID 17 (progress: 1/16) 47 | 10/12/04 23:10:52 INFO SimpleJob: Starting task 1:4 as TID 20 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 48 | 10/12/04 23:10:53 INFO SimpleJob: Finished TID 19 (progress: 2/16) 49 | 10/12/04 23:10:53 INFO SimpleJob: Starting task 1:5 as TID 21 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 50 | 10/12/04 23:10:57 INFO SimpleJob: Finished TID 21 (progress: 3/16) 51 | 10/12/04 23:10:57 INFO SimpleJob: Starting task 1:6 as TID 22 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 52 | 10/12/04 23:10:57 INFO SimpleJob: Finished TID 18 (progress: 4/16) 53 | 10/12/04 23:10:57 INFO SimpleJob: Starting task 1:7 as TID 23 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 54 | 10/12/04 23:10:58 INFO SimpleJob: Finished TID 16 (progress: 5/16) 55 | 10/12/04 23:10:58 INFO SimpleJob: Starting task 1:8 as TID 24 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 56 | 10/12/04 23:11:01 INFO SimpleJob: Finished TID 23 (progress: 6/16) 57 | 10/12/04 23:11:01 INFO SimpleJob: Starting task 1:9 as TID 25 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 58 | 10/12/04 23:11:01 INFO SimpleJob: Finished TID 20 (progress: 7/16) 59 | 10/12/04 23:11:01 INFO SimpleJob: Starting task 1:10 as TID 26 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 60 | 10/12/04 23:11:05 INFO SimpleJob: Finished TID 25 (progress: 8/16) 61 | 10/12/04 23:11:05 INFO SimpleJob: Starting task 1:11 as TID 27 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 62 | 10/12/04 23:11:06 INFO SimpleJob: Finished TID 22 (progress: 9/16) 63 | 10/12/04 23:11:06 INFO SimpleJob: Starting task 1:12 as TID 28 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 64 | 10/12/04 23:11:06 INFO SimpleJob: Finished TID 24 (progress: 10/16) 65 | 10/12/04 23:11:08 INFO SimpleJob: Finished TID 27 (progress: 11/16) 66 | 10/12/04 23:11:09 INFO SimpleJob: Starting task 1:13 as TID 29 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 67 | 10/12/04 23:11:09 INFO SimpleJob: Starting task 1:14 as TID 30 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 68 | 10/12/04 23:11:11 INFO SimpleJob: Finished TID 26 (progress: 12/16) 69 | 10/12/04 23:11:11 INFO SimpleJob: Starting task 1:15 as TID 31 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 70 | 10/12/04 23:11:13 INFO SimpleJob: Finished TID 29 (progress: 13/16) 71 | 10/12/04 23:11:15 INFO SimpleJob: Finished TID 31 (progress: 14/16) 72 | 10/12/04 23:11:15 INFO SimpleJob: Finished TID 28 (progress: 15/16) 73 | 10/12/04 23:11:17 INFO SimpleJob: Finished TID 30 (progress: 16/16) 74 | 10/12/04 23:11:17 INFO SparkContext: Tasks finished in 28.667409663 s 75 | Centroids changed by 76 | (1.020119, 0.433392, 2.183411) 77 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 78 | 10/12/04 23:11:17 INFO SparkContext: Running 16 tasks in parallel 79 | 10/12/04 23:11:17 INFO SimpleJob: Starting task 2:0 as TID 32 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 80 | 10/12/04 23:11:17 INFO SimpleJob: Starting task 2:1 as TID 33 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 81 | 10/12/04 23:11:17 INFO SimpleJob: Starting task 2:2 as TID 34 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 82 | 10/12/04 23:11:17 INFO SimpleJob: Starting task 2:3 as TID 35 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 83 | 10/12/04 23:11:21 INFO SimpleJob: Finished TID 35 (progress: 1/16) 84 | 10/12/04 23:11:21 INFO SimpleJob: Starting task 2:4 as TID 36 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 85 | 10/12/04 23:11:21 INFO SimpleJob: Finished TID 33 (progress: 2/16) 86 | 10/12/04 23:11:21 INFO SimpleJob: Starting task 2:5 as TID 37 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 87 | 10/12/04 23:11:25 INFO SimpleJob: Finished TID 37 (progress: 3/16) 88 | 10/12/04 23:11:25 INFO SimpleJob: Starting task 2:6 as TID 38 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 89 | 10/12/04 23:11:26 INFO SimpleJob: Finished TID 32 (progress: 4/16) 90 | 10/12/04 23:11:26 INFO SimpleJob: Starting task 2:7 as TID 39 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 91 | 10/12/04 23:11:26 INFO SimpleJob: Finished TID 34 (progress: 5/16) 92 | 10/12/04 23:11:26 INFO SimpleJob: Starting task 2:8 as TID 40 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 93 | 10/12/04 23:11:30 INFO SimpleJob: Finished TID 39 (progress: 6/16) 94 | 10/12/04 23:11:30 INFO SimpleJob: Starting task 2:9 as TID 41 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 95 | 10/12/04 23:11:31 INFO SimpleJob: Finished TID 36 (progress: 7/16) 96 | 10/12/04 23:11:31 INFO SimpleJob: Starting task 2:10 as TID 42 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 97 | 10/12/04 23:11:34 INFO SimpleJob: Finished TID 41 (progress: 8/16) 98 | 10/12/04 23:11:34 INFO SimpleJob: Starting task 2:11 as TID 43 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 99 | 10/12/04 23:11:35 INFO SimpleJob: Finished TID 38 (progress: 9/16) 100 | 10/12/04 23:11:35 INFO SimpleJob: Starting task 2:12 as TID 44 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 101 | 10/12/04 23:11:36 INFO SimpleJob: Finished TID 40 (progress: 10/16) 102 | 10/12/04 23:11:36 INFO SimpleJob: Starting task 2:13 as TID 45 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 103 | 10/12/04 23:11:38 INFO SimpleJob: Finished TID 43 (progress: 11/16) 104 | 10/12/04 23:11:38 INFO SimpleJob: Starting task 2:14 as TID 46 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 105 | 10/12/04 23:11:40 INFO SimpleJob: Finished TID 45 (progress: 12/16) 106 | 10/12/04 23:11:41 INFO SimpleJob: Finished TID 42 (progress: 13/16) 107 | 10/12/04 23:11:41 INFO SimpleJob: Starting task 2:15 as TID 47 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 108 | 10/12/04 23:11:44 INFO SimpleJob: Finished TID 44 (progress: 14/16) 109 | 10/12/04 23:11:45 INFO SimpleJob: Finished TID 47 (progress: 15/16) 110 | 10/12/04 23:11:47 INFO SimpleJob: Finished TID 46 (progress: 16/16) 111 | 10/12/04 23:11:47 INFO SparkContext: Tasks finished in 30.231147664 s 112 | Centroids changed by 113 | (0.000000, 0.000000, 0.000000) 114 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 115 | Final centroids: 116 | 7.776982 40.661016 117 | 44.751415 35.355014 118 | 17.872132 7.510652 119 | 120 | -------------------------------------------------------------------------------- /run-dataset-20101205T014106/3-1.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1205 01:50:16.431300 5098 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/05 01:50:16 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 4 | 10/12/05 01:50:16 INFO MesosScheduler: Registered as framework ID 201012050150-0-0035 5 | 10/12/05 01:50:16 INFO FileInputFormat: Total input paths to process : 8 6 | 10/12/05 01:50:17 INFO SparkContext: Running 16 tasks in parallel 7 | 10/12/05 01:50:17 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/05 01:50:17 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/05 01:50:17 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/05 01:50:17 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/05 01:50:32 INFO SimpleJob: Finished TID 3 (progress: 1/16) 12 | 10/12/05 01:50:32 INFO SimpleJob: Starting task 0:4 as TID 4 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 13 | 10/12/05 01:50:32 INFO SimpleJob: Finished TID 1 (progress: 2/16) 14 | 10/12/05 01:50:32 INFO SimpleJob: Starting task 0:5 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 15 | 10/12/05 01:50:33 INFO SimpleJob: Finished TID 5 (progress: 3/16) 16 | 10/12/05 01:50:33 INFO SimpleJob: Starting task 0:6 as TID 6 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 17 | 10/12/05 01:50:52 INFO SimpleJob: Finished TID 0 (progress: 4/16) 18 | 10/12/05 01:50:52 INFO SimpleJob: Starting task 0:7 as TID 7 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/05 01:50:52 INFO SimpleJob: Finished TID 2 (progress: 5/16) 20 | 10/12/05 01:50:52 INFO SimpleJob: Starting task 0:8 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 21 | 10/12/05 01:50:52 INFO SimpleJob: Finished TID 8 (progress: 6/16) 22 | 10/12/05 01:50:52 INFO SimpleJob: Starting task 0:9 as TID 9 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 23 | 10/12/05 01:50:58 INFO SimpleJob: Finished TID 4 (progress: 7/16) 24 | 10/12/05 01:50:58 INFO SimpleJob: Starting task 0:10 as TID 10 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 25 | 10/12/05 01:50:58 INFO SimpleJob: Finished TID 6 (progress: 8/16) 26 | 10/12/05 01:50:58 INFO SimpleJob: Starting task 0:11 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 27 | 10/12/05 01:50:58 INFO SimpleJob: Finished TID 11 (progress: 9/16) 28 | 10/12/05 01:50:58 INFO SimpleJob: Starting task 0:12 as TID 12 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 29 | 10/12/05 01:50:58 INFO SimpleJob: Finished TID 10 (progress: 10/16) 30 | 10/12/05 01:50:58 INFO SimpleJob: Starting task 0:13 as TID 13 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 31 | 10/12/05 01:50:58 INFO SimpleJob: Finished TID 13 (progress: 11/16) 32 | 10/12/05 01:50:58 INFO SimpleJob: Starting task 0:14 as TID 14 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 33 | 10/12/05 01:50:59 INFO SimpleJob: Finished TID 12 (progress: 12/16) 34 | 10/12/05 01:50:59 INFO SimpleJob: Starting task 0:15 as TID 15 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 35 | 10/12/05 01:50:59 INFO SimpleJob: Finished TID 15 (progress: 13/16) 36 | 10/12/05 01:50:59 INFO SimpleJob: Finished TID 14 (progress: 14/16) 37 | 10/12/05 01:51:04 INFO SimpleJob: Finished TID 9 (progress: 15/16) 38 | 10/12/05 01:51:04 INFO SimpleJob: Finished TID 7 (progress: 16/16) 39 | 10/12/05 01:51:04 INFO SparkContext: Tasks finished in 47.788900872 s 40 | Read 80000000 points. 41 | 10/12/05 01:51:04 INFO SparkContext: Running 16 tasks in parallel 42 | 10/12/05 01:51:04 INFO SimpleJob: Starting task 1:0 as TID 16 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 43 | 10/12/05 01:51:04 INFO SimpleJob: Starting task 1:1 as TID 17 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 44 | 10/12/05 01:51:04 INFO SimpleJob: Starting task 1:2 as TID 18 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 45 | 10/12/05 01:51:04 INFO SimpleJob: Starting task 1:3 as TID 19 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 46 | 10/12/05 01:51:09 INFO SimpleJob: Finished TID 17 (progress: 1/16) 47 | 10/12/05 01:51:09 INFO SimpleJob: Starting task 1:4 as TID 20 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 48 | 10/12/05 01:51:10 INFO SimpleJob: Finished TID 19 (progress: 2/16) 49 | 10/12/05 01:51:10 INFO SimpleJob: Starting task 1:5 as TID 21 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 50 | 10/12/05 01:51:13 INFO SimpleJob: Finished TID 16 (progress: 3/16) 51 | 10/12/05 01:51:13 INFO SimpleJob: Starting task 1:7 as TID 22 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 52 | 10/12/05 01:51:13 INFO SimpleJob: Finished TID 18 (progress: 4/16) 53 | 10/12/05 01:51:13 INFO SimpleJob: Starting task 1:8 as TID 23 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 54 | 10/12/05 01:51:14 INFO SimpleJob: Finished TID 21 (progress: 5/16) 55 | 10/12/05 01:51:14 INFO SimpleJob: Starting task 1:6 as TID 24 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 56 | 10/12/05 01:51:16 INFO SimpleJob: Finished TID 22 (progress: 6/16) 57 | 10/12/05 01:51:16 INFO SimpleJob: Starting task 1:9 as TID 25 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 58 | 10/12/05 01:51:19 INFO SimpleJob: Finished TID 20 (progress: 7/16) 59 | 10/12/05 01:51:19 INFO SimpleJob: Starting task 1:10 as TID 26 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 60 | 10/12/05 01:51:20 INFO SimpleJob: Finished TID 25 (progress: 8/16) 61 | 10/12/05 01:51:21 INFO SimpleJob: Finished TID 23 (progress: 9/16) 62 | 10/12/05 01:51:23 INFO SimpleJob: Finished TID 24 (progress: 10/16) 63 | 10/12/05 01:51:23 INFO SimpleJob: Starting task 1:11 as TID 27 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 64 | 10/12/05 01:51:26 INFO SimpleJob: Starting task 1:12 as TID 28 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 65 | 10/12/05 01:51:26 INFO SimpleJob: Starting task 1:13 as TID 29 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 66 | 10/12/05 01:51:27 INFO SimpleJob: Finished TID 27 (progress: 11/16) 67 | 10/12/05 01:51:27 INFO SimpleJob: Starting task 1:14 as TID 30 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 68 | 10/12/05 01:51:29 INFO SimpleJob: Finished TID 26 (progress: 12/16) 69 | 10/12/05 01:51:29 INFO SimpleJob: Starting task 1:15 as TID 31 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 70 | 10/12/05 01:51:30 INFO SimpleJob: Finished TID 29 (progress: 13/16) 71 | 10/12/05 01:51:33 INFO SimpleJob: Finished TID 28 (progress: 14/16) 72 | 10/12/05 01:51:33 INFO SimpleJob: Finished TID 31 (progress: 15/16) 73 | 10/12/05 01:51:35 INFO SimpleJob: Finished TID 30 (progress: 16/16) 74 | 10/12/05 01:51:35 INFO SparkContext: Tasks finished in 30.308659379 s 75 | Centroids changed by 76 | (1.020119, 0.433392, 2.183411) 77 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 78 | 10/12/05 01:51:35 INFO SparkContext: Running 16 tasks in parallel 79 | 10/12/05 01:51:35 INFO SimpleJob: Starting task 2:0 as TID 32 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 80 | 10/12/05 01:51:35 INFO SimpleJob: Starting task 2:1 as TID 33 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 81 | 10/12/05 01:51:35 INFO SimpleJob: Starting task 2:2 as TID 34 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 82 | 10/12/05 01:51:35 INFO SimpleJob: Starting task 2:3 as TID 35 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 83 | 10/12/05 01:51:39 INFO SimpleJob: Finished TID 35 (progress: 1/16) 84 | 10/12/05 01:51:39 INFO SimpleJob: Starting task 2:4 as TID 36 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 85 | 10/12/05 01:51:39 INFO SimpleJob: Finished TID 33 (progress: 2/16) 86 | 10/12/05 01:51:39 INFO SimpleJob: Starting task 2:5 as TID 37 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 87 | 10/12/05 01:51:43 INFO SimpleJob: Finished TID 32 (progress: 3/16) 88 | 10/12/05 01:51:43 INFO SimpleJob: Starting task 2:7 as TID 38 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 89 | 10/12/05 01:51:43 INFO SimpleJob: Finished TID 34 (progress: 4/16) 90 | 10/12/05 01:51:43 INFO SimpleJob: Starting task 2:8 as TID 39 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 91 | 10/12/05 01:51:43 INFO SimpleJob: Finished TID 37 (progress: 5/16) 92 | 10/12/05 01:51:43 INFO SimpleJob: Starting task 2:6 as TID 40 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 93 | 10/12/05 01:51:47 INFO SimpleJob: Finished TID 38 (progress: 6/16) 94 | 10/12/05 01:51:47 INFO SimpleJob: Starting task 2:9 as TID 41 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 95 | 10/12/05 01:51:49 INFO SimpleJob: Finished TID 36 (progress: 7/16) 96 | 10/12/05 01:51:49 INFO SimpleJob: Starting task 2:10 as TID 42 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 97 | 10/12/05 01:51:50 INFO SimpleJob: Finished TID 41 (progress: 8/16) 98 | 10/12/05 01:51:50 INFO SimpleJob: Starting task 2:12 as TID 43 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 99 | 10/12/05 01:51:51 INFO SimpleJob: Finished TID 39 (progress: 9/16) 100 | 10/12/05 01:51:51 INFO SimpleJob: Starting task 2:13 as TID 44 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 101 | 10/12/05 01:51:53 INFO SimpleJob: Finished TID 40 (progress: 10/16) 102 | 10/12/05 01:51:53 INFO SimpleJob: Starting task 2:11 as TID 45 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 103 | 10/12/05 01:51:55 INFO SimpleJob: Finished TID 44 (progress: 11/16) 104 | 10/12/05 01:51:56 INFO SimpleJob: Starting task 2:14 as TID 46 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 105 | 10/12/05 01:51:57 INFO SimpleJob: Finished TID 45 (progress: 12/16) 106 | 10/12/05 01:51:57 INFO SimpleJob: Starting task 2:15 as TID 47 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 107 | 10/12/05 01:51:58 INFO SimpleJob: Finished TID 42 (progress: 13/16) 108 | 10/12/05 01:51:58 INFO SimpleJob: Finished TID 43 (progress: 14/16) 109 | 10/12/05 01:52:00 INFO SimpleJob: Finished TID 47 (progress: 15/16) 110 | 10/12/05 01:52:03 INFO SimpleJob: Finished TID 46 (progress: 16/16) 111 | 10/12/05 01:52:03 INFO SparkContext: Tasks finished in 28.062001976 s 112 | Centroids changed by 113 | (0.000000, 0.000000, 0.000000) 114 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 115 | Final centroids: 116 | 7.776982 40.661016 117 | 44.751415 35.355014 118 | 17.872132 7.510652 119 | 120 | -------------------------------------------------------------------------------- /run-dataset-20101205T014106/3-2.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1205 01:52:04.822257 5139 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/05 01:52:04 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 4 | 10/12/05 01:52:04 INFO MesosScheduler: Registered as framework ID 201012050152-0-0036 5 | 10/12/05 01:52:05 INFO FileInputFormat: Total input paths to process : 8 6 | 10/12/05 01:52:05 INFO SparkContext: Running 16 tasks in parallel 7 | 10/12/05 01:52:05 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/05 01:52:05 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/05 01:52:05 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/05 01:52:05 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/05 01:52:20 INFO SimpleJob: Finished TID 1 (progress: 1/16) 12 | 10/12/05 01:52:20 INFO SimpleJob: Starting task 0:4 as TID 4 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 13 | 10/12/05 01:52:20 INFO SimpleJob: Finished TID 3 (progress: 2/16) 14 | 10/12/05 01:52:20 INFO SimpleJob: Starting task 0:5 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 15 | 10/12/05 01:52:21 INFO SimpleJob: Finished TID 5 (progress: 3/16) 16 | 10/12/05 01:52:21 INFO SimpleJob: Starting task 0:6 as TID 6 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 17 | 10/12/05 01:52:40 INFO SimpleJob: Finished TID 2 (progress: 4/16) 18 | 10/12/05 01:52:40 INFO SimpleJob: Starting task 0:7 as TID 7 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/05 01:52:40 INFO SimpleJob: Finished TID 0 (progress: 5/16) 20 | 10/12/05 01:52:40 INFO SimpleJob: Starting task 0:8 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 21 | 10/12/05 01:52:41 INFO SimpleJob: Finished TID 8 (progress: 6/16) 22 | 10/12/05 01:52:41 INFO SimpleJob: Starting task 0:9 as TID 9 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 23 | 10/12/05 01:52:45 INFO SimpleJob: Finished TID 4 (progress: 7/16) 24 | 10/12/05 01:52:45 INFO SimpleJob: Starting task 0:10 as TID 10 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 25 | 10/12/05 01:52:45 INFO SimpleJob: Finished TID 6 (progress: 8/16) 26 | 10/12/05 01:52:45 INFO SimpleJob: Starting task 0:11 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 27 | 10/12/05 01:52:46 INFO SimpleJob: Finished TID 11 (progress: 9/16) 28 | 10/12/05 01:52:46 INFO SimpleJob: Starting task 0:12 as TID 12 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 29 | 10/12/05 01:52:46 INFO SimpleJob: Finished TID 10 (progress: 10/16) 30 | 10/12/05 01:52:46 INFO SimpleJob: Starting task 0:13 as TID 13 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 31 | 10/12/05 01:52:46 INFO SimpleJob: Finished TID 12 (progress: 11/16) 32 | 10/12/05 01:52:46 INFO SimpleJob: Starting task 0:14 as TID 14 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 33 | 10/12/05 01:52:46 INFO SimpleJob: Finished TID 13 (progress: 12/16) 34 | 10/12/05 01:52:46 INFO SimpleJob: Starting task 0:15 as TID 15 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 35 | 10/12/05 01:52:47 INFO SimpleJob: Finished TID 15 (progress: 13/16) 36 | 10/12/05 01:52:47 INFO SimpleJob: Finished TID 14 (progress: 14/16) 37 | 10/12/05 01:52:52 INFO SimpleJob: Finished TID 9 (progress: 15/16) 38 | 10/12/05 01:52:52 INFO SimpleJob: Finished TID 7 (progress: 16/16) 39 | 10/12/05 01:52:52 INFO SparkContext: Tasks finished in 47.367215284 s 40 | Read 80000000 points. 41 | 10/12/05 01:52:52 INFO SparkContext: Running 16 tasks in parallel 42 | 10/12/05 01:52:52 INFO SimpleJob: Starting task 1:0 as TID 16 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 43 | 10/12/05 01:52:52 INFO SimpleJob: Starting task 1:1 as TID 17 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 44 | 10/12/05 01:52:52 INFO SimpleJob: Starting task 1:2 as TID 18 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 45 | 10/12/05 01:52:52 INFO SimpleJob: Starting task 1:3 as TID 19 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 46 | 10/12/05 01:52:57 INFO SimpleJob: Finished TID 19 (progress: 1/16) 47 | 10/12/05 01:52:57 INFO SimpleJob: Starting task 1:4 as TID 20 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 48 | 10/12/05 01:52:57 INFO SimpleJob: Finished TID 17 (progress: 2/16) 49 | 10/12/05 01:52:57 INFO SimpleJob: Starting task 1:5 as TID 21 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 50 | 10/12/05 01:53:01 INFO SimpleJob: Finished TID 18 (progress: 3/16) 51 | 10/12/05 01:53:01 INFO SimpleJob: Starting task 1:7 as TID 22 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 52 | 10/12/05 01:53:01 INFO SimpleJob: Finished TID 16 (progress: 4/16) 53 | 10/12/05 01:53:01 INFO SimpleJob: Starting task 1:8 as TID 23 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 54 | 10/12/05 01:53:01 INFO SimpleJob: Finished TID 21 (progress: 5/16) 55 | 10/12/05 01:53:01 INFO SimpleJob: Starting task 1:6 as TID 24 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 56 | 10/12/05 01:53:04 INFO SimpleJob: Finished TID 22 (progress: 6/16) 57 | 10/12/05 01:53:04 INFO SimpleJob: Starting task 1:9 as TID 25 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 58 | 10/12/05 01:53:06 INFO SimpleJob: Finished TID 20 (progress: 7/16) 59 | 10/12/05 01:53:06 INFO SimpleJob: Starting task 1:10 as TID 26 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 60 | 10/12/05 01:53:08 INFO SimpleJob: Finished TID 25 (progress: 8/16) 61 | 10/12/05 01:53:08 INFO SimpleJob: Finished TID 23 (progress: 9/16) 62 | 10/12/05 01:53:10 INFO SimpleJob: Starting task 1:11 as TID 27 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 63 | 10/12/05 01:53:10 INFO SimpleJob: Starting task 1:12 as TID 28 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 64 | 10/12/05 01:53:10 INFO SimpleJob: Finished TID 24 (progress: 10/16) 65 | 10/12/05 01:53:10 INFO SimpleJob: Starting task 1:13 as TID 29 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 66 | 10/12/05 01:53:14 INFO SimpleJob: Finished TID 27 (progress: 11/16) 67 | 10/12/05 01:53:14 INFO SimpleJob: Starting task 1:14 as TID 30 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 68 | 10/12/05 01:53:15 INFO SimpleJob: Finished TID 29 (progress: 12/16) 69 | 10/12/05 01:53:15 INFO SimpleJob: Starting task 1:15 as TID 31 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 70 | 10/12/05 01:53:15 INFO SimpleJob: Finished TID 26 (progress: 13/16) 71 | 10/12/05 01:53:17 INFO SimpleJob: Finished TID 31 (progress: 14/16) 72 | 10/12/05 01:53:18 INFO SimpleJob: Finished TID 28 (progress: 15/16) 73 | 10/12/05 01:53:20 INFO SimpleJob: Finished TID 30 (progress: 16/16) 74 | 10/12/05 01:53:20 INFO SparkContext: Tasks finished in 27.704078054 s 75 | Centroids changed by 76 | (1.020119, 0.433392, 2.183411) 77 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 78 | 10/12/05 01:53:20 INFO SparkContext: Running 16 tasks in parallel 79 | 10/12/05 01:53:20 INFO SimpleJob: Starting task 2:0 as TID 32 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 80 | 10/12/05 01:53:20 INFO SimpleJob: Starting task 2:1 as TID 33 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 81 | 10/12/05 01:53:20 INFO SimpleJob: Starting task 2:2 as TID 34 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 82 | 10/12/05 01:53:20 INFO SimpleJob: Starting task 2:3 as TID 35 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 83 | 10/12/05 01:53:24 INFO SimpleJob: Finished TID 33 (progress: 1/16) 84 | 10/12/05 01:53:24 INFO SimpleJob: Starting task 2:4 as TID 36 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 85 | 10/12/05 01:53:25 INFO SimpleJob: Finished TID 35 (progress: 2/16) 86 | 10/12/05 01:53:25 INFO SimpleJob: Starting task 2:5 as TID 37 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 87 | 10/12/05 01:53:29 INFO SimpleJob: Finished TID 37 (progress: 3/16) 88 | 10/12/05 01:53:29 INFO SimpleJob: Starting task 2:6 as TID 38 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 89 | 10/12/05 01:53:29 INFO SimpleJob: Finished TID 32 (progress: 4/16) 90 | 10/12/05 01:53:29 INFO SimpleJob: Starting task 2:7 as TID 39 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 91 | 10/12/05 01:53:29 INFO SimpleJob: Finished TID 34 (progress: 5/16) 92 | 10/12/05 01:53:29 INFO SimpleJob: Starting task 2:8 as TID 40 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 93 | 10/12/05 01:53:33 INFO SimpleJob: Finished TID 39 (progress: 6/16) 94 | 10/12/05 01:53:33 INFO SimpleJob: Starting task 2:9 as TID 41 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 95 | 10/12/05 01:53:34 INFO SimpleJob: Finished TID 36 (progress: 7/16) 96 | 10/12/05 01:53:34 INFO SimpleJob: Starting task 2:10 as TID 42 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 97 | 10/12/05 01:53:36 INFO SimpleJob: Finished TID 41 (progress: 8/16) 98 | 10/12/05 01:53:36 INFO SimpleJob: Starting task 2:11 as TID 43 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 99 | 10/12/05 01:53:37 INFO SimpleJob: Finished TID 40 (progress: 9/16) 100 | 10/12/05 01:53:37 INFO SimpleJob: Starting task 2:12 as TID 44 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 101 | 10/12/05 01:53:38 INFO SimpleJob: Finished TID 38 (progress: 10/16) 102 | 10/12/05 01:53:38 INFO SimpleJob: Starting task 2:13 as TID 45 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 103 | 10/12/05 01:53:40 INFO SimpleJob: Finished TID 43 (progress: 11/16) 104 | 10/12/05 01:53:40 INFO SimpleJob: Starting task 2:14 as TID 46 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 105 | 10/12/05 01:53:42 INFO SimpleJob: Finished TID 45 (progress: 12/16) 106 | 10/12/05 01:53:42 INFO SimpleJob: Starting task 2:15 as TID 47 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 107 | 10/12/05 01:53:44 INFO SimpleJob: Finished TID 42 (progress: 13/16) 108 | 10/12/05 01:53:45 INFO SimpleJob: Finished TID 47 (progress: 14/16) 109 | 10/12/05 01:53:46 INFO SimpleJob: Finished TID 44 (progress: 15/16) 110 | 10/12/05 01:53:48 INFO SimpleJob: Finished TID 46 (progress: 16/16) 111 | 10/12/05 01:53:48 INFO SparkContext: Tasks finished in 27.920177889 s 112 | Centroids changed by 113 | (0.000000, 0.000000, 0.000000) 114 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 115 | Final centroids: 116 | 7.776982 40.661016 117 | 44.751415 35.355014 118 | 17.872132 7.510652 119 | 120 | -------------------------------------------------------------------------------- /run-dataset-20101205T014106/3-3.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1205 01:53:50.323355 5186 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/05 01:53:50 INFO MesosScheduler: Registered as framework ID 201012050153-0-0037 4 | 10/12/05 01:53:50 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/05 01:53:50 INFO FileInputFormat: Total input paths to process : 8 6 | 10/12/05 01:53:50 INFO SparkContext: Running 16 tasks in parallel 7 | 10/12/05 01:53:50 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/05 01:53:50 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/05 01:53:50 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/05 01:53:50 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/05 01:54:06 INFO SimpleJob: Finished TID 3 (progress: 1/16) 12 | 10/12/05 01:54:06 INFO SimpleJob: Starting task 0:4 as TID 4 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 13 | 10/12/05 01:54:06 INFO SimpleJob: Finished TID 1 (progress: 2/16) 14 | 10/12/05 01:54:06 INFO SimpleJob: Starting task 0:5 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 15 | 10/12/05 01:54:06 INFO SimpleJob: Finished TID 5 (progress: 3/16) 16 | 10/12/05 01:54:06 INFO SimpleJob: Starting task 0:6 as TID 6 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 17 | 10/12/05 01:54:25 INFO SimpleJob: Finished TID 2 (progress: 4/16) 18 | 10/12/05 01:54:25 INFO SimpleJob: Starting task 0:7 as TID 7 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/05 01:54:25 INFO SimpleJob: Finished TID 0 (progress: 5/16) 20 | 10/12/05 01:54:25 INFO SimpleJob: Starting task 0:8 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 21 | 10/12/05 01:54:26 INFO SimpleJob: Finished TID 8 (progress: 6/16) 22 | 10/12/05 01:54:26 INFO SimpleJob: Starting task 0:9 as TID 9 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 23 | 10/12/05 01:54:30 INFO SimpleJob: Finished TID 4 (progress: 7/16) 24 | 10/12/05 01:54:30 INFO SimpleJob: Starting task 0:10 as TID 10 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 25 | 10/12/05 01:54:30 INFO SimpleJob: Finished TID 6 (progress: 8/16) 26 | 10/12/05 01:54:30 INFO SimpleJob: Starting task 0:11 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 27 | 10/12/05 01:54:30 INFO SimpleJob: Finished TID 11 (progress: 9/16) 28 | 10/12/05 01:54:30 INFO SimpleJob: Starting task 0:12 as TID 12 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 29 | 10/12/05 01:54:31 INFO SimpleJob: Finished TID 10 (progress: 10/16) 30 | 10/12/05 01:54:31 INFO SimpleJob: Starting task 0:13 as TID 13 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 31 | 10/12/05 01:54:31 INFO SimpleJob: Finished TID 12 (progress: 11/16) 32 | 10/12/05 01:54:31 INFO SimpleJob: Starting task 0:14 as TID 14 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 33 | 10/12/05 01:54:31 INFO SimpleJob: Finished TID 13 (progress: 12/16) 34 | 10/12/05 01:54:31 INFO SimpleJob: Starting task 0:15 as TID 15 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 35 | 10/12/05 01:54:31 INFO SimpleJob: Finished TID 15 (progress: 13/16) 36 | 10/12/05 01:54:32 INFO SimpleJob: Finished TID 14 (progress: 14/16) 37 | 10/12/05 01:54:38 INFO SimpleJob: Finished TID 9 (progress: 15/16) 38 | 10/12/05 01:54:38 INFO SimpleJob: Finished TID 7 (progress: 16/16) 39 | 10/12/05 01:54:38 INFO SparkContext: Tasks finished in 47.81986464 s 40 | Read 80000000 points. 41 | 10/12/05 01:54:38 INFO SparkContext: Running 16 tasks in parallel 42 | 10/12/05 01:54:38 INFO SimpleJob: Starting task 1:0 as TID 16 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 43 | 10/12/05 01:54:38 INFO SimpleJob: Starting task 1:1 as TID 17 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 44 | 10/12/05 01:54:38 INFO SimpleJob: Starting task 1:2 as TID 18 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 45 | 10/12/05 01:54:38 INFO SimpleJob: Starting task 1:3 as TID 19 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 46 | 10/12/05 01:54:43 INFO SimpleJob: Finished TID 17 (progress: 1/16) 47 | 10/12/05 01:54:43 INFO SimpleJob: Starting task 1:4 as TID 20 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 48 | 10/12/05 01:54:44 INFO SimpleJob: Finished TID 19 (progress: 2/16) 49 | 10/12/05 01:54:44 INFO SimpleJob: Starting task 1:5 as TID 21 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 50 | 10/12/05 01:54:46 INFO SimpleJob: Finished TID 18 (progress: 3/16) 51 | 10/12/05 01:54:46 INFO SimpleJob: Starting task 1:7 as TID 22 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 52 | 10/12/05 01:54:46 INFO SimpleJob: Finished TID 16 (progress: 4/16) 53 | 10/12/05 01:54:46 INFO SimpleJob: Starting task 1:8 as TID 23 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 54 | 10/12/05 01:54:48 INFO SimpleJob: Finished TID 21 (progress: 5/16) 55 | 10/12/05 01:54:48 INFO SimpleJob: Starting task 1:6 as TID 24 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 56 | 10/12/05 01:54:50 INFO SimpleJob: Finished TID 22 (progress: 6/16) 57 | 10/12/05 01:54:50 INFO SimpleJob: Starting task 1:9 as TID 25 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 58 | 10/12/05 01:54:52 INFO SimpleJob: Finished TID 20 (progress: 7/16) 59 | 10/12/05 01:54:52 INFO SimpleJob: Starting task 1:10 as TID 26 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 60 | 10/12/05 01:54:53 INFO SimpleJob: Finished TID 25 (progress: 8/16) 61 | 10/12/05 01:54:54 INFO SimpleJob: Finished TID 23 (progress: 9/16) 62 | 10/12/05 01:54:57 INFO SimpleJob: Starting task 1:11 as TID 27 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 63 | 10/12/05 01:54:57 INFO SimpleJob: Starting task 1:12 as TID 28 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 64 | 10/12/05 01:54:57 INFO SimpleJob: Finished TID 24 (progress: 10/16) 65 | 10/12/05 01:54:57 INFO SimpleJob: Starting task 1:13 as TID 29 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 66 | 10/12/05 01:55:01 INFO SimpleJob: Finished TID 27 (progress: 11/16) 67 | 10/12/05 01:55:01 INFO SimpleJob: Starting task 1:14 as TID 30 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 68 | 10/12/05 01:55:01 INFO SimpleJob: Finished TID 29 (progress: 12/16) 69 | 10/12/05 01:55:01 INFO SimpleJob: Starting task 1:15 as TID 31 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 70 | 10/12/05 01:55:01 INFO SimpleJob: Finished TID 26 (progress: 13/16) 71 | 10/12/05 01:55:04 INFO SimpleJob: Finished TID 31 (progress: 14/16) 72 | 10/12/05 01:55:05 INFO SimpleJob: Finished TID 28 (progress: 15/16) 73 | 10/12/05 01:55:08 INFO SimpleJob: Finished TID 30 (progress: 16/16) 74 | 10/12/05 01:55:08 INFO SparkContext: Tasks finished in 29.578870389 s 75 | Centroids changed by 76 | (1.020119, 0.433392, 2.183411) 77 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 78 | 10/12/05 01:55:08 INFO SparkContext: Running 16 tasks in parallel 79 | 10/12/05 01:55:08 INFO SimpleJob: Starting task 2:0 as TID 32 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 80 | 10/12/05 01:55:08 INFO SimpleJob: Starting task 2:1 as TID 33 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 81 | 10/12/05 01:55:08 INFO SimpleJob: Starting task 2:2 as TID 34 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 82 | 10/12/05 01:55:08 INFO SimpleJob: Starting task 2:3 as TID 35 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 83 | 10/12/05 01:55:12 INFO SimpleJob: Finished TID 33 (progress: 1/16) 84 | 10/12/05 01:55:12 INFO SimpleJob: Starting task 2:4 as TID 36 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 85 | 10/12/05 01:55:12 INFO SimpleJob: Finished TID 35 (progress: 2/16) 86 | 10/12/05 01:55:12 INFO SimpleJob: Starting task 2:5 as TID 37 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 87 | 10/12/05 01:55:16 INFO SimpleJob: Finished TID 32 (progress: 3/16) 88 | 10/12/05 01:55:16 INFO SimpleJob: Starting task 2:7 as TID 38 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 89 | 10/12/05 01:55:16 INFO SimpleJob: Finished TID 34 (progress: 4/16) 90 | 10/12/05 01:55:16 INFO SimpleJob: Starting task 2:8 as TID 39 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 91 | 10/12/05 01:55:16 INFO SimpleJob: Finished TID 37 (progress: 5/16) 92 | 10/12/05 01:55:16 INFO SimpleJob: Starting task 2:6 as TID 40 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 93 | 10/12/05 01:55:20 INFO SimpleJob: Finished TID 38 (progress: 6/16) 94 | 10/12/05 01:55:20 INFO SimpleJob: Starting task 2:9 as TID 41 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 95 | 10/12/05 01:55:21 INFO SimpleJob: Finished TID 36 (progress: 7/16) 96 | 10/12/05 01:55:21 INFO SimpleJob: Starting task 2:10 as TID 42 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 97 | 10/12/05 01:55:24 INFO SimpleJob: Finished TID 41 (progress: 8/16) 98 | 10/12/05 01:55:24 INFO SimpleJob: Starting task 2:11 as TID 43 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 99 | 10/12/05 01:55:25 INFO SimpleJob: Finished TID 39 (progress: 9/16) 100 | 10/12/05 01:55:25 INFO SimpleJob: Starting task 2:12 as TID 44 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 101 | 10/12/05 01:55:27 INFO SimpleJob: Finished TID 40 (progress: 10/16) 102 | 10/12/05 01:55:27 INFO SimpleJob: Starting task 2:13 as TID 45 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 103 | 10/12/05 01:55:28 INFO SimpleJob: Finished TID 43 (progress: 11/16) 104 | 10/12/05 01:55:28 INFO SimpleJob: Starting task 2:14 as TID 46 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 105 | 10/12/05 01:55:30 INFO SimpleJob: Finished TID 42 (progress: 12/16) 106 | 10/12/05 01:55:30 INFO SimpleJob: Starting task 2:15 as TID 47 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 107 | 10/12/05 01:55:32 INFO SimpleJob: Finished TID 45 (progress: 13/16) 108 | 10/12/05 01:55:33 INFO SimpleJob: Finished TID 47 (progress: 14/16) 109 | 10/12/05 01:55:34 INFO SimpleJob: Finished TID 44 (progress: 15/16) 110 | 10/12/05 01:55:36 INFO SimpleJob: Finished TID 46 (progress: 16/16) 111 | 10/12/05 01:55:36 INFO SparkContext: Tasks finished in 28.187332365 s 112 | Centroids changed by 113 | (0.000000, 0.000000, 0.000000) 114 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 115 | Final centroids: 116 | 7.776982 40.661016 117 | 44.751415 35.355014 118 | 17.872132 7.510652 119 | 120 | -------------------------------------------------------------------------------- /run-dataset-20101204T225747/3-1.log: -------------------------------------------------------------------------------- 1 | WARNING: Logging before InitGoogleLogging() is written to STDERR 2 | I1204 23:06:31.644093 4142 sched.cpp:181] New master at 1@10.202.70.163:5050 with ID:0 3 | 10/12/04 23:06:31 INFO MesosScheduler: Registered as framework ID 201012042306-0-0020 4 | 10/12/04 23:06:31 INFO BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/04 23:06:32 INFO FileInputFormat: Total input paths to process : 8 6 | 10/12/04 23:06:32 INFO SparkContext: Running 16 tasks in parallel 7 | 10/12/04 23:06:32 INFO SimpleJob: Starting task 0:0 as TID 0 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 8 | 10/12/04 23:06:32 INFO SimpleJob: Starting task 0:1 as TID 1 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 9 | 10/12/04 23:06:32 INFO SimpleJob: Starting task 0:2 as TID 2 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 10 | 10/12/04 23:06:32 INFO SimpleJob: Starting task 0:3 as TID 3 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 11 | 10/12/04 23:06:47 INFO SimpleJob: Finished TID 3 (progress: 1/16) 12 | 10/12/04 23:06:47 INFO SimpleJob: Starting task 0:4 as TID 4 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 13 | 10/12/04 23:06:47 INFO SimpleJob: Finished TID 1 (progress: 2/16) 14 | 10/12/04 23:06:47 INFO SimpleJob: Starting task 0:5 as TID 5 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 15 | 10/12/04 23:06:48 INFO SimpleJob: Finished TID 5 (progress: 3/16) 16 | 10/12/04 23:06:48 INFO SimpleJob: Starting task 0:6 as TID 6 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 17 | 10/12/04 23:07:06 INFO SimpleJob: Finished TID 0 (progress: 4/16) 18 | 10/12/04 23:07:06 INFO SimpleJob: Starting task 0:7 as TID 7 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 19 | 10/12/04 23:07:06 INFO SimpleJob: Finished TID 2 (progress: 5/16) 20 | 10/12/04 23:07:06 INFO SimpleJob: Starting task 0:8 as TID 8 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 21 | 10/12/04 23:07:07 INFO SimpleJob: Finished TID 8 (progress: 6/16) 22 | 10/12/04 23:07:07 INFO SimpleJob: Starting task 0:9 as TID 9 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 23 | 10/12/04 23:07:12 INFO SimpleJob: Finished TID 4 (progress: 7/16) 24 | 10/12/04 23:07:12 INFO SimpleJob: Starting task 0:10 as TID 10 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 25 | 10/12/04 23:07:12 INFO SimpleJob: Finished TID 6 (progress: 8/16) 26 | 10/12/04 23:07:12 INFO SimpleJob: Starting task 0:11 as TID 11 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 27 | 10/12/04 23:07:12 INFO SimpleJob: Finished TID 11 (progress: 9/16) 28 | 10/12/04 23:07:12 INFO SimpleJob: Starting task 0:12 as TID 12 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 29 | 10/12/04 23:07:13 INFO SimpleJob: Finished TID 10 (progress: 10/16) 30 | 10/12/04 23:07:13 INFO SimpleJob: Starting task 0:13 as TID 13 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 31 | 10/12/04 23:07:13 INFO SimpleJob: Finished TID 12 (progress: 11/16) 32 | 10/12/04 23:07:13 INFO SimpleJob: Starting task 0:14 as TID 14 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 33 | 10/12/04 23:07:13 INFO SimpleJob: Finished TID 13 (progress: 12/16) 34 | 10/12/04 23:07:13 INFO SimpleJob: Starting task 0:15 as TID 15 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 35 | 10/12/04 23:07:13 INFO SimpleJob: Finished TID 15 (progress: 13/16) 36 | 10/12/04 23:07:14 INFO SimpleJob: Finished TID 14 (progress: 14/16) 37 | 10/12/04 23:07:19 INFO SimpleJob: Finished TID 7 (progress: 15/16) 38 | 10/12/04 23:07:19 INFO SimpleJob: Finished TID 9 (progress: 16/16) 39 | 10/12/04 23:07:19 INFO SparkContext: Tasks finished in 47.488080548 s 40 | Read 80000000 points. 41 | 10/12/04 23:07:19 INFO SparkContext: Running 16 tasks in parallel 42 | 10/12/04 23:07:19 INFO SimpleJob: Starting task 1:0 as TID 16 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 43 | 10/12/04 23:07:19 INFO SimpleJob: Starting task 1:1 as TID 17 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 44 | 10/12/04 23:07:19 INFO SimpleJob: Starting task 1:2 as TID 18 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 45 | 10/12/04 23:07:19 INFO SimpleJob: Starting task 1:3 as TID 19 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 46 | 10/12/04 23:07:23 INFO SimpleJob: Finished TID 19 (progress: 1/16) 47 | 10/12/04 23:07:23 INFO SimpleJob: Starting task 1:4 as TID 20 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 48 | 10/12/04 23:07:23 INFO SimpleJob: Finished TID 17 (progress: 2/16) 49 | 10/12/04 23:07:23 INFO SimpleJob: Starting task 1:5 as TID 21 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 50 | 10/12/04 23:07:27 INFO SimpleJob: Finished TID 18 (progress: 3/16) 51 | 10/12/04 23:07:27 INFO SimpleJob: Starting task 1:7 as TID 22 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 52 | 10/12/04 23:07:27 INFO SimpleJob: Finished TID 16 (progress: 4/16) 53 | 10/12/04 23:07:27 INFO SimpleJob: Starting task 1:8 as TID 23 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 54 | 10/12/04 23:07:27 INFO SimpleJob: Finished TID 21 (progress: 5/16) 55 | 10/12/04 23:07:27 INFO SimpleJob: Starting task 1:6 as TID 24 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 56 | 10/12/04 23:07:31 INFO SimpleJob: Finished TID 22 (progress: 6/16) 57 | 10/12/04 23:07:31 INFO SimpleJob: Starting task 1:9 as TID 25 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 58 | 10/12/04 23:07:32 INFO SimpleJob: Finished TID 20 (progress: 7/16) 59 | 10/12/04 23:07:32 INFO SimpleJob: Starting task 1:10 as TID 26 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 60 | 10/12/04 23:07:34 INFO SimpleJob: Finished TID 25 (progress: 8/16) 61 | 10/12/04 23:07:35 INFO SimpleJob: Finished TID 23 (progress: 9/16) 62 | 10/12/04 23:07:36 INFO SimpleJob: Starting task 1:11 as TID 27 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 63 | 10/12/04 23:07:36 INFO SimpleJob: Starting task 1:12 as TID 28 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 64 | 10/12/04 23:07:36 INFO SimpleJob: Finished TID 24 (progress: 10/16) 65 | 10/12/04 23:07:36 INFO SimpleJob: Starting task 1:13 as TID 29 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 66 | 10/12/04 23:07:40 INFO SimpleJob: Finished TID 27 (progress: 11/16) 67 | 10/12/04 23:07:40 INFO SimpleJob: Starting task 1:14 as TID 30 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 68 | 10/12/04 23:07:40 INFO SimpleJob: Finished TID 29 (progress: 12/16) 69 | 10/12/04 23:07:40 INFO SimpleJob: Starting task 1:15 as TID 31 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 70 | 10/12/04 23:07:40 INFO SimpleJob: Finished TID 26 (progress: 13/16) 71 | 10/12/04 23:07:42 INFO SimpleJob: Finished TID 31 (progress: 14/16) 72 | 10/12/04 23:07:44 INFO SimpleJob: Finished TID 28 (progress: 15/16) 73 | 10/12/04 23:07:47 INFO SimpleJob: Finished TID 30 (progress: 16/16) 74 | 10/12/04 23:07:47 INFO SparkContext: Tasks finished in 27.445370504 s 75 | Centroids changed by 76 | (1.020119, 0.433392, 2.183411) 77 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 78 | 10/12/04 23:07:47 INFO SparkContext: Running 16 tasks in parallel 79 | 10/12/04 23:07:47 INFO SimpleJob: Starting task 2:0 as TID 32 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 80 | 10/12/04 23:07:47 INFO SimpleJob: Starting task 2:1 as TID 33 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 81 | 10/12/04 23:07:47 INFO SimpleJob: Starting task 2:2 as TID 34 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 82 | 10/12/04 23:07:47 INFO SimpleJob: Starting task 2:3 as TID 35 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 83 | 10/12/04 23:07:51 INFO SimpleJob: Finished TID 33 (progress: 1/16) 84 | 10/12/04 23:07:51 INFO SimpleJob: Starting task 2:4 as TID 36 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 85 | 10/12/04 23:07:51 INFO SimpleJob: Finished TID 35 (progress: 2/16) 86 | 10/12/04 23:07:51 INFO SimpleJob: Starting task 2:5 as TID 37 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 87 | 10/12/04 23:07:55 INFO SimpleJob: Finished TID 37 (progress: 3/16) 88 | 10/12/04 23:07:55 INFO SimpleJob: Starting task 2:6 as TID 38 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 89 | 10/12/04 23:07:55 INFO SimpleJob: Finished TID 32 (progress: 4/16) 90 | 10/12/04 23:07:55 INFO SimpleJob: Starting task 2:7 as TID 39 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 91 | 10/12/04 23:07:55 INFO SimpleJob: Finished TID 34 (progress: 5/16) 92 | 10/12/04 23:07:55 INFO SimpleJob: Starting task 2:8 as TID 40 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 93 | 10/12/04 23:07:59 INFO SimpleJob: Finished TID 39 (progress: 6/16) 94 | 10/12/04 23:07:59 INFO SimpleJob: Starting task 2:9 as TID 41 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 95 | 10/12/04 23:08:00 INFO SimpleJob: Finished TID 36 (progress: 7/16) 96 | 10/12/04 23:08:00 INFO SimpleJob: Starting task 2:10 as TID 42 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 97 | 10/12/04 23:08:03 INFO SimpleJob: Finished TID 41 (progress: 8/16) 98 | 10/12/04 23:08:03 INFO SimpleJob: Starting task 2:11 as TID 43 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 99 | 10/12/04 23:08:04 INFO SimpleJob: Finished TID 38 (progress: 9/16) 100 | 10/12/04 23:08:04 INFO SimpleJob: Starting task 2:12 as TID 44 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 101 | 10/12/04 23:08:04 INFO SimpleJob: Finished TID 40 (progress: 10/16) 102 | 10/12/04 23:08:04 INFO SimpleJob: Starting task 2:14 as TID 45 on slave 0-0: ip-10-202-71-135.ec2.internal (preferred) 103 | 10/12/04 23:08:07 INFO SimpleJob: Finished TID 43 (progress: 11/16) 104 | 10/12/04 23:08:08 INFO SimpleJob: Starting task 2:13 as TID 46 on slave 0-0: ip-10-202-71-135.ec2.internal (non-preferred) 105 | 10/12/04 23:08:09 INFO SimpleJob: Finished TID 42 (progress: 12/16) 106 | 10/12/04 23:08:09 INFO SimpleJob: Starting task 2:15 as TID 47 on slave 0-1: ip-10-196-103-159.ec2.internal (preferred) 107 | 10/12/04 23:08:12 INFO SimpleJob: Finished TID 46 (progress: 13/16) 108 | 10/12/04 23:08:12 INFO SimpleJob: Finished TID 45 (progress: 14/16) 109 | 10/12/04 23:08:13 INFO SimpleJob: Finished TID 47 (progress: 15/16) 110 | 10/12/04 23:08:13 INFO SimpleJob: Finished TID 44 (progress: 16/16) 111 | 10/12/04 23:08:13 INFO SparkContext: Tasks finished in 26.479855753 s 112 | Centroids changed by 113 | (0.000000, 0.000000, 0.000000) 114 | to ((7.78,40.66), (44.75,35.36), (17.87,7.51)) 115 | Final centroids: 116 | 7.776982 40.661016 117 | 44.751415 35.355014 118 | 17.872132 7.510652 119 | 120 | -------------------------------------------------------------------------------- /run-scaleout-20101205T231425/2-3.log: -------------------------------------------------------------------------------- 1 | stdin: is not a tty 2 | WARNING: Logging before InitGoogleLogging() is written to STDERR 3 | I1206 08:55:26.248726 2850 sched.cpp:181] New master at 1@10.244.130.127:5050 with ID:0 4 | 10/12/06 08:55:26 INFO spark.BoundedMemoryCache: BoundedMemoryCache.maxBytes = 3087007744 5 | 10/12/06 08:55:26 INFO spark.MesosScheduler: Registered as framework ID 201012060855-0-0002 6 | 10/12/06 08:55:26 INFO mapred.FileInputFormat: Total input paths to process : 1 7 | 10/12/06 08:55:26 INFO spark.SparkContext: Running 15 tasks in parallel 8 | 10/12/06 08:55:26 INFO spark.SimpleJob: Starting task 0:0 as TID 0 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 9 | 10/12/06 08:55:26 INFO spark.SimpleJob: Starting task 0:2 as TID 1 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 10 | 10/12/06 08:55:26 INFO spark.SimpleJob: Starting task 0:1 as TID 2 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 11 | 10/12/06 08:55:26 INFO spark.SimpleJob: Starting task 0:3 as TID 3 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 12 | 10/12/06 08:55:26 INFO spark.SimpleJob: Starting task 0:4 as TID 4 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 13 | 10/12/06 08:55:26 INFO spark.SimpleJob: Starting task 0:6 as TID 5 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 14 | 10/12/06 08:56:06 INFO spark.SimpleJob: Finished TID 5 (progress: 1/15) 15 | 10/12/06 08:56:06 INFO spark.SimpleJob: Starting task 0:8 as TID 6 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 16 | 10/12/06 08:56:06 INFO spark.SimpleJob: Finished TID 2 (progress: 2/15) 17 | 10/12/06 08:56:06 INFO spark.SimpleJob: Starting task 0:11 as TID 7 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 18 | 10/12/06 08:56:09 INFO spark.SimpleJob: Finished TID 3 (progress: 3/15) 19 | 10/12/06 08:56:09 INFO spark.SimpleJob: Starting task 0:5 as TID 8 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 20 | 10/12/06 08:56:10 INFO spark.SimpleJob: Finished TID 0 (progress: 4/15) 21 | 10/12/06 08:56:10 INFO spark.SimpleJob: Starting task 0:7 as TID 9 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 22 | 10/12/06 08:56:11 INFO spark.SimpleJob: Finished TID 1 (progress: 5/15) 23 | 10/12/06 08:56:11 INFO spark.SimpleJob: Starting task 0:9 as TID 10 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 24 | 10/12/06 08:56:11 INFO spark.SimpleJob: Finished TID 4 (progress: 6/15) 25 | 10/12/06 08:56:11 INFO spark.SimpleJob: Starting task 0:10 as TID 11 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 26 | 10/12/06 08:56:45 INFO spark.SimpleJob: Finished TID 6 (progress: 7/15) 27 | 10/12/06 08:56:45 INFO spark.SimpleJob: Starting task 0:12 as TID 12 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 28 | 10/12/06 08:56:45 INFO spark.SimpleJob: Finished TID 7 (progress: 8/15) 29 | 10/12/06 08:56:45 INFO spark.SimpleJob: Starting task 0:13 as TID 13 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 30 | 10/12/06 08:56:50 INFO spark.SimpleJob: Finished TID 10 (progress: 9/15) 31 | 10/12/06 08:56:50 INFO spark.SimpleJob: Starting task 0:14 as TID 14 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 32 | 10/12/06 08:56:50 INFO spark.SimpleJob: Finished TID 11 (progress: 10/15) 33 | 10/12/06 08:56:50 INFO spark.SimpleJob: Finished TID 9 (progress: 11/15) 34 | 10/12/06 08:56:51 INFO spark.SimpleJob: Finished TID 8 (progress: 12/15) 35 | 10/12/06 08:57:04 INFO spark.SimpleJob: Finished TID 14 (progress: 13/15) 36 | 10/12/06 08:57:26 INFO spark.SimpleJob: Finished TID 12 (progress: 14/15) 37 | 10/12/06 08:57:27 INFO spark.SimpleJob: Finished TID 13 (progress: 15/15) 38 | 10/12/06 08:57:27 INFO spark.SparkContext: Tasks finished in 120.874912596 s 39 | Read 100000000 points. 40 | 10/12/06 08:57:27 INFO spark.SparkContext: Running 15 tasks in parallel 41 | 10/12/06 08:57:27 INFO spark.SimpleJob: Starting task 1:0 as TID 15 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 42 | 10/12/06 08:57:27 INFO spark.SimpleJob: Starting task 1:2 as TID 16 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 43 | 10/12/06 08:57:27 INFO spark.SimpleJob: Starting task 1:1 as TID 17 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 44 | 10/12/06 08:57:27 INFO spark.SimpleJob: Starting task 1:3 as TID 18 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 45 | 10/12/06 08:57:27 INFO spark.SimpleJob: Starting task 1:4 as TID 19 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 46 | 10/12/06 08:57:27 INFO spark.SimpleJob: Starting task 1:6 as TID 20 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 47 | 10/12/06 08:57:38 INFO spark.SimpleJob: Finished TID 16 (progress: 1/15) 48 | 10/12/06 08:57:38 INFO spark.SimpleJob: Starting task 1:9 as TID 21 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 49 | 10/12/06 08:57:38 INFO spark.SimpleJob: Finished TID 19 (progress: 2/15) 50 | 10/12/06 08:57:38 INFO spark.SimpleJob: Starting task 1:10 as TID 22 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 51 | 10/12/06 08:57:40 INFO spark.SimpleJob: Finished TID 18 (progress: 3/15) 52 | 10/12/06 08:57:40 INFO spark.SimpleJob: Starting task 1:5 as TID 23 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 53 | 10/12/06 08:57:42 INFO spark.SimpleJob: Finished TID 15 (progress: 4/15) 54 | 10/12/06 08:57:42 INFO spark.SimpleJob: Starting task 1:7 as TID 24 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 55 | 10/12/06 08:57:48 INFO spark.SimpleJob: Finished TID 21 (progress: 5/15) 56 | 10/12/06 08:57:48 INFO spark.SimpleJob: Starting task 1:14 as TID 25 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 57 | 10/12/06 08:57:48 INFO spark.SimpleJob: Finished TID 22 (progress: 6/15) 58 | 10/12/06 08:57:48 INFO spark.SimpleJob: Finished TID 17 (progress: 7/15) 59 | 10/12/06 08:57:48 INFO spark.SimpleJob: Starting task 1:8 as TID 26 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 60 | 10/12/06 08:57:48 INFO spark.SimpleJob: Finished TID 20 (progress: 8/15) 61 | 10/12/06 08:57:48 INFO spark.SimpleJob: Starting task 1:11 as TID 27 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 62 | 10/12/06 08:57:51 INFO spark.SimpleJob: Finished TID 25 (progress: 9/15) 63 | 10/12/06 08:57:51 INFO spark.SimpleJob: Starting task 1:12 as TID 28 on slave 0-0: ip-10-244-129-188.ec2.internal (non-preferred) 64 | 10/12/06 08:57:51 INFO spark.SimpleJob: Starting task 1:13 as TID 29 on slave 0-0: ip-10-244-129-188.ec2.internal (non-preferred) 65 | 10/12/06 08:57:56 INFO spark.SimpleJob: Finished TID 24 (progress: 10/15) 66 | 10/12/06 08:57:56 INFO spark.SimpleJob: Finished TID 23 (progress: 11/15) 67 | 10/12/06 08:57:58 INFO spark.SimpleJob: Finished TID 26 (progress: 12/15) 68 | 10/12/06 08:57:58 INFO spark.SimpleJob: Finished TID 27 (progress: 13/15) 69 | 10/12/06 08:58:55 INFO spark.SimpleJob: Finished TID 29 (progress: 14/15) 70 | 10/12/06 08:58:58 INFO spark.SimpleJob: Finished TID 28 (progress: 15/15) 71 | 10/12/06 08:58:58 INFO spark.SparkContext: Tasks finished in 91.02753719 s 72 | Centroids changed by 73 | (2.255668, 26.978290, 27.152732) 74 | to ((6.17,42.10), (48.24,8.22), (47.14,7.02)) 75 | 10/12/06 08:58:58 INFO spark.SparkContext: Running 15 tasks in parallel 76 | 10/12/06 08:58:58 INFO spark.SimpleJob: Starting task 2:0 as TID 30 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 77 | 10/12/06 08:58:58 INFO spark.SimpleJob: Starting task 2:2 as TID 31 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 78 | 10/12/06 08:58:58 INFO spark.SimpleJob: Starting task 2:1 as TID 32 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 79 | 10/12/06 08:58:58 INFO spark.SimpleJob: Starting task 2:3 as TID 33 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 80 | 10/12/06 08:58:58 INFO spark.SimpleJob: Starting task 2:4 as TID 34 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 81 | 10/12/06 08:58:58 INFO spark.SimpleJob: Starting task 2:6 as TID 35 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 82 | 10/12/06 08:59:09 INFO spark.SimpleJob: Finished TID 34 (progress: 1/15) 83 | 10/12/06 08:59:09 INFO spark.SimpleJob: Starting task 2:9 as TID 36 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 84 | 10/12/06 08:59:09 INFO spark.SimpleJob: Finished TID 31 (progress: 2/15) 85 | 10/12/06 08:59:09 INFO spark.SimpleJob: Starting task 2:10 as TID 37 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 86 | 10/12/06 08:59:09 INFO spark.SimpleJob: Finished TID 32 (progress: 3/15) 87 | 10/12/06 08:59:09 INFO spark.SimpleJob: Starting task 2:8 as TID 38 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 88 | 10/12/06 08:59:09 INFO spark.SimpleJob: Finished TID 35 (progress: 4/15) 89 | 10/12/06 08:59:09 INFO spark.SimpleJob: Starting task 2:11 as TID 39 on slave 0-1: ip-10-195-97-235.ec2.internal (preferred) 90 | 10/12/06 08:59:12 INFO spark.SimpleJob: Finished TID 30 (progress: 5/15) 91 | 10/12/06 08:59:12 INFO spark.SimpleJob: Starting task 2:5 as TID 40 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 92 | 10/12/06 08:59:13 INFO spark.SimpleJob: Finished TID 33 (progress: 6/15) 93 | 10/12/06 08:59:13 INFO spark.SimpleJob: Starting task 2:7 as TID 41 on slave 0-2: ip-10-99-87-160.ec2.internal (preferred) 94 | 10/12/06 08:59:20 INFO spark.SimpleJob: Finished TID 37 (progress: 7/15) 95 | 10/12/06 08:59:20 INFO spark.SimpleJob: Starting task 2:12 as TID 42 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 96 | 10/12/06 08:59:20 INFO spark.SimpleJob: Finished TID 36 (progress: 8/15) 97 | 10/12/06 08:59:20 INFO spark.SimpleJob: Starting task 2:13 as TID 43 on slave 0-0: ip-10-244-129-188.ec2.internal (preferred) 98 | 10/12/06 08:59:20 INFO spark.SimpleJob: Finished TID 38 (progress: 9/15) 99 | 10/12/06 08:59:20 INFO spark.SimpleJob: Finished TID 39 (progress: 10/15) 100 | 10/12/06 08:59:24 INFO spark.SimpleJob: Starting task 2:14 as TID 44 on slave 0-1: ip-10-195-97-235.ec2.internal (non-preferred) 101 | 10/12/06 08:59:25 INFO spark.SimpleJob: Finished TID 40 (progress: 11/15) 102 | 10/12/06 08:59:26 INFO spark.SimpleJob: Finished TID 41 (progress: 12/15) 103 | 10/12/06 08:59:31 INFO spark.SimpleJob: Finished TID 42 (progress: 13/15) 104 | 10/12/06 08:59:31 INFO spark.SimpleJob: Finished TID 43 (progress: 14/15) 105 | 10/12/06 08:59:41 INFO spark.SimpleJob: Finished TID 44 (progress: 15/15) 106 | 10/12/06 08:59:41 INFO spark.SparkContext: Tasks finished in 42.40061315 s 107 | Centroids changed by 108 | (0.000000, 0.076094, 0.093685) 109 | to ((6.17,42.10), (48.29,8.27), (47.20,7.09))6.173993 42.098988 110 | 48.291269 8.272699 111 | 47.198808 7.093458 112 | 113 | 114 | Final centroids: 115 | --------------------------------------------------------------------------------