├── src
    └── main
    │   └── scala
    │       ├── som
    │           ├── SOMTrainingSummary.scala
    │           ├── SOMModel.scala
    │           ├── SOMParams.scala
    │           └── SOM.scala
    │       ├── util
    │           ├── MLUtils.scala
    │           └── SchemaUtils.scala
    │       └── linalg
    │           └── BLAS.scala
├── README.md
└── LICENSE


/src/main/scala/som/SOMTrainingSummary.scala:
--------------------------------------------------------------------------------
 1 | package xyz.florentforest.spark.ml.som
 2 | 
 3 | import org.apache.spark.sql.DataFrame
 4 | 
 5 | class SOMTrainingSummary(val predictions: DataFrame,
 6 |                          val predictionCol: String,
 7 |                          val featuresCol: String,
 8 |                          val height: Int,
 9 |                          val width: Int,
10 |                          val tMax: Double,
11 |                          val tMin: Double,
12 |                          val maxIter: Int,
13 |                          val tol: Double,
14 |                          val topology: String,
15 |                          val neighborhoodKernel: String,
16 |                          val temperatureDecay: String,
17 |                          val trainingCost: Double,
18 |                          val objectiveHistory: Array[Double]) extends Serializable
19 | 


--------------------------------------------------------------------------------
/src/main/scala/util/MLUtils.scala:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Licensed to the Apache Software Foundation (ASF) under one or more
 3 |  * contributor license agreements.  See the NOTICE file distributed with
 4 |  * this work for additional information regarding copyright ownership.
 5 |  * The ASF licenses this file to You under the Apache License, Version 2.0
 6 |  * (the "License"); you may not use this file except in compliance with
 7 |  * the License.  You may obtain a copy of the License at
 8 |  *
 9 |  *    http://www.apache.org/licenses/LICENSE-2.0
10 |  *
11 |  * Unless required by applicable law or agreed to in writing, software
12 |  * distributed under the License is distributed on an "AS IS" BASIS,
13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 |  * See the License for the specific language governing permissions and
15 |  * limitations under the License.
16 |  */
17 | 
18 | package xyz.florentforest.spark.ml.util
19 | 
20 | import org.apache.spark.ml.linalg.{SparseVector, Vector, Vectors}
21 | import xyz.florentforest.spark.ml.linalg.BLAS.dot
22 | 
23 | object MLUtils {
24 | 
25 |   /**
26 |     * Machine precision
27 |     */
28 |   lazy val EPSILON = {
29 |     var eps = 1.0
30 |     while ((1.0 + (eps / 2.0)) != 1.0) {
31 |       eps /= 2.0
32 |     }
33 |     eps
34 |   }
35 | 
36 |   /**
37 |     * Returns the squared Euclidean distance between two vectors. The following formula will be used
38 |     * if it does not introduce too much numerical error:
39 |     * <pre>
40 |     *   \|a - b\|_2^2 = \|a\|_2^2 + \|b\|_2^2 - 2 a^T b.
41 |     * </pre>
42 |     * When both vector norms are given, this is faster than computing the squared distance directly,
43 |     * especially when one of the vectors is a sparse vector.
44 |     * @param v1 the first vector
45 |     * @param norm1 the norm of the first vector, non-negative
46 |     * @param v2 the second vector
47 |     * @param norm2 the norm of the second vector, non-negative
48 |     * @param precision desired relative precision for the squared distance
49 |     * @return squared distance between v1 and v2 within the specified precision
50 |     */
51 |   def fastSquaredDistance(v1: Vector,
52 |                           norm1: Double,
53 |                           v2: Vector,
54 |                           norm2: Double,
55 |                           precision: Double = 1e-6): Double = {
56 |     val n = v1.size
57 |     require(v2.size == n)
58 |     require(norm1 >= 0.0 && norm2 >= 0.0)
59 |     val sumSquaredNorm = norm1 * norm1 + norm2 * norm2
60 |     val normDiff = norm1 - norm2
61 |     var sqDist = 0.0
62 |     /*
63 |      * The relative error is
64 |      * <pre>
65 |      * EPSILON * ( \|a\|_2^2 + \|b\\_2^2 + 2 |a^T b|) / ( \|a - b\|_2^2 ),
66 |      * </pre>
67 |      * which is bounded by
68 |      * <pre>
69 |      * 2.0 * EPSILON * ( \|a\|_2^2 + \|b\|_2^2 ) / ( (\|a\|_2 - \|b\|_2)^2 ).
70 |      * </pre>
71 |      * The bound doesn't need the inner product, so we can use it as a sufficient condition to
72 |      * check quickly whether the inner product approach is accurate.
73 |      */
74 |     val precisionBound1 = 2.0 * EPSILON * sumSquaredNorm / (normDiff * normDiff + EPSILON)
75 |     if (precisionBound1 < precision) {
76 |       sqDist = sumSquaredNorm - 2.0 * dot(v1, v2)
77 |     } else if (v1.isInstanceOf[SparseVector] || v2.isInstanceOf[SparseVector]) {
78 |       val dotValue = dot(v1, v2)
79 |       sqDist = math.max(sumSquaredNorm - 2.0 * dotValue, 0.0)
80 |       val precisionBound2 = EPSILON * (sumSquaredNorm + 2.0 * math.abs(dotValue)) /
81 |         (sqDist + EPSILON)
82 |       if (precisionBound2 > precision) {
83 |         sqDist = Vectors.sqdist(v1, v2)
84 |       }
85 |     } else {
86 |       sqDist = Vectors.sqdist(v1, v2)
87 |     }
88 |     sqDist
89 |   }
90 | 
91 | }
92 | 


--------------------------------------------------------------------------------
/src/main/scala/som/SOMModel.scala:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Licensed to the Apache Software Foundation (ASF) under one or more
 3 |  * contributor license agreements.  See the NOTICE file distributed with
 4 |  * this work for additional information regarding copyright ownership.
 5 |  * The ASF licenses this file to You under the Apache License, Version 2.0
 6 |  * (the "License"); you may not use this file except in compliance with
 7 |  * the License.  You may obtain a copy of the License at
 8 |  *
 9 |  *    http://www.apache.org/licenses/LICENSE-2.0
10 |  *
11 |  * Unless required by applicable law or agreed to in writing, software
12 |  * distributed under the License is distributed on an "AS IS" BASIS,
13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 |  * See the License for the specific language governing permissions and
15 |  * limitations under the License.
16 |  */
17 | 
18 | package xyz.florentforest.spark.ml.som
19 | 
20 | import org.apache.spark.ml.Model
21 | import org.apache.spark.ml.linalg.Vector
22 | import org.apache.spark.ml.param.ParamMap
23 | import org.apache.spark.sql.functions.{col, udf}
24 | import org.apache.spark.sql.types.StructType
25 | import org.apache.spark.sql.{DataFrame, Dataset, Row}
26 | 
27 | class SOMModel(override val uid: String, val prototypes: Array[Vector]) extends Model[SOMModel] with SOMParams {
28 | 
29 |   private val prototypesWithNorm =
30 |     if (prototypes == null) null else prototypes.map(new VectorWithNorm(_))
31 | 
32 |   private var trainingCost: Option[Double] = None
33 | 
34 |   def cost: Double = trainingCost.getOrElse {
35 |     throw new Exception("No training cost available for this SOMModel")
36 |   }
37 | 
38 |   def setCost(cost: Option[Double]): this.type = {
39 |     this.trainingCost = cost
40 |     this
41 |   }
42 | 
43 |   private var objectiveHistory: Option[Array[Double]] = None
44 | 
45 |   def history: Array[Double] = objectiveHistory.getOrElse {
46 |     throw new Exception("No objective history available for this SOMModel")
47 |   }
48 | 
49 |   def setHistory(history: Option[Array[Double]]): this.type = {
50 |     this.objectiveHistory = history
51 |     this
52 |   }
53 | 
54 |   private var trainingSummary: Option[SOMTrainingSummary] = None
55 | 
56 |   def summary: SOMTrainingSummary = trainingSummary.getOrElse {
57 |     throw new Exception("No training summary available for this SOMModel")
58 |   }
59 | 
60 |   def setSummary(summary: Option[SOMTrainingSummary]): this.type = {
61 |     this.trainingSummary = summary
62 |     this
63 |   }
64 | 
65 |   def hasSummary: Boolean = trainingSummary.isDefined
66 | 
67 |   override def copy(extra: ParamMap): SOMModel = {
68 |     val newModel = copyValues(new SOMModel(uid, prototypes), extra)
69 |     newModel.setSummary(trainingSummary).setParent(parent)
70 |   }
71 | 
72 |   def setFeaturesCol(value: String): this.type = set(featuresCol, value)
73 | 
74 |   def setPredictionCol(value: String): this.type = set(predictionCol, value)
75 | 
76 |   def transform(dataset: Dataset[_]): DataFrame = {
77 |     transformSchema(dataset.schema, logging = true)
78 | 
79 |     val predictUDF = udf((vector: Vector) => predict(vector))
80 | 
81 |     dataset.withColumn($(predictionCol), predictUDF(col($(featuresCol))))
82 |   }
83 | 
84 |   override def transformSchema(schema: StructType): StructType = {
85 |     validateAndTransformSchema(schema)
86 |   }
87 | 
88 |   def predict(features: Vector): Int = {
89 |     SOM.findClosest(prototypesWithNorm, new VectorWithNorm(features))._1
90 |   }
91 | 
92 |   def computeCost(dataset: Dataset[_]): Double = {
93 |     val bcPrototypesWithNorm = dataset.sparkSession.sparkContext.broadcast(prototypesWithNorm)
94 |     dataset.select(col($(featuresCol))).rdd.map {
95 |       case Row(point: Vector) => SOM.pointCost(bcPrototypesWithNorm.value, new VectorWithNorm(point))
96 |     }.sum()
97 |   }
98 | 
99 | }


--------------------------------------------------------------------------------
/src/main/scala/util/SchemaUtils.scala:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Licensed to the Apache Software Foundation (ASF) under one or more
  3 |  * contributor license agreements.  See the NOTICE file distributed with
  4 |  * this work for additional information regarding copyright ownership.
  5 |  * The ASF licenses this file to You under the Apache License, Version 2.0
  6 |  * (the "License"); you may not use this file except in compliance with
  7 |  * the License.  You may obtain a copy of the License at
  8 |  *
  9 |  *    http://www.apache.org/licenses/LICENSE-2.0
 10 |  *
 11 |  * Unless required by applicable law or agreed to in writing, software
 12 |  * distributed under the License is distributed on an "AS IS" BASIS,
 13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 14 |  * See the License for the specific language governing permissions and
 15 |  * limitations under the License.
 16 |  */
 17 | 
 18 | package xyz.florentforest.spark.ml.util
 19 | 
 20 | import org.apache.spark.sql.types.{DataType, NumericType, StructField, StructType}
 21 | 
 22 | 
 23 | /**
 24 |   * Utils for handling schemas.
 25 |   */
 26 | object SchemaUtils {
 27 | 
 28 |   // TODO: Move the utility methods to SQL.
 29 | 
 30 |   /**
 31 |     * Check whether the given schema contains a column of the required data type.
 32 |     * @param colName  column name
 33 |     * @param dataType  required column data type
 34 |     */
 35 |   def checkColumnType(
 36 |                        schema: StructType,
 37 |                        colName: String,
 38 |                        dataType: DataType,
 39 |                        msg: String = ""): Unit = {
 40 |     val actualDataType = schema(colName).dataType
 41 |     val message = if (msg != null && msg.trim.length > 0) " " + msg else ""
 42 |     require(actualDataType.equals(dataType),
 43 |       s"Column $colName must be of type $dataType but was actually $actualDataType.$message")
 44 |   }
 45 | 
 46 |   /**
 47 |     * Check whether the given schema contains a column of one of the require data types.
 48 |     * @param colName  column name
 49 |     * @param dataTypes  required column data types
 50 |     */
 51 |   def checkColumnTypes(
 52 |                         schema: StructType,
 53 |                         colName: String,
 54 |                         dataTypes: Seq[DataType],
 55 |                         msg: String = ""): Unit = {
 56 |     val actualDataType = schema(colName).dataType
 57 |     val message = if (msg != null && msg.trim.length > 0) " " + msg else ""
 58 |     require(dataTypes.exists(actualDataType.equals),
 59 |       s"Column $colName must be of type equal to one of the following types: " +
 60 |         s"${dataTypes.mkString("[", ", ", "]")} but was actually of type $actualDataType.$message")
 61 |   }
 62 | 
 63 |   /**
 64 |     * Check whether the given schema contains a column of the numeric data type.
 65 |     * @param colName  column name
 66 |     */
 67 |   def checkNumericType(
 68 |                         schema: StructType,
 69 |                         colName: String,
 70 |                         msg: String = ""): Unit = {
 71 |     val actualDataType = schema(colName).dataType
 72 |     val message = if (msg != null && msg.trim.length > 0) " " + msg else ""
 73 |     require(actualDataType.isInstanceOf[NumericType], s"Column $colName must be of type " +
 74 |       s"NumericType but was actually of type $actualDataType.$message")
 75 |   }
 76 | 
 77 |   /**
 78 |     * Appends a new column to the input schema. This fails if the given output column already exists.
 79 |     * @param schema input schema
 80 |     * @param colName new column name. If this column name is an empty string "", this method returns
 81 |     *                the input schema unchanged. This allows users to disable output columns.
 82 |     * @param dataType new column data type
 83 |     * @return new schema with the input column appended
 84 |     */
 85 |   def appendColumn(
 86 |                     schema: StructType,
 87 |                     colName: String,
 88 |                     dataType: DataType,
 89 |                     nullable: Boolean = false): StructType = {
 90 |     if (colName.isEmpty) return schema
 91 |     appendColumn(schema, StructField(colName, dataType, nullable))
 92 |   }
 93 | 
 94 |   /**
 95 |     * Appends a new column to the input schema. This fails if the given output column already exists.
 96 |     * @param schema input schema
 97 |     * @param col New column schema
 98 |     * @return new schema with the input column appended
 99 |     */
100 |   def appendColumn(schema: StructType, col: StructField): StructType = {
101 |     require(!schema.fieldNames.contains(col.name), s"Column ${col.name} already exists.")
102 |     StructType(schema.fields :+ col)
103 |   }
104 | }


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Spark ML SOM (Self-Organizing Map)
 2 | 
 3 | **NEW**: cost function history can be retrieved in SOMTrainingSummary, in order to check convergence! (see Quickstart)
 4 | 
 5 | SparkML-SOM is the only available distributed implementation of Kohonen's Self-Organizing-Map algorithm built on top of Spark ML (the Dataset-based API of Spark MLlib) and fully compatible with Spark versions 2.2.0 and newer. It extends Spark's [`Estimator`](https://github.com/apache/spark/blob/v2.2.0/mllib/src/main/scala/org/apache/spark/ml/Estimator.scala) and [`Model`](https://github.com/apache/spark/blob/v2.2.0/mllib/src/main/scala/org/apache/spark/ml/Model.scala) classes.
 6 | 
 7 | * SparkML-SOM can be used as any other MLlib algorithm with a simple `fit` + `transform` syntax
 8 | * It is compatible with Datasets/DataFrames
 9 | * It can be integrated in a Spark ML Pipeline
10 | * It leverages fast native linear algebra with BLAS
11 | 
12 | The implemented algorithm is the Kohonen batch algorithm, which is very close to the $k$-means algorithm, but the computation of the average code vector is replaced with a topology-preserving weighted average. For this reason, most of the code is identical to MLlib's $k$-means implementation (see [`org.apache.spark.ml.clustering.KMeans`](https://github.com/apache/spark/blob/v2.2.0/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala) and [`org.apache.spark.mllib.clustering.KMeans`](https://github.com/apache/spark/blob/v2.2.0/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala)).
13 | 
14 | The same algorithm was implemented by one of my colleagues: https://github.com/TugdualSarazin/spark-clustering (project now maintained by [C4E](https://github.com/Clustering4Ever/Clustering4Ever)).
15 | This version is meant to be simpler to use and more concise, performant and compatible with Spark ML Pipelines and Datasets/DataFrames.
16 | 
17 | **This code will soon be integrated into the [C4E clustering project](https://github.com/Clustering4Ever/Clustering4Ever)**, so be sure to check out this project if you want to explore more clustering algorithms. In case you only need SOM, keep using this code which will remain independent and up-to-date.
18 | 
19 | ## Quickstart
20 | 
21 | ```scala
22 | import xyz.florentforest.spark.ml.som.SOM
23 | 
24 | val data: DataFrame = ???
25 | 
26 | val som = new SOM()
27 |     .setHeight(20)
28 |     .setWidth(20)
29 | 
30 | val model = som.fit(data)
31 | 
32 | val summary = model.summary // training summary
33 | 
34 | val res: DataFrame = summary.predictions
35 | // or predict on another dataset
36 | val res: DataFrame = model.transform(otherData)
37 | ```
38 | 
39 | Retrieve the cost function history to check convergence:
40 | 
41 | ```scala
42 | val cost: Array[Double] = summary.objectiveHistory
43 | println(cost.mkString("[", ",", "]"))
44 | ```
45 | 
46 | ...now plot it easily in your favorite visualization tool!
47 | 
48 | ## Installation
49 | 
50 | The sparkml-som artifact is available on Maven Central, so the quickest way to use this package in your projects is by simply adding this dependency line to sbt:
51 | 
52 | ```sbt
53 | "xyz.florentforest" %% "sparkml-som" % "0.2.1"
54 | ```
55 | 
56 | An alternative way, if for example your want to modify this project, is to fork/clone the repository, compile it using sbt and publish it locally:
57 | 
58 | ```shell
59 | $ git clone git@github.com:FlorentF9/sparkml-som.git
60 | $ cd sparkml-som
61 | $ sbt publishLocal
62 | ```
63 | 
64 | Then, add the same dependency line to your sbt.
65 | 
66 | ## Parameters
67 | 
68 | Self-organizing maps essentially depend on their topology, the neighborhood function and the neighborhood radius decay. The algorithm uses a temperature parameter that decays after each iteration and controls the neighborhood radius. It starts at a value $T_{max}$ that should cover the entire map and decreases to a value $T_{min}$ that should cover a single map cell. Here are the configuration parameters:
69 | 
70 | * **Map grid topology** (`topology`)
71 |   * rectangular _(default)_
72 | * **Height and width**: `height` _(default=10)_, `width`_(default=10)_
73 | * **Neighborhood kernel** (`neighborhoodKernel`)
74 |   * gaussian _(default)_
75 |   * rectangular window
76 | * **Temperature (or radius) decay** (`temperatureDecay`)
77 |   * exponential _(default)_
78 |   * linear
79 | * **Initial and final temperatures**: `tMax` _(default=10.0)_, `tMin` _(default=1.0)_
80 | * **Maximum number of iterations**: `maxIter` _(default=20)_
81 | * **Tolerance (for convergence)**: `tol` _(default=1e-4)_
82 | 
83 | ## Implementation details
84 | 
85 | The package depends only on spark (core, sql and mllib) and netlib for native linear algebra. It will use native BLAS libraries if possible. Because of classes and methods marked as private in spark, some utility and linear algebra code from spark had to be included into the project: _util.SchemaUtils_, _util.MLUtils_ and _linalg.BLAS_. I kept the original license and tried to keep the code minimal with only the parts needed by SOM.
86 | 
87 | ## To-dos
88 | 
89 | * Add hexagonal grid topology
90 | * Add visualization capabilities
91 | * I did not extend MLWritable/MLReadable yet, so the model cannot be saved or loaded. However, as all the parameteres are stored in the `SOMModel.prototypes` variable of type `Array[Vector]`, it is straightforward to save the parameters into a file.
92 | 


--------------------------------------------------------------------------------
/src/main/scala/som/SOMParams.scala:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Licensed to the Apache Software Foundation (ASF) under one or more
  3 |  * contributor license agreements.  See the NOTICE file distributed with
  4 |  * this work for additional information regarding copyright ownership.
  5 |  * The ASF licenses this file to You under the Apache License, Version 2.0
  6 |  * (the "License"); you may not use this file except in compliance with
  7 |  * the License.  You may obtain a copy of the License at
  8 |  *
  9 |  *    http://www.apache.org/licenses/LICENSE-2.0
 10 |  *
 11 |  * Unless required by applicable law or agreed to in writing, software
 12 |  * distributed under the License is distributed on an "AS IS" BASIS,
 13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 14 |  * See the License for the specific language governing permissions and
 15 |  * limitations under the License.
 16 |  */
 17 | 
 18 | package xyz.florentforest.spark.ml.som
 19 | 
 20 | import org.apache.spark.ml.param._
 21 | import org.apache.spark.sql.types.{IntegerType, StructType}
 22 | import xyz.florentforest.spark.ml.util.SchemaUtils
 23 | 
 24 | trait SOMParams extends Params with HasMaxIter with HasFeaturesCol with HasSeed with HasPredictionCol with HasTol {
 25 | 
 26 |   /**
 27 |     * The height of the map to create (height). Must be &gt; 1.
 28 |     * Default: 10.
 29 |     */
 30 |   final val height = new IntParam(this, "height", "The height of the map to create. " +
 31 |     "Must be > 1.", ParamValidators.gt(1))
 32 | 
 33 |   def getHeight: Int = $(height)
 34 | 
 35 |   /**
 36 |     * The width of the map to create (width). Must be &gt; 1.
 37 |     * Default: 10.
 38 |     */
 39 |   final val width = new IntParam(this, "width", "The width of the map to create. " +
 40 |     "Must be > 1.", ParamValidators.gt(1))
 41 | 
 42 |   def getWidth: Int = $(width)
 43 | 
 44 |   /**
 45 |     * Initial temperature parameter value (tMax). Must be &gt; 0.0.
 46 |     * Default: 10.0.
 47 |     */
 48 |   final val tMax = new DoubleParam(this, "tMax", "The initial temperature parameter. " +
 49 |     "Must be > 0.0.", ParamValidators.gt(0.0))
 50 | 
 51 |   def getTMax: Double = $(tMax)
 52 | 
 53 |   /**
 54 |     * Final temperature parameter value (tMin). Must be &gt; 0.0.
 55 |     * Default: 0.1.
 56 |     */
 57 |   final val tMin = new DoubleParam(this, "tMin", "The final temperature parameter. " +
 58 |     "Must be > 0.0.", ParamValidators.gt(0.0))
 59 | 
 60 |   def getTMin: Double = $(tMin)
 61 | 
 62 |   /**
 63 |     * Param for the map grid topology type. Only "rectangular" is available at the moment, hexagonal will soon be added.
 64 |     * Default: rectangular.
 65 |     */
 66 |   final val topology = new Param[String](this, "topology", "The map grid topology type. " +
 67 |     "Supported options: 'rectangular'.",
 68 |     ParamValidators.inArray(Array("rectangular")))
 69 | 
 70 |   def getTopology: String = $(topology)
 71 | 
 72 |   /**
 73 |     * Param for the neighborhood kernel type. This can be either "gaussian" or "rectangular". Default: gaussian.
 74 |     */
 75 |   final val neighborhoodKernel = new Param[String](this, "neighborhoodKernel", "The neighborhood kernel type. " +
 76 |     "Supported options: 'gaussian' and 'rectangular'.",
 77 |     ParamValidators.inArray(Array("gaussian", "rectangular")))
 78 | 
 79 |   def getNeighborhoodKernel: String = $(neighborhoodKernel)
 80 | 
 81 |   /**
 82 |     * Param for the temperature decay type. This can be either "exponential" or "linear". Default: exponential.
 83 |     */
 84 |   final val temperatureDecay = new Param[String](this, "temperatureDecay", "The temperature decay type. " +
 85 |     "Supported options: 'exponential' and 'linear'.",
 86 |     ParamValidators.inArray(Array("exponential", "linear")))
 87 | 
 88 |   def getTemperatureDecay: String = $(temperatureDecay)
 89 | 
 90 |   /**
 91 |     * Validates and transforms the input schema.
 92 |     * @param schema input schema
 93 |     * @return output schema
 94 |     */
 95 |   protected def validateAndTransformSchema(schema: StructType): StructType = {
 96 |     SchemaUtils.appendColumn(schema, $(predictionCol), IntegerType)
 97 |   }
 98 | 
 99 | }
100 | 
101 | /**
102 |   * Trait for shared param maxIter.
103 |   */
104 |  trait HasMaxIter extends Params {
105 | 
106 |   /**
107 |     * Param for maximum number of iterations (&gt;= 0).
108 |     */
109 |   final val maxIter: IntParam = new IntParam(this, "maxIter", "maximum number of iterations (>= 0)", ParamValidators.gtEq(0))
110 | 
111 |   final def getMaxIter: Int = $(maxIter)
112 | }
113 | 
114 | /**
115 |   * Trait for shared param featuresCol (default: "features").
116 |   */
117 |  trait HasFeaturesCol extends Params {
118 | 
119 |   /**
120 |     * Param for features column name.
121 |     */
122 |   final val featuresCol: Param[String] = new Param[String](this, "featuresCol", "features column name")
123 | 
124 |   setDefault(featuresCol, "features")
125 | 
126 |   final def getFeaturesCol: String = $(featuresCol)
127 | }
128 | 
129 | /**
130 |   * Trait for shared param predictionCol (default: "prediction").
131 |   */
132 |  trait HasPredictionCol extends Params {
133 | 
134 |   /**
135 |     * Param for prediction column name.
136 |     */
137 |   final val predictionCol: Param[String] = new Param[String](this, "predictionCol", "prediction column name")
138 | 
139 |   setDefault(predictionCol, "prediction")
140 | 
141 |   final def getPredictionCol: String = $(predictionCol)
142 | }
143 | 
144 | /**
145 |   * Trait for shared param seed (default: this.getClass.getName.hashCode.toLong).
146 |   */
147 |  trait HasSeed extends Params {
148 | 
149 |   /**
150 |     * Param for random seed.
151 |     */
152 |   final val seed: LongParam = new LongParam(this, "seed", "random seed")
153 | 
154 |   setDefault(seed, this.getClass.getName.hashCode.toLong)
155 | 
156 |   final def getSeed: Long = $(seed)
157 | }
158 | 
159 | /**
160 |   * Trait for shared param tol.
161 |   */
162 |  trait HasTol extends Params {
163 | 
164 |   /**
165 |     * Param for the convergence tolerance for iterative algorithms (&gt;= 0).
166 |     */
167 |   final val tol: DoubleParam = new DoubleParam(this, "tol", "the convergence tolerance for iterative algorithms (>= 0)", ParamValidators.gtEq(0))
168 | 
169 |   final def getTol: Double = $(tol)
170 | }


--------------------------------------------------------------------------------
/src/main/scala/linalg/BLAS.scala:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Licensed to the Apache Software Foundation (ASF) under one or more
  3 |  * contributor license agreements.  See the NOTICE file distributed with
  4 |  * this work for additional information regarding copyright ownership.
  5 |  * The ASF licenses this file to You under the Apache License, Version 2.0
  6 |  * (the "License"); you may not use this file except in compliance with
  7 |  * the License.  You may obtain a copy of the License at
  8 |  *
  9 |  *    http://www.apache.org/licenses/LICENSE-2.0
 10 |  *
 11 |  * Unless required by applicable law or agreed to in writing, software
 12 |  * distributed under the License is distributed on an "AS IS" BASIS,
 13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 14 |  * See the License for the specific language governing permissions and
 15 |  * limitations under the License.
 16 |  */
 17 | 
 18 | package xyz.florentforest.spark.ml.linalg
 19 | 
 20 | import org.apache.spark.ml.linalg._
 21 | import com.github.fommil.netlib.BLAS.{getInstance => NativeBLAS}
 22 | import com.github.fommil.netlib.{F2jBLAS, BLAS => NetlibBLAS}
 23 | 
 24 | /**
 25 |   * BLAS routines for MLlib's vectors and matrices.
 26 |   */
 27 | object BLAS extends Serializable {
 28 | 
 29 |   @transient private var _f2jBLAS: NetlibBLAS = _
 30 |   @transient private var _nativeBLAS: NetlibBLAS = _
 31 | 
 32 |   // For level-1 routines, we use Java implementation.
 33 |   def f2jBLAS: NetlibBLAS = {
 34 |     if (_f2jBLAS == null) {
 35 |       _f2jBLAS = new F2jBLAS
 36 |     }
 37 |     _f2jBLAS
 38 |   }
 39 | 
 40 |   /**
 41 |     * y += a * x
 42 |     */
 43 |   def axpy(a: Double, x: Vector, y: Vector): Unit = {
 44 |     require(x.size == y.size)
 45 |     y match {
 46 |       case dy: DenseVector =>
 47 |         x match {
 48 |           case sx: SparseVector =>
 49 |             axpy(a, sx, dy)
 50 |           case dx: DenseVector =>
 51 |             axpy(a, dx, dy)
 52 |           case _ =>
 53 |             throw new UnsupportedOperationException(
 54 |               s"axpy doesn't support x type ${x.getClass}.")
 55 |         }
 56 |       case _ =>
 57 |         throw new IllegalArgumentException(
 58 |           s"axpy only supports adding to a dense vector but got type ${y.getClass}.")
 59 |     }
 60 |   }
 61 | 
 62 |   /**
 63 |     * y += a * x
 64 |     */
 65 |   private def axpy(a: Double, x: DenseVector, y: DenseVector): Unit = {
 66 |     val n = x.size
 67 |     f2jBLAS.daxpy(n, a, x.values, 1, y.values, 1)
 68 |   }
 69 | 
 70 |   /**
 71 |     * y += a * x
 72 |     */
 73 |   private def axpy(a: Double, x: SparseVector, y: DenseVector): Unit = {
 74 |     val xValues = x.values
 75 |     val xIndices = x.indices
 76 |     val yValues = y.values
 77 |     val nnz = xIndices.length
 78 | 
 79 |     if (a == 1.0) {
 80 |       var k = 0
 81 |       while (k < nnz) {
 82 |         yValues(xIndices(k)) += xValues(k)
 83 |         k += 1
 84 |       }
 85 |     } else {
 86 |       var k = 0
 87 |       while (k < nnz) {
 88 |         yValues(xIndices(k)) += a * xValues(k)
 89 |         k += 1
 90 |       }
 91 |     }
 92 |   }
 93 | 
 94 |   /** Y += a * x */
 95 |   def axpy(a: Double, X: DenseMatrix, Y: DenseMatrix): Unit = {
 96 |     require(X.numRows == Y.numRows && X.numCols == Y.numCols, "Dimension mismatch: " +
 97 |       s"size(X) = ${(X.numRows, X.numCols)} but size(Y) = ${(Y.numRows, Y.numCols)}.")
 98 |     f2jBLAS.daxpy(X.numRows * X.numCols, a, X.values, 1, Y.values, 1)
 99 |   }
100 | 
101 |   /**
102 |     * dot(x, y)
103 |     */
104 |   def dot(x: Vector, y: Vector): Double = {
105 |     require(x.size == y.size,
106 |       "BLAS.dot(x: Vector, y:Vector) was given Vectors with non-matching sizes:" +
107 |         " x.size = " + x.size + ", y.size = " + y.size)
108 |     (x, y) match {
109 |       case (dx: DenseVector, dy: DenseVector) =>
110 |         dot(dx, dy)
111 |       case (sx: SparseVector, dy: DenseVector) =>
112 |         dot(sx, dy)
113 |       case (dx: DenseVector, sy: SparseVector) =>
114 |         dot(sy, dx)
115 |       case (sx: SparseVector, sy: SparseVector) =>
116 |         dot(sx, sy)
117 |       case _ =>
118 |         throw new IllegalArgumentException(s"dot doesn't support (${x.getClass}, ${y.getClass}).")
119 |     }
120 |   }
121 | 
122 |   /**
123 |     * dot(x, y)
124 |     */
125 |   private def dot(x: DenseVector, y: DenseVector): Double = {
126 |     val n = x.size
127 |     f2jBLAS.ddot(n, x.values, 1, y.values, 1)
128 |   }
129 | 
130 |   /**
131 |     * dot(x, y)
132 |     */
133 |   private def dot(x: SparseVector, y: DenseVector): Double = {
134 |     val xValues = x.values
135 |     val xIndices = x.indices
136 |     val yValues = y.values
137 |     val nnz = xIndices.length
138 | 
139 |     var sum = 0.0
140 |     var k = 0
141 |     while (k < nnz) {
142 |       sum += xValues(k) * yValues(xIndices(k))
143 |       k += 1
144 |     }
145 |     sum
146 |   }
147 | 
148 |   /**
149 |     * dot(x, y)
150 |     */
151 |   private def dot(x: SparseVector, y: SparseVector): Double = {
152 |     val xValues = x.values
153 |     val xIndices = x.indices
154 |     val yValues = y.values
155 |     val yIndices = y.indices
156 |     val nnzx = xIndices.length
157 |     val nnzy = yIndices.length
158 | 
159 |     var kx = 0
160 |     var ky = 0
161 |     var sum = 0.0
162 |     // y catching x
163 |     while (kx < nnzx && ky < nnzy) {
164 |       val ix = xIndices(kx)
165 |       while (ky < nnzy && yIndices(ky) < ix) {
166 |         ky += 1
167 |       }
168 |       if (ky < nnzy && yIndices(ky) == ix) {
169 |         sum += xValues(kx) * yValues(ky)
170 |         ky += 1
171 |       }
172 |       kx += 1
173 |     }
174 |     sum
175 |   }
176 | 
177 |   /**
178 |     * y = x
179 |     */
180 |   def copy(x: Vector, y: Vector): Unit = {
181 |     val n = y.size
182 |     require(x.size == n)
183 |     y match {
184 |       case dy: DenseVector =>
185 |         x match {
186 |           case sx: SparseVector =>
187 |             val sxIndices = sx.indices
188 |             val sxValues = sx.values
189 |             val dyValues = dy.values
190 |             val nnz = sxIndices.length
191 | 
192 |             var i = 0
193 |             var k = 0
194 |             while (k < nnz) {
195 |               val j = sxIndices(k)
196 |               while (i < j) {
197 |                 dyValues(i) = 0.0
198 |                 i += 1
199 |               }
200 |               dyValues(i) = sxValues(k)
201 |               i += 1
202 |               k += 1
203 |             }
204 |             while (i < n) {
205 |               dyValues(i) = 0.0
206 |               i += 1
207 |             }
208 |           case dx: DenseVector =>
209 |             Array.copy(dx.values, 0, dy.values, 0, n)
210 |         }
211 |       case _ =>
212 |         throw new IllegalArgumentException(s"y must be dense in copy but got ${y.getClass}")
213 |     }
214 |   }
215 | 
216 |   /**
217 |     * x = a * x
218 |     */
219 |   def scal(a: Double, x: Vector): Unit = {
220 |     x match {
221 |       case sx: SparseVector =>
222 |         f2jBLAS.dscal(sx.values.length, a, sx.values, 1)
223 |       case dx: DenseVector =>
224 |         f2jBLAS.dscal(dx.values.length, a, dx.values, 1)
225 |       case _ =>
226 |         throw new IllegalArgumentException(s"scal doesn't support vector type ${x.getClass}.")
227 |     }
228 |   }
229 | 
230 |   // For level-3 routines, we use the native BLAS.
231 |   private def nativeBLAS: NetlibBLAS = {
232 |     if (_nativeBLAS == null) {
233 |       _nativeBLAS = NativeBLAS
234 |     }
235 |     _nativeBLAS
236 |   }
237 | 
238 |   /**
239 |     * Adds alpha * v * v.t to a matrix in-place. This is the same as BLAS's ?SPR.
240 |     *
241 |     * @param U the upper triangular part of the matrix in a [[DenseVector]](column major)
242 |     */
243 |   def spr(alpha: Double, v: Vector, U: DenseVector): Unit = {
244 |     spr(alpha, v, U.values)
245 |   }
246 | 
247 |   /**
248 |     * Adds alpha * v * v.t to a matrix in-place. This is the same as BLAS's ?SPR.
249 |     *
250 |     * @param U the upper triangular part of the matrix packed in an array (column major)
251 |     */
252 |   def spr(alpha: Double, v: Vector, U: Array[Double]): Unit = {
253 |     val n = v.size
254 |     v match {
255 |       case DenseVector(values) =>
256 |         NativeBLAS.dspr("U", n, alpha, values, 1, U)
257 |       case SparseVector(size, indices, values) =>
258 |         val nnz = indices.length
259 |         var colStartIdx = 0
260 |         var prevCol = 0
261 |         var col = 0
262 |         var j = 0
263 |         var i = 0
264 |         var av = 0.0
265 |         while (j < nnz) {
266 |           col = indices(j)
267 |           // Skip empty columns.
268 |           colStartIdx += (col - prevCol) * (col + prevCol + 1) / 2
269 |           av = alpha * values(j)
270 |           i = 0
271 |           while (i <= j) {
272 |             U(colStartIdx + indices(i)) += av * values(i)
273 |             i += 1
274 |           }
275 |           j += 1
276 |           prevCol = col
277 |         }
278 |     }
279 |   }
280 | 
281 | }


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright 2019 Florent Forest
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------
/src/main/scala/som/SOM.scala:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Licensed to the Apache Software Foundation (ASF) under one or more
  3 |  * contributor license agreements.  See the NOTICE file distributed with
  4 |  * this work for additional information regarding copyright ownership.
  5 |  * The ASF licenses this file to You under the Apache License, Version 2.0
  6 |  * (the "License"); you may not use this file except in compliance with
  7 |  * the License.  You may obtain a copy of the License at
  8 |  *
  9 |  *    http://www.apache.org/licenses/LICENSE-2.0
 10 |  *
 11 |  * Unless required by applicable law or agreed to in writing, software
 12 |  * distributed under the License is distributed on an "AS IS" BASIS,
 13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 14 |  * See the License for the specific language governing permissions and
 15 |  * limitations under the License.
 16 |  */
 17 | 
 18 | package xyz.florentforest.spark.ml.som
 19 | 
 20 | import java.nio.ByteBuffer
 21 | import java.util.{Random => JavaRandom}
 22 | 
 23 | import breeze.numerics.{abs, exp, pow}
 24 | import org.apache.spark.ml.Estimator
 25 | import org.apache.spark.ml.linalg._
 26 | import org.apache.spark.ml.param.ParamMap
 27 | import org.apache.spark.ml.util._
 28 | import org.apache.spark.rdd.RDD
 29 | import org.apache.spark.sql.functions.col
 30 | import org.apache.spark.sql.types.StructType
 31 | import org.apache.spark.sql.{Dataset, Row, SparkSession}
 32 | import org.apache.spark.storage.StorageLevel
 33 | import xyz.florentforest.spark.ml.util.MLUtils
 34 | 
 35 | import scala.util.hashing.MurmurHash3
 36 | import xyz.florentforest.spark.ml.linalg.BLAS.{axpy, scal}
 37 | 
 38 | class SOM(override val uid: String) extends Estimator[SOMModel] with SOMParams {
 39 | 
 40 |   setDefault(
 41 |     height -> 10,
 42 |     width -> 10,
 43 |     tMax -> 10.0,
 44 |     tMin -> 1.0,
 45 |     maxIter -> 20,
 46 |     tol -> 1e-4,
 47 |     topology -> "rectangular",
 48 |     neighborhoodKernel -> "gaussian",
 49 |     temperatureDecay -> "exponential")
 50 | 
 51 |   override def copy(extra: ParamMap): SOM = defaultCopy(extra)
 52 | 
 53 |   def this() = this(Identifiable.randomUID("SOM"))
 54 | 
 55 |   def setFeaturesCol(value: String): this.type = set(featuresCol, value)
 56 | 
 57 |   def setPredictionCol(value: String): this.type = set(predictionCol, value)
 58 | 
 59 |   def setHeight(value: Int): this.type = set(height, value)
 60 | 
 61 |   def setWidth(value: Int): this.type = set(width, value)
 62 | 
 63 |   def setTMax(value: Double): this.type = set(tMax, value)
 64 | 
 65 |   def setTMin(value: Double): this.type = set(tMin, value)
 66 | 
 67 |   def setTopology(value: String): this.type = set(topology, value)
 68 | 
 69 |   def setNeighborhoodKernel(value: String): this.type = set(neighborhoodKernel, value)
 70 | 
 71 |   def setTemperatureDecay(value: String): this.type = set(temperatureDecay, value)
 72 | 
 73 |   def setMaxIter(value: Int): this.type = set(maxIter, value)
 74 | 
 75 |   def setTol(value: Double): this.type = set(tol, value)
 76 | 
 77 |   def setSeed(value: Long): this.type = set(seed, value)
 78 | 
 79 |   override def fit(dataset: Dataset[_]): SOMModel = {
 80 |     transformSchema(dataset.schema, logging = true)
 81 | 
 82 |     val handlePersistence = dataset.storageLevel == StorageLevel.NONE
 83 | 
 84 |     val instances: RDD[Vector] = dataset.select(col($(featuresCol))).rdd.map {
 85 |       case Row(point: Vector) => point
 86 |     }
 87 | 
 88 |     if (handlePersistence) {
 89 |       instances.persist(StorageLevel.MEMORY_AND_DISK)
 90 |     }
 91 | 
 92 |     val parentModel = run(instances)
 93 |     val model = copyValues(new SOMModel(uid, parentModel.prototypes).setParent(this))
 94 |     val summary = new SOMTrainingSummary(
 95 |       model.transform(dataset),
 96 |       $(predictionCol),
 97 |       $(featuresCol),
 98 |       $(height),
 99 |       $(width),
100 |       $(tMax),
101 |       $(tMin),
102 |       $(maxIter),
103 |       $(tol),
104 |       $(topology),
105 |       $(neighborhoodKernel),
106 |       $(temperatureDecay),
107 |       parentModel.cost,
108 |       parentModel.history)
109 |     model.setSummary(Some(summary))
110 | 
111 |     if (handlePersistence) {
112 |       instances.unpersist()
113 |     }
114 |     model
115 | 
116 |   }
117 | 
118 |   override def transformSchema(schema: StructType): StructType = {
119 |     validateAndTransformSchema(schema)
120 |   }
121 | 
122 |   // Initial prototypes can be provided as a SOMModel object rather than using the
123 |   // random initialization.
124 |   private var initialModel: Option[SOMModel] = None
125 | 
126 |   def setInitialModel(model: SOMModel): this.type = {
127 |     require(model.height == height, "mismatched map height")
128 |     require(model.width == width, "mismatched map width")
129 |     initialModel = Some(model)
130 |     this
131 |   }
132 | 
133 |   def run(data: RDD[Vector]): SOMModel = {
134 | 
135 |     if (data.getStorageLevel == StorageLevel.NONE) {
136 |       logWarning("The input data is not directly cached, which may hurt performance if its"
137 |         + " parent RDDs are also uncached.")
138 |     }
139 | 
140 |     // Compute squared norms and cache them.
141 |     val norms = data.map(Vectors.norm(_, 2.0))
142 |     norms.persist()
143 |     val zippedData = data.zip(norms).map { case (v, norm) =>
144 |       new VectorWithNorm(v, norm)
145 |     }
146 |     val model = runAlgorithm(zippedData)
147 |     norms.unpersist()
148 | 
149 |     // Warn at the end of the run as well, for increased visibility.
150 |     if (data.getStorageLevel == StorageLevel.NONE) {
151 |       logWarning("The input data was not directly cached, which may hurt performance if its"
152 |         + " parent RDDs are also uncached.")
153 |     }
154 | 
155 |     model
156 |   }
157 | 
158 |   def runAlgorithm(data: RDD[VectorWithNorm]): SOMModel = {
159 | 
160 |     val sc = data.sparkContext
161 | 
162 |     val initStartTime = System.nanoTime()
163 | 
164 |     val codeVectors = initialModel match {
165 |       case Some(initialMap) => initialMap.prototypes.map(new VectorWithNorm(_))
166 |       case None => initRandom(data)
167 |     }
168 |     val dims = codeVectors.head.vector.size
169 | 
170 |     val initTimeInSeconds = (System.nanoTime() - initStartTime) / 1e9
171 |     logInfo(f"Initialization took $initTimeInSeconds%.3f seconds.")
172 | 
173 |     var converged = false
174 |     var cost = 0.0
175 |     val arrayBuilder = scala.collection.mutable.ArrayBuilder.make[Double]
176 |     var iteration = 0
177 | 
178 |     val iterationStartTime = System.nanoTime()
179 | 
180 |     while (iteration < $(maxIter) && !converged) {
181 |       val costAccum = sc.doubleAccumulator
182 |       val bcCodeVectors = sc.broadcast(codeVectors)
183 | 
184 |       /*
185 |        * Kohonen batch algorithm
186 |        * (M. Lebbah, Thesis, p.26)
187 |        */
188 | 
189 |       val T = computeTemperature(iteration)
190 | 
191 |       // Find the sum and count of points mapping to each code vector
192 |       val totalContribs = data.mapPartitions { points =>
193 |         val thisCodeVectors = bcCodeVectors.value
194 |         val dims = thisCodeVectors.head.vector.size
195 | 
196 |         val sums = Array.fill(thisCodeVectors.length)(Vectors.zeros(dims))
197 |         val counts = Array.fill(thisCodeVectors.length)(0L)
198 | 
199 |         points.foreach { point =>
200 |           val (bestMatchingUnit, cost) = SOM.findClosest(thisCodeVectors, point)
201 |           costAccum.add(cost)
202 |           val sum = sums(bestMatchingUnit)
203 | 
204 |           axpy(1.0, point.vector, sum)
205 |           counts(bestMatchingUnit) += 1
206 |         }
207 | 
208 |         counts.indices.filter(counts(_) > 0).map(j => (j, (sums(j), counts(j)))).iterator
209 |       }.reduceByKey { case ((sum1, count1), (sum2, count2)) =>
210 |         axpy(1.0, sum2, sum1)
211 |         (sum1, count1 + count2)
212 |       }.collectAsMap()
213 | 
214 |       bcCodeVectors.destroy()
215 | 
216 |       converged = true
217 | 
218 |       // Compute the neighborhood-weighted sums and counts
219 |       val weights = Array.tabulate(codeVectors.length) { i =>
220 |         Array.tabulate(codeVectors.length)(j => computeNeighborhood(cellDist(i, j), T))
221 |       }
222 | 
223 |       val weightedContribs = (0 until codeVectors.length).map { k =>
224 | 
225 |         val weightedSum = Vectors.zeros(dims)
226 |         var weightedCount = 0D
227 | 
228 |         totalContribs.foreach { case (j, (sum, count)) =>
229 |             axpy(weights(k)(j), sum, weightedSum)
230 |             weightedCount += weights(k)(j) * count
231 |         }
232 | 
233 |         (k, (weightedSum, weightedCount))
234 |       }
235 | 
236 |       // Update the map code vectors and costs
237 |       weightedContribs.foreach { case (k, (sum, count)) =>
238 |         scal(1.0 / count, sum)
239 |         val newCodeVector = new VectorWithNorm(sum)
240 |         if (converged && SOM.fastSquaredDistance(newCodeVector, codeVectors(k)) > $(tol) * $(tol)) {
241 |           converged = false
242 |         }
243 |         codeVectors(k) = newCodeVector
244 |       }
245 | 
246 |       cost = costAccum.value
247 |       logInfo(s"SOM quantization error: $cost")
248 |       iteration += 1
249 |       arrayBuilder += cost
250 |     }
251 | 
252 |     val iterationTimeInSeconds = (System.nanoTime() - iterationStartTime) / 1e9
253 |     logInfo(f"Iterations took $iterationTimeInSeconds%.3f seconds.")
254 | 
255 |     if (iteration == $(maxIter)) {
256 |       logInfo(s"SOM reached the max number of iterations: ${$(maxIter)}.")
257 |     } else {
258 |       logInfo(s"SOM converged in $iteration iterations.")
259 |     }
260 | 
261 |     logInfo(s"The cost is $cost.")
262 | 
263 |     new SOMModel(Identifiable.randomUID("SOMModel"), codeVectors.map(_.vector))
264 |       .setCost(Some(cost))
265 |       .setHistory(Some(arrayBuilder.result()))
266 | 
267 |   }
268 | 
269 |   /**
270 |     * Temperature decay function
271 |     */
272 |   private def computeTemperature(iter: Int): Double = $(temperatureDecay) match {
273 |     case "exponential" => $(tMax) * pow( ($(tMin) / $(tMax)), (iter.toDouble / ($(maxIter) - 1) ) )
274 |     case "linear" => $(tMax) + (iter.toDouble / ($(maxIter) - 1)) * ($(tMin) - $(tMax))
275 |   }
276 | 
277 |   /**
278 |     * Neighborhood kernel function
279 |     */
280 |   private def computeNeighborhood(d: Int, T: Double): Double = $(neighborhoodKernel) match {
281 |     case "gaussian" => exp(-(d*d).toDouble / (T*T))
282 |     case "rectangular" => if (d <= T) 1.0 else 0.0
283 |   }
284 | 
285 |   /**
286 |     * Manhattan distance between two map cells
287 |     */
288 |   private def cellDist(id1: Int, id2: Int): Int = $(topology) match {
289 |     case "rectangular" => abs(id2 / $(width) - id1 / $(width)) + abs(id2 % $(width) - id1 % $(width))
290 |   }
291 | 
292 |   /**
293 |     * Initialize a set of code vectors at random.
294 |     */
295 |   private def initRandom(data: RDD[VectorWithNorm]): Array[VectorWithNorm] = {
296 |     // Select with replacement
297 |     data.takeSample(true, $(height) * $(width), new XORShiftRandom($(seed)).nextInt())
298 |   }
299 | 
300 | }
301 | 
302 | object SOM {
303 |   /**
304 |     * Returns the index of the closest center to the given point, as well as the squared distance.
305 |     */
306 |   def findClosest(centers: TraversableOnce[VectorWithNorm],
307 |                   point: VectorWithNorm): (Int, Double) = {
308 |     var bestDistance = Double.PositiveInfinity
309 |     var bestIndex = 0
310 |     var i = 0
311 |     centers.foreach { center =>
312 |       // Since `\|a - b\| \geq |\|a\| - \|b\||`, we can use this lower bound to avoid unnecessary
313 |       // distance computation.
314 |       var lowerBoundOfSqDist = center.norm - point.norm
315 |       lowerBoundOfSqDist = lowerBoundOfSqDist * lowerBoundOfSqDist
316 |       if (lowerBoundOfSqDist < bestDistance) {
317 |         val distance: Double = fastSquaredDistance(center, point)
318 |         if (distance < bestDistance) {
319 |           bestDistance = distance
320 |           bestIndex = i
321 |         }
322 |       }
323 |       i += 1
324 |     }
325 |     (bestIndex, bestDistance)
326 |   }
327 | 
328 |   /**
329 |     * Returns the K-means cost of a given point against the given cluster centers.
330 |     */
331 |   def pointCost(centers: TraversableOnce[VectorWithNorm],
332 |                 point: VectorWithNorm): Double =
333 |     findClosest(centers, point)._2
334 | 
335 |   /**
336 |     * Returns the squared Euclidean distance between two vectors computed by
337 |     * [[xyz.florentforest.spark.ml.util.MLUtils#fastSquaredDistance]].
338 |     */
339 |   def fastSquaredDistance(v1: VectorWithNorm, v2: VectorWithNorm): Double = {
340 |     MLUtils.fastSquaredDistance(v1.vector, v1.norm, v2.vector, v2.norm)
341 |     //Vectors.sqdist(v1.vector, v2.vector)
342 |   }
343 | 
344 | }
345 | 
346 | /**
347 |   * A vector with its norm for fast distance computation.
348 |   *
349 |   * @see [[xyz.florentforest.spark.ml.som.SOM#fastSquaredDistance]]
350 |   */
351 | class VectorWithNorm(val vector: Vector, val norm: Double) extends Serializable {
352 | 
353 |   def this(vector: Vector) = this(vector, Vectors.norm(vector, 2.0))
354 | 
355 |   def this(array: Array[Double]) = this(Vectors.dense(array))
356 | 
357 |   /** Converts the vector to a dense vector. */
358 |   def toDense: VectorWithNorm = new VectorWithNorm(Vectors.dense(vector.toArray), norm)
359 | }
360 | 
361 | 
362 | class XORShiftRandom(init: Long) extends JavaRandom(init) {
363 | 
364 |   def this() = this(System.nanoTime)
365 | 
366 |   private var seed = XORShiftRandom.hashSeed(init)
367 | 
368 |   // we need to just override next - this will be called by nextInt, nextDouble,
369 |   // nextGaussian, nextLong, etc.
370 |   override protected def next(bits: Int): Int = {
371 |     var nextSeed = seed ^ (seed << 21)
372 |     nextSeed ^= (nextSeed >>> 35)
373 |     nextSeed ^= (nextSeed << 4)
374 |     seed = nextSeed
375 |     (nextSeed & ((1L << bits) -1)).asInstanceOf[Int]
376 |   }
377 | 
378 |   override def setSeed(s: Long) {
379 |     seed = XORShiftRandom.hashSeed(s)
380 |   }
381 | }
382 | 
383 | object XORShiftRandom {
384 | 
385 |   /** Hash seeds to have 0/1 bits throughout. */
386 |   def hashSeed(seed: Long): Long = {
387 |     val bytes = ByteBuffer.allocate(java.lang.Long.SIZE).putLong(seed).array()
388 |     val lowBits = MurmurHash3.bytesHash(bytes)
389 |     val highBits = MurmurHash3.bytesHash(bytes, lowBits)
390 |     (highBits.toLong << 32) | (lowBits.toLong & 0xFFFFFFFFL)
391 |   }
392 | }
393 | 
394 | /**
395 |   * Main object for test and benchmark purpose
396 |   */
397 | object Main extends App {
398 | 
399 |   println("Spark ML SOM test")
400 | 
401 |   val spark = SparkSession
402 |     .builder()
403 |     .appName("Spark SOM test (xyz.florentforest.spark.ml.som)")
404 |     .master("local[*]")
405 |     .getOrCreate()
406 | 
407 |   import spark.implicits._
408 | 
409 |   final val N = 10000
410 |   val rng = new JavaRandom()
411 |   val data = Seq.tabulate(N){ _ => (0.0, Vectors.dense(rng.nextDouble, rng.nextDouble, rng.nextDouble)) }
412 | 
413 |   val df = data.toDF("label", "features")
414 | 
415 |   val som: SOM = new SOM()
416 |     .setMaxIter(100)
417 | 
418 |   val map: SOMModel = som.fit(df)
419 | 
420 |   val summary = map.summary
421 |   println(summary.trainingCost)
422 |   println(summary.objectiveHistory.mkString("[", ",", "]"))
423 |   println(s"Iterations: ${summary.objectiveHistory.length}")
424 |   summary.predictions.show(false)
425 | 
426 |   println(map.prototypes.length)
427 | 
428 |   println(map.prototypes.mkString("[", ",", "]"))
429 | 
430 |   spark.stop()
431 | }


--------------------------------------------------------------------------------