├── LICENSE ├── README.md ├── backup ├── CNNTopology.scala └── MnistCSVDriver.scala ├── dataset └── train.format └── src ├── communityInterface ├── CNNUpdater.scala ├── ConvolutionalLayer.scala ├── Driver.scala ├── MapSize.scala └── MeanPoolingLayer.scala └── main ├── CNN.scala ├── CNNLayer.scala ├── ConvolutionLayer.scala ├── Example.scala ├── MaxPoolingLayer.scala ├── MeanPoolingLayer.scala ├── Scale.scala └── printMatrix.scala /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | 3 | Apache License 4 | Version 2.0, January 2004 5 | http://www.apache.org/licenses/ 6 | 7 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 8 | 9 | 1. Definitions. 10 | 11 | "License" shall mean the terms and conditions for use, reproduction, 12 | and distribution as defined by Sections 1 through 9 of this document. 13 | 14 | "Licensor" shall mean the copyright owner or entity authorized by 15 | the copyright owner that is granting the License. 16 | 17 | "Legal Entity" shall mean the union of the acting entity and all 18 | other entities that control, are controlled by, or are under common 19 | control with that entity. For the purposes of this definition, 20 | "control" means (i) the power, direct or indirect, to cause the 21 | direction or management of such entity, whether by contract or 22 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 23 | outstanding shares, or (iii) beneficial ownership of such entity. 24 | 25 | "You" (or "Your") shall mean an individual or Legal Entity 26 | exercising permissions granted by this License. 27 | 28 | "Source" form shall mean the preferred form for making modifications, 29 | including but not limited to software source code, documentation 30 | source, and configuration files. 31 | 32 | "Object" form shall mean any form resulting from mechanical 33 | transformation or translation of a Source form, including but 34 | not limited to compiled object code, generated documentation, 35 | and conversions to other media types. 36 | 37 | "Work" shall mean the work of authorship, whether in Source or 38 | Object form, made available under the License, as indicated by a 39 | copyright notice that is included in or attached to the work 40 | (an example is provided in the Appendix below). 41 | 42 | "Derivative Works" shall mean any work, whether in Source or Object 43 | form, that is based on (or derived from) the Work and for which the 44 | editorial revisions, annotations, elaborations, or other modifications 45 | represent, as a whole, an original work of authorship. For the purposes 46 | of this License, Derivative Works shall not include works that remain 47 | separable from, or merely link (or bind by name) to the interfaces of, 48 | the Work and Derivative Works thereof. 49 | 50 | "Contribution" shall mean any work of authorship, including 51 | the original version of the Work and any modifications or additions 52 | to that Work or Derivative Works thereof, that is intentionally 53 | submitted to Licensor for inclusion in the Work by the copyright owner 54 | or by an individual or Legal Entity authorized to submit on behalf of 55 | the copyright owner. For the purposes of this definition, "submitted" 56 | means any form of electronic, verbal, or written communication sent 57 | to the Licensor or its representatives, including but not limited to 58 | communication on electronic mailing lists, source code control systems, 59 | and issue tracking systems that are managed by, or on behalf of, the 60 | Licensor for the purpose of discussing and improving the Work, but 61 | excluding communication that is conspicuously marked or otherwise 62 | designated in writing by the copyright owner as "Not a Contribution." 63 | 64 | "Contributor" shall mean Licensor and any individual or Legal Entity 65 | on behalf of whom a Contribution has been received by Licensor and 66 | subsequently incorporated within the Work. 67 | 68 | 2. Grant of Copyright License. Subject to the terms and conditions of 69 | this License, each Contributor hereby grants to You a perpetual, 70 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 71 | copyright license to reproduce, prepare Derivative Works of, 72 | publicly display, publicly perform, sublicense, and distribute the 73 | Work and such Derivative Works in Source or Object form. 74 | 75 | 3. Grant of Patent License. Subject to the terms and conditions of 76 | this License, each Contributor hereby grants to You a perpetual, 77 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 78 | (except as stated in this section) patent license to make, have made, 79 | use, offer to sell, sell, import, and otherwise transfer the Work, 80 | where such license applies only to those patent claims licensable 81 | by such Contributor that are necessarily infringed by their 82 | Contribution(s) alone or by combination of their Contribution(s) 83 | with the Work to which such Contribution(s) was submitted. If You 84 | institute patent litigation against any entity (including a 85 | cross-claim or counterclaim in a lawsuit) alleging that the Work 86 | or a Contribution incorporated within the Work constitutes direct 87 | or contributory patent infringement, then any patent licenses 88 | granted to You under this License for that Work shall terminate 89 | as of the date such litigation is filed. 90 | 91 | 4. Redistribution. You may reproduce and distribute copies of the 92 | Work or Derivative Works thereof in any medium, with or without 93 | modifications, and in Source or Object form, provided that You 94 | meet the following conditions: 95 | 96 | (a) You must give any other recipients of the Work or 97 | Derivative Works a copy of this License; and 98 | 99 | (b) You must cause any modified files to carry prominent notices 100 | stating that You changed the files; and 101 | 102 | (c) You must retain, in the Source form of any Derivative Works 103 | that You distribute, all copyright, patent, trademark, and 104 | attribution notices from the Source form of the Work, 105 | excluding those notices that do not pertain to any part of 106 | the Derivative Works; and 107 | 108 | (d) If the Work includes a "NOTICE" text file as part of its 109 | distribution, then any Derivative Works that You distribute must 110 | include a readable copy of the attribution notices contained 111 | within such NOTICE file, excluding those notices that do not 112 | pertain to any part of the Derivative Works, in at least one 113 | of the following places: within a NOTICE text file distributed 114 | as part of the Derivative Works; within the Source form or 115 | documentation, if provided along with the Derivative Works; or, 116 | within a display generated by the Derivative Works, if and 117 | wherever such third-party notices normally appear. The contents 118 | of the NOTICE file are for informational purposes only and 119 | do not modify the License. You may add Your own attribution 120 | notices within Derivative Works that You distribute, alongside 121 | or as an addendum to the NOTICE text from the Work, provided 122 | that such additional attribution notices cannot be construed 123 | as modifying the License. 124 | 125 | You may add Your own copyright statement to Your modifications and 126 | may provide additional or different license terms and conditions 127 | for use, reproduction, or distribution of Your modifications, or 128 | for any such Derivative Works as a whole, provided Your use, 129 | reproduction, and distribution of the Work otherwise complies with 130 | the conditions stated in this License. 131 | 132 | 5. Submission of Contributions. Unless You explicitly state otherwise, 133 | any Contribution intentionally submitted for inclusion in the Work 134 | by You to the Licensor shall be under the terms and conditions of 135 | this License, without any additional terms or conditions. 136 | Notwithstanding the above, nothing herein shall supersede or modify 137 | the terms of any separate license agreement you may have executed 138 | with Licensor regarding such Contributions. 139 | 140 | 6. Trademarks. This License does not grant permission to use the trade 141 | names, trademarks, service marks, or product names of the Licensor, 142 | except as required for reasonable and customary use in describing the 143 | origin of the Work and reproducing the content of the NOTICE file. 144 | 145 | 7. Disclaimer of Warranty. Unless required by applicable law or 146 | agreed to in writing, Licensor provides the Work (and each 147 | Contributor provides its Contributions) on an "AS IS" BASIS, 148 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 149 | implied, including, without limitation, any warranties or conditions 150 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 151 | PARTICULAR PURPOSE. You are solely responsible for determining the 152 | appropriateness of using or redistributing the Work and assume any 153 | risks associated with Your exercise of permissions under this License. 154 | 155 | 8. Limitation of Liability. In no event and under no legal theory, 156 | whether in tort (including negligence), contract, or otherwise, 157 | unless required by applicable law (such as deliberate and grossly 158 | negligent acts) or agreed to in writing, shall any Contributor be 159 | liable to You for damages, including any direct, indirect, special, 160 | incidental, or consequential damages of any character arising as a 161 | result of this License or out of the use or inability to use the 162 | Work (including but not limited to damages for loss of goodwill, 163 | work stoppage, computer failure or malfunction, or any and all 164 | other commercial damages or losses), even if such Contributor 165 | has been advised of the possibility of such damages. 166 | 167 | 9. Accepting Warranty or Additional Liability. While redistributing 168 | the Work or Derivative Works thereof, You may choose to offer, 169 | and charge a fee for, acceptance of support, warranty, indemnity, 170 | or other liability obligations and/or rights consistent with this 171 | License. However, in accepting such obligations, You may act only 172 | on Your own behalf and on Your sole responsibility, not on behalf 173 | of any other Contributor, and only if You agree to indemnify, 174 | defend, and hold each Contributor harmless for any liability 175 | incurred by, or claims asserted against, such Contributor by reason 176 | of your accepting any such warranty or additional liability. 177 | 178 | END OF TERMS AND CONDITIONS -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CNN based on Spark 2 | 3 | A Spark machine learning package containing the implementation of classical [Convolutional Neural Network](https://en.wikipedia.org/wiki/Convolutional_neural_network). 4 | 5 | ## Requirements 6 | 7 | This documentation is for Spark 1.3+. Other version will probably work yet not tested. 8 | 9 | ## Features 10 | 11 | `mCNN` supports training of a Convolutional Neural Network. Currently `ConvolutionLayer`, `MeanPoolingLayer` are included. `MaxPooling` and `SumPooling` are under test. 12 | A version compatible to the ANN interface in Spark 1.5 is also under development in the communityInterface folder. 13 | 14 | ## Example 15 | 16 | ### Scala API 17 | 18 | ```scala 19 | // training for Mnist data set. 20 | val topology = new CNNTopology 21 | topology.addLayer(CNNLayer.buildConvolutionLayer(1, 6, new Scale(5, 5))) 22 | topology.addLayer(CNNLayer.buildMeanPoolingLayer(new Scale(2, 2))) 23 | topology.addLayer(CNNLayer.buildConvolutionLayer(6, 12, new Scale(5, 5))) 24 | topology.addLayer(CNNLayer.buildMeanPoolingLayer(new Scale(2, 2))) 25 | topology.addLayer(CNNLayer.buildConvolutionLayer(12, 12, new Scale(4, 4))) 26 | val cnn: CNN = new CNN(topology).setMaxIterations(1000).setMiniBatchSize(16) 27 | cnn.trainOneByOne(data) 28 | ``` 29 | -------------------------------------------------------------------------------- /backup/CNNTopology.scala: -------------------------------------------------------------------------------- 1 | 2 | package org.apache.spark.ml.ann 3 | 4 | import breeze.linalg.{DenseMatrix => BDM, Vector => BV, _} 5 | import org.apache.spark.mllib.linalg.{Vector, Vectors} 6 | import org.apache.spark.mllib.optimization.Updater 7 | 8 | /** 9 | * Feed forward ANN 10 | * @param layers 11 | */ 12 | private[ann] class CNNTopology private(val layers: Array[Layer]) extends Topology { 13 | 14 | override def getInstance(weights: Vector): TopologyModel = CNNModel(this, weights) 15 | 16 | override def getInstance(seed: Long): TopologyModel = CNNModel(this, seed) 17 | } 18 | 19 | /** 20 | * Factory for some of the frequently-used topologies 21 | */ 22 | private[ml] object CNNTopology { 23 | /** 24 | * Creates a feed forward topology from the array of layers 25 | * @param layers array of layers 26 | * @return feed forward topology 27 | */ 28 | def apply(layers: Array[Layer]): CNNTopology = { 29 | new CNNTopology(layers) 30 | } 31 | } 32 | 33 | /** 34 | * Model of Feed Forward Neural Network. 35 | * Implements forward, gradient computation and can return weights in vector format. 36 | * @param layerModels models of layers 37 | * @param topology topology of the network 38 | */ 39 | private[ml] class CNNModel private( 40 | val layerModels: Array[LayerModel], 41 | val topology: CNNTopology) extends TopologyModel { 42 | 43 | override def forward(data: BDM[Double]): Array[BDM[Double]] = { 44 | val outputs = new Array[BDM[Double]](layerModels.length) 45 | outputs(0) = layerModels(0).eval(data) 46 | for (i <- 1 until layerModels.length) { 47 | outputs(i) = layerModels(i).eval(outputs(i-1)) 48 | } 49 | outputs 50 | } 51 | 52 | private def setOutLayerErrors( 53 | label: BDM[Double], 54 | output: BDM[Double]): (Double, BDM[Double]) = { 55 | val mapNum: Int = output.rows 56 | val outValues: Array[Double] = output(::, 0).toArray 57 | val target: Array[Double] = label(::, 0).toArray 58 | val layerError: BDM[Double] = new BDM(mapNum, 1) 59 | for(i <- 0 until mapNum){ 60 | layerError(i, 0) = outValues(i) * (1 - outValues(i)) * (target(i) - outValues(i)) 61 | } 62 | (sum(layerError), layerError) 63 | } 64 | 65 | private def backPropagation( 66 | lastError: BDM[Double], 67 | outputs: Array[BDM[Double]]): Array[BDM[Double]] = { 68 | val errors = new Array[BDM[Double]](layerModels.size) 69 | errors(layerModels.length - 1) = lastError 70 | var l: Int = layerModels.size - 2 71 | while (l >= 0) { 72 | val layer: LayerModel = layerModels(l + 1) 73 | errors(l) = layer.prevDelta(errors(l + 1), outputs(l)) 74 | l -= 1 75 | } 76 | errors 77 | } 78 | 79 | override def computeGradient( 80 | data: BDM[Double], 81 | target: BDM[Double], 82 | cumGradient: Vector, 83 | realBatchSize: Int): Double = { 84 | 85 | val outputs = forward(data) 86 | val (loss, lastError) = setOutLayerErrors(target, outputs.last) 87 | val errors = backPropagation(lastError, outputs) 88 | 89 | val grads = new Array[Array[Double]](layerModels.length) 90 | for (i <- 0 until layerModels.length) { 91 | val input = if (i == 0) data else outputs(i - 1) 92 | grads(i) = layerModels(i).grad(errors(i), input) 93 | } 94 | // update cumGradient 95 | val cumGradientArray = cumGradient.toArray 96 | var offset = 0 97 | // TODO: extract roll 98 | for (i <- 0 until grads.length) { 99 | val gradArray = grads(i) 100 | var k = 0 101 | while (k < gradArray.length) { 102 | cumGradientArray(offset + k) += gradArray(k) 103 | k += 1 104 | } 105 | offset += gradArray.length 106 | } 107 | loss 108 | } 109 | 110 | // TODO: do we really need to copy the weights? they should be read-only 111 | override def weights(): Vector = { 112 | // TODO: extract roll 113 | var size = 0 114 | for (i <- 0 until layerModels.length) { 115 | size += layerModels(i).size 116 | } 117 | val array = new Array[Double](size) 118 | var offset = 0 119 | for (i <- 0 until layerModels.length) { 120 | val layerWeights = layerModels(i).weights().toArray 121 | System.arraycopy(layerWeights, 0, array, offset, layerWeights.length) 122 | offset += layerWeights.length 123 | } 124 | Vectors.dense(array) 125 | } 126 | 127 | override def predict(data: Vector): Vector = { 128 | val size = data.size 129 | val result = forward(new BDM[Double](size, 1, data.toArray)) 130 | Vectors.dense(result.last.toArray) 131 | } 132 | } 133 | 134 | /** 135 | * Fabric for feed forward ANN models 136 | */ 137 | private[ann] object CNNModel { 138 | 139 | /** 140 | * Creates a model from a topology and weights 141 | * @param topology topology 142 | * @param weights weights 143 | * @return model 144 | */ 145 | def apply(topology: CNNTopology, weights: Vector): CNNModel = { 146 | val layers = topology.layers 147 | val layerModels = new Array[LayerModel](layers.length) 148 | var offset = 0 149 | for (i <- 0 until layers.length) { 150 | layerModels(i) = layers(i).getInstance(weights, offset) 151 | offset += layerModels(i).size 152 | } 153 | new CNNModel(layerModels, topology) 154 | } 155 | 156 | /** 157 | * Creates a model given a topology and seed 158 | * @param topology topology 159 | * @param seed seed for generating the weights 160 | * @return model 161 | */ 162 | def apply(topology: CNNTopology, seed: Long = 11L): CNNModel = { 163 | val layers = topology.layers 164 | val layerModels = new Array[LayerModel](layers.length) 165 | for(i <- 0 until layers.length){ 166 | layerModels(i) = layers(i).getInstance(seed) 167 | } 168 | new CNNModel(layerModels, topology) 169 | } 170 | 171 | private[ann] def getMaxIndex(out: Array[Double]): Int = { 172 | var max: Double = out(0) 173 | var index: Int = 0 174 | var i: Int = 1 175 | while (i < out.length) { 176 | if (out(i) > max) { 177 | max = out(i) 178 | index = i 179 | } 180 | i += 1 181 | } 182 | index 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /backup/MnistCSVDriver.scala: -------------------------------------------------------------------------------- 1 | package hhbyyh.mCNN 2 | 3 | import org.apache.log4j.{Level, Logger} 4 | import org.apache.spark.mllib.linalg.Vectors 5 | import org.apache.spark.mllib.regression.LabeledPoint 6 | import org.apache.spark.{SparkConf, SparkContext} 7 | 8 | object MnistCSVDriver { 9 | 10 | def main(args: Array[String]) { 11 | val topology = new CNNTopology 12 | topology.addLayer(CNNLayer.buildConvolutionLayer(new Scale(28, 28))) 13 | topology.addLayer(CNNLayer.buildConvLayer(6, new Scale(5, 5))) 14 | topology.addLayer(CNNLayer.buildSampLayer(new Scale(2, 2))) 15 | topology.addLayer(CNNLayer.buildConvLayer(12, new Scale(5, 5))) 16 | topology.addLayer(CNNLayer.buildSampLayer(new Scale(2, 2))) 17 | topology.addLayer(CNNLayer.buildConvLayer(12, new Scale(4, 4))) 18 | val cnn: CNN = new CNN(topology).setMaxIterations(500000).setMiniBatchSize(16) 19 | 20 | Logger.getLogger("org").setLevel(Level.WARN) 21 | Logger.getLogger("akka").setLevel(Level.WARN) 22 | val conf = new SparkConf().setMaster("local[8]").setAppName("ttt") 23 | val sc = new SparkContext(conf) 24 | val lines = sc.textFile("dataset/mnist/mnist_train.csv", 8) 25 | val data = lines.map(line => line.split(",")).map(arr => arr.map(_.toDouble)) 26 | .map(arr => new LabeledPoint(arr(0), Vectors.dense(arr.slice(1, 785).map(v => if(v > 0) 1.0 else 0)))) 27 | 28 | val start = System.nanoTime() 29 | cnn.trainOneByOne(data) 30 | println("Training time: " + (System.nanoTime() - start) / 1e9) 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/communityInterface/CNNUpdater.scala: -------------------------------------------------------------------------------- 1 | package org.apache.spark.ml.ann 2 | 3 | import breeze.linalg.{*, DenseMatrix => BDM, DenseVector => BDV, Vector => BV, axpy => Baxpy, 4 | sum => Bsum} 5 | import org.apache.spark.mllib.linalg.{Vectors, Vector} 6 | import org.apache.spark.mllib.optimization.Updater 7 | 8 | private[ann] class CNNUpdater(alpha: Double) extends Updater { 9 | 10 | override def compute( 11 | weightsOld: Vector, 12 | gradient: Vector, 13 | stepSize: Double, 14 | iter: Int, 15 | regParam: Double): (Vector, Double) = { 16 | val thisIterStepSize = stepSize 17 | val brzWeights: BV[Double] = weightsOld.toBreeze.toDenseVector 18 | Baxpy(-thisIterStepSize, gradient.toBreeze * alpha, brzWeights) 19 | (Vectors.fromBreeze(brzWeights), 0) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/communityInterface/ConvolutionalLayer.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.spark.ml.ann 18 | 19 | import breeze.linalg.{DenseMatrix => BDM, DenseVector => BDV, fliplr, flipud, sum} 20 | import org.apache.spark.mllib.linalg.{Vectors, DenseVector, Vector} 21 | 22 | /** 23 | * Layer properties of Convolutional Layer 24 | * @param numInMap number of inputs 25 | * @param numOutMap number of outputs 26 | * @param kernelSize kernelSize of format x * y 27 | * @param inputMapSize size of each input feature map. 28 | */ 29 | private[ann] class ConvolutionalLayer( 30 | val numInMap: Int, 31 | val numOutMap: Int, 32 | val kernelSize: MapSize, 33 | val inputMapSize: MapSize) extends Layer { 34 | 35 | override def getInstance(weights: Vector, position: Int): LayerModel = { 36 | ConvolutionalLayerModel(this, weights, position) 37 | } 38 | 39 | override def getInstance(seed: Long = 11L): LayerModel = { 40 | ConvolutionalLayerModel(this, seed) 41 | } 42 | } 43 | 44 | /** 45 | * ConvolutionLayerModel contains multiple kernels and applies a 2D convolution over the input maps. 46 | * @param inMapNum number of input feature maps 47 | * @param outMapNum number of output feature maps 48 | * @param kernels weights in the form of inNum * outNum * kernelSize 49 | * @param bias bias for each outMap, with length equal to outMapNum 50 | * @param inputMapSize size of each input feature map 51 | */ 52 | private[ann] class ConvolutionalLayerModel private( 53 | inMapNum: Int, 54 | outMapNum: Int, 55 | kernels: Array[Array[BDM[Double]]], 56 | bias: Array[Double], 57 | inputMapSize: MapSize) extends LayerModel { 58 | 59 | require(kernels.length == inMapNum) 60 | require(kernels.length > 0 && kernels(0).length == outMapNum) 61 | require(bias.length == outMapNum) 62 | 63 | private val kernelSize = new MapSize(kernels(0)(0).rows, kernels(0)(0).cols) 64 | private val outputSize = inputMapSize.subtract(kernelSize, 1) 65 | 66 | override val size = kernelSize.x * kernelSize.y * inMapNum * outMapNum + bias.length 67 | 68 | /** 69 | * @param data each column contains all the data for one sample, with size of 70 | * inMapNum * inputMapSize. 71 | */ 72 | override def eval(data: BDM[Double]): BDM[Double] = { 73 | require(data.rows == this.inMapNum * inputMapSize.x * inputMapSize.y) 74 | // local copy 75 | val inMapNum = this.inMapNum 76 | val outMapNum = this.outMapNum 77 | val kernels = this.kernels 78 | val bias = this.bias 79 | 80 | val batchOutput = new BDM[Double](outputSize.x * outputSize.y * outMapNum, data.cols) 81 | (0 until data.cols).foreach{ col => 82 | val inputMaps = FeatureMapRolling.extractMaps(data(::, col), inputMapSize) 83 | val output = new Array[BDM[Double]](this.outMapNum) 84 | var j = 0 85 | while (j < outMapNum) { 86 | var sum: BDM[Double] = ConvolutionalLayerModel.convValid(inputMaps(0), kernels(0)(j)) 87 | var i = 1 88 | while (i < inMapNum) { 89 | sum += ConvolutionalLayerModel.convValid(inputMaps(i), kernels(i)(j)) 90 | i += 1 91 | } 92 | output(j) = sum + bias(j) 93 | j += 1 94 | } 95 | // reorganize feature maps to a single column in a dense matrix. 96 | batchOutput(::, col) := FeatureMapRolling.mergeMaps(output) 97 | } 98 | batchOutput 99 | } 100 | 101 | override def prevDelta(nextDelta: BDM[Double], output: BDM[Double]): BDM[Double] = { 102 | 103 | // local copy 104 | val inMapNum = this.inMapNum 105 | val outMapNum = this.outMapNum 106 | val kernels = this.kernels 107 | 108 | val batchDelta = new BDM[Double](inputMapSize.x * inputMapSize.y * inMapNum, output.cols) 109 | (0 until output.cols).foreach{ col => 110 | val nextDeltaMaps = FeatureMapRolling.extractMaps(nextDelta(::, col), outputSize) 111 | val deltas = new Array[BDM[Double]](inMapNum) 112 | var i = 0 113 | while (i < inMapNum) { 114 | // rotate kernel by 180 degrees and get full convolution 115 | var sum: BDM[Double] = ConvolutionalLayerModel.convFull(nextDeltaMaps(0), 116 | flipud(fliplr(kernels(i)(0)))) 117 | var j = 1 118 | while (j < outMapNum) { 119 | sum += ConvolutionalLayerModel.convFull(nextDeltaMaps(j), flipud(fliplr(kernels(i)(j)))) 120 | j += 1 121 | } 122 | deltas(i) = sum 123 | i += 1 124 | } 125 | // reorganize delta maps to a single column in a dense matrix. 126 | batchDelta(::, col) := FeatureMapRolling.mergeMaps(deltas) 127 | } 128 | 129 | batchDelta 130 | } 131 | 132 | override def grad(delta: BDM[Double], input: BDM[Double]): Array[Double] = { 133 | 134 | var batchGradient = new BDV[Double](this.size) 135 | (0 until input.cols).foreach{ col => 136 | val inputMaps = FeatureMapRolling.extractMaps(input(::, col), inputMapSize) 137 | val deltaMaps = FeatureMapRolling.extractMaps(delta(::, col), outputSize) 138 | 139 | val kernelGradient = getKernelsGradient(inputMaps, deltaMaps) 140 | val biasGradient = getBiasGradient(deltaMaps) 141 | val single = ConvolutionalLayerModel.roll(kernelGradient, biasGradient) 142 | batchGradient = batchGradient + single 143 | } 144 | 145 | (batchGradient / input.cols.toDouble).toArray 146 | } 147 | 148 | /** 149 | * get kernels gradient 150 | */ 151 | private def getKernelsGradient(input: Array[BDM[Double]], 152 | delta: Array[BDM[Double]]): Array[Array[BDM[Double]]] = { 153 | val outMapNum = this.outMapNum 154 | val inMapNum = this.inMapNum 155 | val kernelGradient = Array.ofDim[BDM[Double]](inMapNum, outMapNum) 156 | var j = 0 157 | while (j < outMapNum) { 158 | var i = 0 159 | while (i < inMapNum) { 160 | kernelGradient(i)(j) = ConvolutionalLayerModel.convValid(input(i), delta(j)) 161 | i += 1 162 | } 163 | j += 1 164 | } 165 | kernelGradient 166 | } 167 | 168 | /** 169 | * get bias gradient 170 | * 171 | * @param deltas errors of this layer 172 | */ 173 | private def getBiasGradient(deltas: Array[BDM[Double]]): Array[Double] = { 174 | val outMapNum = this.outMapNum 175 | 176 | val gradient = new Array[Double](outMapNum) 177 | var j: Int = 0 178 | while (j < outMapNum) { 179 | gradient(j) = sum(deltas(j)) 180 | j += 1 181 | } 182 | gradient 183 | } 184 | 185 | override def weights(): Vector = Vectors.fromBreeze(ConvolutionalLayerModel.roll(kernels, bias)) 186 | 187 | } 188 | 189 | /** 190 | * Fabric for Convolutional layer models 191 | */ 192 | private[ann] object ConvolutionalLayerModel { 193 | 194 | /** 195 | * Creates a model of Convolutional layer 196 | * @param layer layer properties 197 | * @param weights vector with weights 198 | * @param position position of weights in the vector 199 | * @return model of Convolutional layer 200 | */ 201 | def apply(layer: ConvolutionalLayer, weights: Vector, position: Int): ConvolutionalLayerModel = { 202 | val (w, b) = unroll(weights, position, 203 | layer.numInMap, 204 | layer.numOutMap, 205 | layer.kernelSize, 206 | layer.inputMapSize) 207 | new ConvolutionalLayerModel(layer.numInMap, layer.numOutMap, w, b, layer.inputMapSize) 208 | } 209 | 210 | /** 211 | * Creates a model of Affine layer 212 | * @param layer layer properties 213 | * @param seed seed 214 | * @return model of Affine layer 215 | */ 216 | def apply(layer: ConvolutionalLayer, seed: Long): ConvolutionalLayerModel = { 217 | val bias = new Array[Double](layer.numOutMap) // bias init to 0 218 | val kernel = Array.ofDim[BDM[Double]](layer.numInMap, layer.numOutMap) 219 | for (i <- 0 until layer.numInMap) 220 | for (j <- 0 until layer.numOutMap) 221 | kernel(i)(j) = (BDM.rand[Double](layer.kernelSize.x, layer.kernelSize.y) - 0.05) / 10.0 222 | 223 | new ConvolutionalLayerModel(layer.numInMap, layer.numOutMap, kernel, bias, layer.inputMapSize) 224 | } 225 | 226 | /** 227 | * roll kernels and bias into the array format for [[org.apache.spark.mllib.optimization]] 228 | */ 229 | private[ann] def roll(kernels: Array[Array[BDM[Double]]], bias: Array[Double]): BDV[Double] = { 230 | val rows = kernels.length 231 | val cols = kernels(0).length 232 | val m = kernels(0)(0).rows * kernels(0)(0).cols 233 | val result = new Array[Double](m * rows * cols + bias.length) 234 | var offset = 0 235 | var i = 0 236 | while(i < rows){ 237 | var j = 0 238 | while(j < cols){ 239 | System.arraycopy(kernels(i)(j).toArray, 0, result, offset, m) 240 | offset += m 241 | j += 1 242 | } 243 | i += 1 244 | } 245 | System.arraycopy(bias, 0, result, offset, bias.length) 246 | new BDV(result) 247 | } 248 | 249 | /** 250 | * Unrolls the weights from the vector 251 | * @param weights vector with weights 252 | * @param position position of weights for this layer 253 | * @param numIn number of layer inputs 254 | * @param numOut number of layer outputs 255 | * @return matrix A and vector b 256 | */ 257 | def unroll(weights: Vector, 258 | position: Int, 259 | numIn: Int, 260 | numOut: Int, 261 | kernelSize: MapSize, 262 | inputSize: MapSize): (Array[Array[BDM[Double]]], Array[Double]) = { 263 | val weightsCopy = weights.toArray 264 | var offset = position 265 | val kernels = new Array[Array[BDM[Double]]](numIn) 266 | for(i <- 0 until numIn){ 267 | kernels(i) = new Array[BDM[Double]](numOut) 268 | for(j <- 0 until numOut){ 269 | val a = new BDM[Double](kernelSize.x, kernelSize.y, weightsCopy, offset) 270 | kernels(i)(j) = a 271 | offset += kernelSize.x * kernelSize.y 272 | } 273 | } 274 | 275 | val b = new BDV[Double](weightsCopy, offset, 1, numOut).toArray 276 | (kernels, b) 277 | } 278 | 279 | /** 280 | * full conv 281 | * 282 | * @param matrix 283 | * @param kernel 284 | * @return 285 | */ 286 | private[ann] def convFull(matrix: BDM[Double], kernel: BDM[Double]): BDM[Double] = { 287 | val m: Int = matrix.rows 288 | val n: Int = matrix.cols 289 | val km: Int = kernel.rows 290 | val kn: Int = kernel.cols 291 | val extendMatrix = new BDM[Double](m + 2 * (km - 1), n + 2 * (kn - 1)) 292 | var i = 0 293 | var j = 0 294 | while (i < m) { 295 | while (j < n) { 296 | extendMatrix(i + km - 1, j + kn - 1) = matrix(i, j) 297 | j += 1 298 | } 299 | i += 1 300 | } 301 | convValid(extendMatrix, kernel) 302 | } 303 | 304 | /** 305 | * valid conv 306 | * 307 | * @param matrix 308 | * @param kernel 309 | * @return 310 | */ 311 | private[ann] def convValid(matrix: BDM[Double], kernel: BDM[Double]): BDM[Double] = { 312 | val m: Int = matrix.rows 313 | val n: Int = matrix.cols 314 | val km: Int = kernel.rows 315 | val kn: Int = kernel.cols 316 | val kns: Int = n - kn + 1 317 | val kms: Int = m - km + 1 318 | val outMatrix: BDM[Double] = new BDM[Double](kms, kns) 319 | var i = 0 320 | while (i < kms) { 321 | var j = 0 322 | while (j < kns) { 323 | var sum = 0.0 324 | for (ki <- 0 until km) { 325 | for (kj <- 0 until kn) 326 | sum += matrix(i + ki, j + kj) * kernel(ki, kj) 327 | } 328 | outMatrix(i, j) = sum 329 | j += 1 330 | } 331 | i += 1 332 | } 333 | outMatrix 334 | } 335 | } -------------------------------------------------------------------------------- /src/communityInterface/Driver.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.spark.ml.ann 19 | 20 | import org.apache.log4j.{Logger, Level} 21 | import breeze.linalg.{DenseMatrix => BDM} 22 | import org.apache.spark.mllib.linalg.{Vectors, Vector} 23 | import org.apache.spark.{SparkContext, SparkConf} 24 | 25 | object CNNDriver { 26 | 27 | def main(args: Array[String]) { 28 | 29 | val myLayers = new Array[Layer](8) 30 | myLayers(0) = new ConvolutionalLayer(1, 6, kernelSize = new MapSize(5, 5), inputMapSize = new MapSize(28, 28)) 31 | myLayers(1) = new FunctionalLayer(new SigmoidFunction()) 32 | myLayers(2) = new MeanPoolingLayer(new MapSize(2, 2), new MapSize(24, 24)) 33 | myLayers(3) = new ConvolutionalLayer(6, 12, new MapSize(5, 5), new MapSize(12, 12)) 34 | myLayers(4) = new FunctionalLayer(new SigmoidFunction()) 35 | myLayers(5) = new MeanPoolingLayer(new MapSize(2, 2), new MapSize(8, 8)) 36 | myLayers(6) = new ConvolutionalLayer(12, 12, new MapSize(4, 4), new MapSize(4, 4)) 37 | myLayers(7) = new FunctionalLayer(new SigmoidFunction()) 38 | val topology = FeedForwardTopology(myLayers) 39 | 40 | Logger.getLogger("org").setLevel(Level.WARN) 41 | Logger.getLogger("akka").setLevel(Level.WARN) 42 | val conf = new SparkConf().setMaster("local[8]").setAppName("ttt") 43 | val sc = new SparkContext(conf) 44 | val lines = sc.textFile("dataset/train.format", 8) 45 | val data = lines.map(line => line.split(",")).map(arr => arr.map(_.toDouble)) 46 | .map(arr => { 47 | val target = new Array[Double](12) 48 | target(arr(784).toInt) = 1 49 | val in = Vector2BDM(Vectors.dense(arr.slice(0, 784))) 50 | (Vectors.fromBreeze(in.toDenseVector), Vectors.dense(target)) 51 | }).cache() 52 | 53 | val feedForwardTrainer = new FeedForwardTrainer(topology, 784, 12) 54 | 55 | feedForwardTrainer.setStackSize(4) // CNN does not benefit from the stacked data 56 | // .LBFGSOptimizer.setNumIterations(20) 57 | .SGDOptimizer 58 | .setMiniBatchFraction(0.002) 59 | .setConvergenceTol(0) 60 | .setNumIterations(1000) 61 | .setUpdater(new CNNUpdater(0.85)) 62 | 63 | for(iter <- 1 to 1000){ 64 | val start = System.nanoTime() 65 | val mlpModel = feedForwardTrainer.train(data) 66 | feedForwardTrainer.setWeights(mlpModel.weights()) 67 | 68 | println(s"Training time $iter: " + (System.nanoTime() - start) / 1e9) 69 | 70 | // predict 71 | val right = data.filter(v => mlpModel.predict(v._1).argmax == v._2.argmax).count() 72 | val precision = right.toDouble / data.count() 73 | println(s"right: $right, count: ${data.count()}, precision: $precision") 74 | } 75 | } 76 | 77 | def Vector2BDM(record: Vector): BDM[Double] = { 78 | val mapSize = new MapSize(28, 28) 79 | val m = new BDM[Double](mapSize.x, mapSize.y) 80 | var i: Int = 0 81 | while (i < mapSize.x) { 82 | var j: Int = 0 83 | while (j < mapSize.y) { 84 | m(i, j) = record(mapSize.x * i + j) 85 | j += 1 86 | } 87 | i += 1 88 | } 89 | m 90 | } 91 | 92 | } 93 | -------------------------------------------------------------------------------- /src/communityInterface/MapSize.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.spark.ml.ann 19 | 20 | import java.io.Serializable 21 | import breeze.linalg.{DenseMatrix => BDM, DenseVector => BDV} 22 | 23 | /** 24 | * Utility class for managing feature map size. x and y can be different 25 | */ 26 | class MapSize(var x: Int, var y: Int) extends Serializable { 27 | 28 | /** 29 | * divide a scale with other scale 30 | * 31 | * @param scaleSize 32 | * @return 33 | */ 34 | private[ann] def divide(scaleSize: MapSize): MapSize = { 35 | val x: Int = this.x / scaleSize.x 36 | val y: Int = this.y / scaleSize.y 37 | if (x * scaleSize.x != this.x || y * scaleSize.y != this.y){ 38 | throw new RuntimeException(this + "can not be divided" + scaleSize) 39 | } 40 | new MapSize(x, y) 41 | } 42 | 43 | private[ann] def multiply(scaleSize: MapSize): MapSize = { 44 | val x: Int = this.x * scaleSize.x 45 | val y: Int = this.y * scaleSize.y 46 | new MapSize(x, y) 47 | } 48 | 49 | /** 50 | * subtract a scale and add append 51 | */ 52 | private[ann] def subtract(other: MapSize, append: Int): MapSize = { 53 | val x: Int = this.x - other.x + append 54 | val y: Int = this.y - other.y + append 55 | new MapSize(x, y) 56 | } 57 | } 58 | 59 | /** 60 | * Utility class for converting feature maps to and from a vector (one column in a matrix). 61 | * The conversion is necessary for compatibility with current Layer interface. 62 | */ 63 | object FeatureMapRolling{ 64 | private[ann] def extractMaps(bdv: BDV[Double], size: MapSize): Array[BDM[Double]] = { 65 | val v = bdv.toArray 66 | val mapSize = size.x * size.y 67 | val mapNum = v.length / mapSize 68 | val maps = new Array[BDM[Double]](mapNum) 69 | var i = 0 70 | var offset = 0 71 | while(i < mapNum){ 72 | maps(i) = new BDM(size.x, size.y, v, offset) 73 | offset += mapSize 74 | i += 1 75 | } 76 | maps 77 | } 78 | 79 | private[ann] def mergeMaps(data: Array[BDM[Double]]): BDV[Double] = { 80 | require(data.length > 0) 81 | val num = data.length 82 | val size = data(0).size 83 | val arr = new Array[Double](size * num) 84 | var offset = 0 85 | var i = 0 86 | while (i < num){ 87 | System.arraycopy(data(i).toArray, 0, arr, offset, size) 88 | offset += size 89 | i += 1 90 | } 91 | val outBDM = new BDV[Double](arr) 92 | outBDM 93 | } 94 | } -------------------------------------------------------------------------------- /src/communityInterface/MeanPoolingLayer.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.spark.ml.ann 19 | 20 | import breeze.linalg.{DenseMatrix => BDM, _} 21 | import org.apache.spark.mllib.linalg.{Vector, Vectors} 22 | 23 | /** 24 | * Layer properties of Mean Pooling transformations 25 | * @param poolingSize number of inputs 26 | */ 27 | private[ann] class MeanPoolingLayer(val poolingSize: MapSize, val inputSize: MapSize) extends Layer { 28 | 29 | override def getInstance(weights: Vector, position: Int): LayerModel = getInstance(0L) 30 | 31 | override def getInstance(seed: Long = 11L): LayerModel = MeanPoolingLayerModel(this, inputSize) 32 | } 33 | 34 | private[ann] class MeanPoolingLayerModel private(poolingSize: MapSize, 35 | inputSize: MapSize) extends LayerModel { 36 | 37 | val outputSize = inputSize.divide(poolingSize) 38 | 39 | override val size = 0 40 | 41 | override def eval(data: BDM[Double]): BDM[Double] = { 42 | val inMapNum = data.rows / (inputSize.x * inputSize.y) 43 | val batchOutput = new BDM[Double](outputSize.x * outputSize.y * inMapNum , data.cols) 44 | (0 until data.cols).foreach { col => 45 | val inputMaps = FeatureMapRolling.extractMaps(data(::, col), inputSize) 46 | val inputMapNum = inputMaps.length 47 | val scaleSize: MapSize = this.poolingSize 48 | 49 | val output = new Array[BDM[Double]](inputMapNum) 50 | var i = 0 51 | while (i < inputMapNum) { 52 | val inputMap: BDM[Double] = inputMaps(i) 53 | output(i) = MeanPoolingLayerModel.avgPooling(inputMap, scaleSize) 54 | i += 1 55 | } 56 | batchOutput(::, col) := FeatureMapRolling.mergeMaps(output) 57 | } 58 | batchOutput 59 | } 60 | 61 | override def prevDelta(nextDelta: BDM[Double], output: BDM[Double]): BDM[Double] = { 62 | 63 | val inMapNum = output.rows / (outputSize.x * outputSize.y) 64 | val batchDelta = new BDM[Double](inputSize.x * inputSize.y * inMapNum, output.cols) 65 | (0 until output.cols).foreach { col => 66 | val nextDeltaMaps = FeatureMapRolling.extractMaps(nextDelta(::, col), outputSize) 67 | val mapNum: Int = nextDeltaMaps.length 68 | val errors = new Array[BDM[Double]](mapNum) 69 | var m: Int = 0 70 | val scale: MapSize = this.poolingSize 71 | while (m < mapNum) { 72 | val nextError: BDM[Double] = nextDeltaMaps(m) 73 | val ones = BDM.ones[Double](scale.x, scale.y) 74 | val outMatrix = kron(nextError, ones) 75 | errors(m) = outMatrix 76 | m += 1 77 | } 78 | 79 | batchDelta(::, col) := FeatureMapRolling.mergeMaps(errors) 80 | } 81 | batchDelta 82 | } 83 | 84 | override def grad(delta: BDM[Double], input: BDM[Double]): Array[Double] = { 85 | new Array[Double](0) 86 | } 87 | 88 | override def weights(): Vector = Vectors.dense(new Array[Double](0)) 89 | 90 | } 91 | 92 | /** 93 | * Fabric for mean pooling layer models 94 | */ 95 | private[ann] object MeanPoolingLayerModel { 96 | 97 | /** 98 | * Creates a model of Mean Pooling layer 99 | * @param layer layer properties 100 | * @return model of Mean Pooling layer 101 | */ 102 | def apply(layer: MeanPoolingLayer, inputSize: MapSize): MeanPoolingLayerModel = { 103 | new MeanPoolingLayerModel(layer.poolingSize, inputSize: MapSize) 104 | } 105 | 106 | /** 107 | * return a new matrix that has been scaled down 108 | * 109 | * @param matrix 110 | */ 111 | private[ann] def avgPooling(matrix: BDM[Double], scale: MapSize): BDM[Double] = { 112 | val m: Int = matrix.rows 113 | val n: Int = matrix.cols 114 | val scaleX = scale.x 115 | val scaleY = scale.y 116 | val sm: Int = m / scaleX 117 | val sn: Int = n / scaleY 118 | val outMatrix = new BDM[Double](sm, sn) 119 | val size = scaleX * scaleY 120 | 121 | var i = 0 // iterate through blocks 122 | while (i < sm) { 123 | var j = 0 124 | while (j < sn) { 125 | var sum = 0.0 // initial to left up corner of the block 126 | var bi = i * scaleX // block i 127 | val biMax = (i + 1) * scaleX 128 | val bjMax = (j + 1) * scaleY 129 | while (bi < biMax) { 130 | var bj = j * scaleY // block j 131 | while (bj < bjMax) { 132 | sum += matrix(bi, bj) 133 | bj += 1 134 | } 135 | bi += 1 136 | } 137 | outMatrix(i, j) = sum / size 138 | j += 1 139 | } 140 | i += 1 141 | } 142 | outMatrix 143 | } 144 | } -------------------------------------------------------------------------------- /src/main/CNN.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package hhbyyh.mCNN 18 | 19 | import java.util.{ArrayList, List} 20 | 21 | import breeze.linalg.{DenseMatrix => BDM, DenseVector => BDV, _} 22 | import org.apache.spark.Logging 23 | import org.apache.spark.rdd.RDD 24 | 25 | /** 26 | * Builder class to define CNN layers structure 27 | * The second layer from the last can not be a convolution layer, 28 | * Typical layers: input, conv, samp, conv, samp 29 | */ 30 | class CNNTopology { 31 | var mLayers: List[CNNLayer] = new ArrayList[CNNLayer] 32 | 33 | def this(layer: CNNLayer) { 34 | this() 35 | mLayers.add(layer) 36 | } 37 | 38 | def addLayer(layer: CNNLayer): CNNTopology = { 39 | mLayers.add(layer) 40 | this 41 | } 42 | } 43 | 44 | /** 45 | * Convolution neural network 46 | */ 47 | class CNN private extends Serializable with Logging{ 48 | private var ALPHA: Double = 0.85 49 | private var layers: List[CNNLayer] = null 50 | private var layerNum: Int = 0 51 | private var maxIterations = 10 52 | private var batchSize = 100 53 | 54 | def this(layerBuilder: CNNTopology) { 55 | this() 56 | layers = layerBuilder.mLayers 57 | layerNum = layers.size 58 | } 59 | 60 | def setMiniBatchSize(batchSize: Int): this.type = { 61 | this.batchSize = batchSize 62 | this 63 | } 64 | 65 | /** 66 | * Maximum number of iterations for learning. 67 | */ 68 | def getMaxIterations: Int = maxIterations 69 | 70 | /** 71 | * Maximum number of iterations for learning. 72 | * (default = 20) 73 | */ 74 | def setMaxIterations(maxIterations: Int): this.type = { 75 | this.maxIterations = maxIterations 76 | this 77 | } 78 | 79 | def trainOneByOne(trainSet: RDD[(Double, Array[BDM[Double]])]) { 80 | var t = 0 81 | val trainSize = trainSet.count().toInt 82 | val dataArr = trainSet.collect() 83 | while (t < maxIterations) { 84 | val epochsNum = trainSize 85 | var right = 0 86 | var count = 0 87 | var i = 0 88 | while (i < epochsNum) { 89 | val record = dataArr(i) 90 | val result = train(record) 91 | if (result._1) right += 1 92 | count += 1 93 | val gradient: Array[(Array[Array[BDM[Double]]], Array[Double])] = result._2 94 | updateParams(gradient, 1) 95 | i += 1 96 | if(i % 1000 == 0){ 97 | println(s"$t:\t$i\t samples precision $right/$count = " + 1.0 * right / count) 98 | right = 0 99 | count = 0 100 | } 101 | } 102 | val p = 1.0 * right / count 103 | if (t % 10 == 1 && p > 0.96) { 104 | ALPHA = 0.001 + ALPHA * 0.9 105 | } 106 | t += 1 107 | } 108 | } 109 | 110 | def train(trainSet: RDD[(Double, Array[BDM[Double]])]) { 111 | var t = 0 112 | val trainSize = trainSet.count().toInt 113 | val gZero = train(trainSet.first)._2 114 | gZero.foreach(tu => if (tu != null){ 115 | tu._1.foreach(m => m.foreach(x => x -= x)) 116 | (0 until tu._2.length).foreach(i => tu._2(i) = 0) 117 | } 118 | ) 119 | var totalCount = 0 120 | var totalRight = 0 121 | while (t < maxIterations) { 122 | val (gradientSum, right, count) = trainSet 123 | .sample(false, batchSize.toDouble/trainSize, 42 + t) 124 | .treeAggregate((gZero, 0, 0))( 125 | seqOp = (c, v) => { 126 | val result = train(v) 127 | val gradient = result._2 128 | val right = if (result._1) 1 else 0 129 | (CNN.combineGradient(c._1, gradient), c._2 + right, c._3 + 1) 130 | }, 131 | combOp = (c1, c2) => { 132 | (CNN.combineGradient(c1._1, c2._1), c1._2 + c2._2, c1._3 + c2._3) 133 | }) 134 | t += 1 135 | if (count > 0){ 136 | updateParams(gradientSum, count) 137 | val p = 1.0 * totalRight / totalCount 138 | if (t % 10 == 1 && p > 0.96) { 139 | ALPHA = 0.001 + ALPHA * 0.9 140 | } 141 | totalCount += count 142 | totalRight += right 143 | if (totalCount > 10000){ 144 | logInfo(s"precision $totalRight/$totalCount = $p") 145 | totalCount = 0 146 | totalRight = 0 147 | } 148 | } 149 | } 150 | } 151 | 152 | def predict(record: Array[BDM[Double]]): Int = { 153 | val outputs: Array[Array[BDM[Double]]] = forward(record) 154 | val outValues: Array[Double] = outputs(layerNum).map(m => m(0, 0)) 155 | CNN.getMaxIndex(outValues) 156 | } 157 | 158 | /** 159 | * train one record 160 | * 161 | * @return (isRight, gradient) 162 | */ 163 | private def train(record: (Double, Array[BDM[Double]])) 164 | : (Boolean, Array[(Array[Array[BDM[Double]]], Array[Double])]) = { 165 | val outputs = forward(record._2) 166 | val (right, errors) = backPropagation(record._1, outputs) 167 | val gradient = getGradient(outputs, errors) 168 | (right, gradient) 169 | } 170 | 171 | /** 172 | * forward for one record 173 | * 174 | * @param record 175 | */ 176 | private def forward(record: Array[BDM[Double]]): Array[Array[BDM[Double]]] = { 177 | // outputs(i + 1) contains output for layer(i) 178 | val outputs = new Array[Array[BDM[Double]]](layers.size + 1) 179 | outputs(0) = record 180 | var l: Int = 0 181 | while (l < layers.size) { 182 | val layer: CNNLayer = layers.get(l) 183 | outputs(l + 1) = layer.forward(outputs(l)) 184 | l += 1 185 | } 186 | outputs 187 | } 188 | 189 | /** 190 | * run BP and get errors for all layers 191 | * 192 | * @return (right, errors for all layers) 193 | */ 194 | private def backPropagation( 195 | label: Double, 196 | outputs: Array[Array[BDM[Double]]]) 197 | : (Boolean, Array[Array[BDM[Double]]]) = { 198 | val errors = new Array[Array[BDM[Double]]](layers.size + 1) 199 | val (right, error) = setOutLayerErrors(label.toInt, outputs(layerNum)) 200 | errors(layerNum) = error 201 | var l: Int = layerNum - 2 202 | while (l >= 0) { 203 | val layer: CNNLayer = layers.get(l + 1) 204 | errors(l + 1) = layer.prevDelta(errors(l + 2), outputs(l + 1)) 205 | l -= 1 206 | } 207 | (right, errors) 208 | } 209 | 210 | private def getGradient( 211 | outputs: Array[Array[BDM[Double]]], 212 | errors: Array[Array[BDM[Double]]]): Array[(Array[Array[BDM[Double]]], Array[Double])] = { 213 | var l: Int = 0 214 | val gradient = new Array[(Array[Array[BDM[Double]]], Array[Double])](layerNum) 215 | while (l < layerNum) { 216 | val layer: CNNLayer = layers.get(l) 217 | gradient(l) = layer.grad(errors(l + 1), outputs(l)) 218 | l += 1 219 | } 220 | gradient 221 | } 222 | 223 | private def updateParams( 224 | gradient: Array[(Array[Array[BDM[Double]]], Array[Double])], 225 | batchSize: Int): Unit = { 226 | var l: Int = 0 227 | while (l < layerNum) { 228 | val layer: CNNLayer = layers.get(l) 229 | if(layer.isInstanceOf[ConvolutionLayer]){ 230 | updateKernels(layer.asInstanceOf[ConvolutionLayer], gradient(l)._1, batchSize) 231 | updateBias(layer.asInstanceOf[ConvolutionLayer], gradient(l)._2, batchSize) 232 | } 233 | l += 1 234 | } 235 | } 236 | 237 | private def updateKernels( 238 | layer: ConvolutionLayer, 239 | gradient: Array[Array[BDM[Double]]], batchSize: Int): Unit = { 240 | val len = gradient.length 241 | val width = gradient(0).length 242 | var j = 0 243 | while (j < width) { 244 | var i = 0 245 | while (i < len) { 246 | // update kernel 247 | val deltaKernel = gradient(i)(j) / batchSize.toDouble * ALPHA 248 | layer.getKernel(i, j) += deltaKernel 249 | i += 1 250 | } 251 | j += 1 252 | } 253 | } 254 | 255 | private def updateBias(layer: ConvolutionLayer, gradient: Array[Double], batchSize: Int): Unit = { 256 | val gv = new BDV[Double](gradient) 257 | layer.getBias += gv * ALPHA / batchSize.toDouble 258 | } 259 | 260 | /** 261 | * set errors for output layer 262 | * 263 | * @param label 264 | * @return 265 | */ 266 | private def setOutLayerErrors(label: Int, 267 | output: Array[BDM[Double]]): (Boolean, Array[BDM[Double]]) = { 268 | val outputLayer: CNNLayer = layers.get(layerNum - 1) 269 | val mapNum: Int = outputLayer.asInstanceOf[ConvolutionLayer].getOutMapNum 270 | val target: Array[Double] = new Array[Double](mapNum) 271 | val outValues: Array[Double] = output.map(m => m(0, 0)) 272 | target(label) = 1 273 | val layerError: Array[BDM[Double]] = (0 until mapNum).map(i => { 274 | val errorMatrix = new BDM[Double](1, 1) 275 | errorMatrix(0, 0) = outValues(i) * (1 - outValues(i)) * (target(i) - outValues(i)) 276 | errorMatrix 277 | }).toArray 278 | val outClass = CNN.getMaxIndex(outValues) 279 | (label == outClass, layerError) 280 | } 281 | } 282 | 283 | 284 | object CNN { 285 | 286 | private[mCNN] def getMaxIndex(out: Array[Double]): Int = { 287 | var max: Double = out(0) 288 | var index: Int = 0 289 | var i: Int = 1 290 | while (i < out.length) { 291 | if (out(i) > max) { 292 | max = out(i) 293 | index = i 294 | } 295 | i += 1 296 | } 297 | index 298 | } 299 | 300 | private[mCNN] def combineGradient( 301 | g1: Array[(Array[Array[BDM[Double]]], Array[Double])], 302 | g2: Array[(Array[Array[BDM[Double]]], Array[Double])]): 303 | Array[(Array[Array[BDM[Double]]], Array[Double])] = { 304 | 305 | val l = g1.length 306 | var li = 0 307 | while(li < l){ 308 | if (g1(li) != null){ 309 | // kernel 310 | val layer = g1(li)._1 311 | val x = layer.length 312 | var xi = 0 313 | while(xi < x){ 314 | val line: Array[BDM[Double]] = layer(xi) 315 | val y = line.length 316 | var yi = 0 317 | while(yi < y){ 318 | line(yi) += g2(li)._1(xi)(yi) 319 | yi += 1 320 | } 321 | xi += 1 322 | } 323 | 324 | // bias 325 | val b = g1(li)._2 326 | val len = b.length 327 | var bi = 0 328 | while(bi < len){ 329 | b(bi) = b(bi) + g2(li)._2(bi) 330 | bi += 1 331 | } 332 | } 333 | li += 1 334 | } 335 | g1 336 | } 337 | } 338 | -------------------------------------------------------------------------------- /src/main/CNNLayer.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package hhbyyh.mCNN 18 | 19 | import java.io.Serializable 20 | 21 | import breeze.linalg.{DenseMatrix => BDM, DenseVector => BDV, fliplr, flipud, sum} 22 | import breeze.numerics._ 23 | 24 | abstract class CNNLayer private[mCNN] extends Serializable { 25 | 26 | def forward(input: Array[BDM[Double]]): Array[BDM[Double]] = input 27 | 28 | def prevDelta(nextDelta: Array[BDM[Double]], input: Array[BDM[Double]]): Array[BDM[Double]] 29 | 30 | def grad(delta: Array[BDM[Double]], 31 | layerInput: Array[BDM[Double]]): (Array[Array[BDM[Double]]], Array[Double]) = null 32 | } 33 | 34 | object CNNLayer { 35 | 36 | def buildConvolutionLayer(inMapNum: Int, outMapNum: Int, kernelSize: Scale): CNNLayer = { 37 | val layer = new ConvolutionLayer(inMapNum, outMapNum, kernelSize) 38 | layer 39 | } 40 | 41 | def buildMeanPoolingLayer(scaleSize: Scale): CNNLayer = { 42 | val layer = new MeanPoolingLayer(scaleSize) 43 | layer 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/ConvolutionLayer.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package hhbyyh.mCNN 18 | 19 | import breeze.linalg.{DenseMatrix => BDM, DenseVector => BDV, fliplr, flipud, sum} 20 | import breeze.numerics._ 21 | 22 | class ConvolutionLayer private[mCNN](inMapNum: Int, outMapNum: Int, kernelSize: Scale) 23 | extends CNNLayer{ 24 | 25 | private var bias: BDV[Double] = null 26 | private var kernel: Array[Array[BDM[Double]]] = null 27 | initBias(inMapNum) 28 | initKernel(inMapNum) 29 | 30 | def getOutMapNum: Int = outMapNum 31 | 32 | private[mCNN] def initBias(frontMapNum: Int) { 33 | this.bias = BDV.zeros[Double](outMapNum) 34 | } 35 | 36 | private[mCNN] def initKernel(frontMapNum: Int) { 37 | this.kernel = Array.ofDim[BDM[Double]](frontMapNum, outMapNum) 38 | for (i <- 0 until frontMapNum) 39 | for (j <- 0 until outMapNum) 40 | kernel(i)(j) = (BDM.rand[Double](kernelSize.x, kernelSize.y) - 0.05) / 10.0 41 | } 42 | 43 | def getBias: BDV[Double] = bias 44 | def setBias(mapNo: Int, value: Double): this.type = { 45 | bias(mapNo) = value 46 | this 47 | } 48 | 49 | def getKernelSize: Scale = kernelSize 50 | def getKernel(i: Int, j: Int): BDM[Double] = kernel(i)(j) 51 | 52 | override def forward(input: Array[BDM[Double]]): Array[BDM[Double]] = { 53 | val mapNum: Int = this.outMapNum 54 | val lastMapNum: Int = input.length 55 | val output = new Array[BDM[Double]](mapNum) 56 | var j = 0 57 | val oldBias = this.getBias 58 | while (j < mapNum) { 59 | var sum: BDM[Double] = null 60 | var i = 0 61 | while (i < lastMapNum) { 62 | val lastMap = input(i) 63 | val kernel = this.getKernel(i, j) 64 | if (sum == null) { 65 | sum = ConvolutionLayer.convnValid(lastMap, kernel) 66 | } 67 | else { 68 | sum += ConvolutionLayer.convnValid(lastMap, kernel) 69 | } 70 | i += 1 71 | } 72 | sum = sigmoid(sum + oldBias(j)) 73 | output(j) = sum 74 | j += 1 75 | } 76 | output 77 | } 78 | 79 | override def prevDelta(nextDelta: Array[BDM[Double]], 80 | layerInput: Array[BDM[Double]]): Array[BDM[Double]] = { 81 | 82 | val mapNum: Int = layerInput.length 83 | val nextMapNum: Int = this.getOutMapNum 84 | val errors = new Array[BDM[Double]](mapNum) 85 | var i = 0 86 | while (i < mapNum) { 87 | var sum: BDM[Double] = null // sum for every kernel 88 | var j = 0 89 | while (j < nextMapNum) { 90 | val nextError = nextDelta(j) 91 | val kernel = this.getKernel(i, j) 92 | // rotate kernel by 180 degrees and get full convolution 93 | if (sum == null) { 94 | sum = ConvolutionLayer.convnFull(nextError, flipud(fliplr(kernel))) 95 | } 96 | else { 97 | sum += ConvolutionLayer.convnFull(nextError, flipud(fliplr(kernel))) 98 | } 99 | j += 1 100 | } 101 | errors(i) = sum 102 | i += 1 103 | } 104 | errors 105 | } 106 | 107 | override def grad(layerError: Array[BDM[Double]], 108 | input: Array[BDM[Double]]): (Array[Array[BDM[Double]]], Array[Double]) = { 109 | val kernelGradient = getKernelsGradient(layerError, input) 110 | val biasGradient = getBiasGradient(layerError) 111 | (kernelGradient, biasGradient) 112 | } 113 | 114 | /** 115 | * get kernels gradient 116 | */ 117 | private def getKernelsGradient(layerError: Array[BDM[Double]], 118 | input: Array[BDM[Double]]): Array[Array[BDM[Double]]] = { 119 | val mapNum: Int = this.getOutMapNum 120 | val lastMapNum: Int = input.length 121 | val delta = Array.ofDim[BDM[Double]](lastMapNum, mapNum) 122 | var j = 0 123 | while (j < mapNum) { 124 | var i = 0 125 | while (i < lastMapNum) { 126 | val error = layerError(j) 127 | val deltaKernel = ConvolutionLayer.convnValid(input(i), error) 128 | delta(i)(j) = deltaKernel 129 | i += 1 130 | } 131 | j += 1 132 | } 133 | delta 134 | } 135 | 136 | /** 137 | * get bias gradient 138 | * 139 | * @param errors errors of this layer 140 | */ 141 | private def getBiasGradient(errors: Array[BDM[Double]]): Array[Double] = { 142 | val mapNum: Int = this.getOutMapNum 143 | var j: Int = 0 144 | val gradient = new Array[Double](mapNum) 145 | while (j < mapNum) { 146 | val error: BDM[Double] = errors(j) 147 | val deltaBias: Double = sum(error) 148 | gradient(j) = deltaBias 149 | j += 1 150 | } 151 | gradient 152 | } 153 | } 154 | 155 | object ConvolutionLayer{ 156 | 157 | /** 158 | * full conv 159 | * 160 | * @param matrix 161 | * @param kernel 162 | * @return 163 | */ 164 | private[mCNN] def convnFull(matrix: BDM[Double], kernel: BDM[Double]): BDM[Double] = { 165 | val m: Int = matrix.rows 166 | val n: Int = matrix.cols 167 | val km: Int = kernel.rows 168 | val kn: Int = kernel.cols 169 | val extendMatrix = new BDM[Double](m + 2 * (km - 1), n + 2 * (kn - 1)) 170 | var i = 0 171 | var j = 0 172 | while (i < m) { 173 | while (j < n) { 174 | extendMatrix(i + km - 1, j + kn - 1) = matrix(i, j) 175 | j += 1 176 | } 177 | i += 1 178 | } 179 | convnValid(extendMatrix, kernel) 180 | } 181 | 182 | /** 183 | * valid conv 184 | * 185 | * @param matrix 186 | * @param kernel 187 | * @return 188 | */ 189 | private[mCNN] def convnValid(matrix: BDM[Double], kernel: BDM[Double]): BDM[Double] = { 190 | val m: Int = matrix.rows 191 | val n: Int = matrix.cols 192 | val km: Int = kernel.rows 193 | val kn: Int = kernel.cols 194 | val kns: Int = n - kn + 1 195 | val kms: Int = m - km + 1 196 | val outMatrix: BDM[Double] = new BDM[Double](kms, kns) 197 | var i = 0 198 | while (i < kms) { 199 | var j = 0 200 | while (j < kns) { 201 | var sum = 0.0 202 | for (ki <- 0 until km) { 203 | for (kj <- 0 until kn) 204 | sum += matrix(i + ki, j + kj) * kernel(ki, kj) 205 | } 206 | outMatrix(i, j) = sum 207 | j += 1 208 | } 209 | i += 1 210 | } 211 | outMatrix 212 | } 213 | } 214 | -------------------------------------------------------------------------------- /src/main/Example.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package hhbyyh.mCNN 18 | 19 | import org.apache.log4j.{Level, Logger} 20 | import org.apache.spark.mllib.linalg.{Vector, Vectors} 21 | import org.apache.spark.{SparkConf, SparkContext} 22 | import breeze.linalg.{DenseMatrix => BDM, _} 23 | 24 | object Example { 25 | def main(args: Array[String]) { 26 | Logger.getLogger("org").setLevel(Level.WARN) 27 | Logger.getLogger("akka").setLevel(Level.WARN) 28 | val conf = new SparkConf().setMaster("local[8]").setAppName("ttt") 29 | val sc = new SparkContext(conf) 30 | val lines = sc.textFile("dataset/train.format", 8) 31 | val data = lines.map(line => line.split(",")).map(arr => arr.map(_.toDouble)) 32 | .map(arr => (arr(784), Vector2Tensor(Vectors.dense(arr.slice(0, 784))))) 33 | 34 | val topology = new CNNTopology 35 | topology.addLayer(CNNLayer.buildConvolutionLayer(1, 6, new Scale(5, 5))) 36 | topology.addLayer(CNNLayer.buildMeanPoolingLayer(new Scale(2, 2))) 37 | topology.addLayer(CNNLayer.buildConvolutionLayer(6, 12, new Scale(5, 5))) 38 | topology.addLayer(CNNLayer.buildMeanPoolingLayer(new Scale(2, 2))) 39 | topology.addLayer(CNNLayer.buildConvolutionLayer(12, 12, new Scale(4, 4))) 40 | val cnn: CNN = new CNN(topology).setMaxIterations(5).setMiniBatchSize(16) 41 | val start = System.nanoTime() 42 | cnn.trainOneByOne(data) 43 | println("Training time: " + (System.nanoTime() - start) / 1e9) 44 | 45 | val right = data.map(record =>{ 46 | val result = cnn.predict(record._2) 47 | if(result == record._1) 1 else 0 48 | }).sum() 49 | println(s"Predicting precision: $right " + right.toDouble/(data.count())) 50 | 51 | // val testData = sc.textFile("dataset/mnist/mnist_test.csv", 8) 52 | // .map(line => line.split(",")).map(arr => arr.map(_.toDouble)) 53 | // .map(arr => (arr(0), Example.Vector2Tensor(Vectors.dense(arr.slice(1, 785).map(v => if(v > 200) 1.0 else 0))))) 54 | 55 | val rightM = data.map(record =>{ 56 | val result = cnn.predict(record._2) 57 | if(result == record._1) 1 else 0 58 | }).sum() 59 | println(s"Mnist Full Predicting precision: $rightM " + rightM.toDouble/(data.count())) 60 | } 61 | 62 | /** 63 | * set inlayer output 64 | * @param record 65 | */ 66 | def Vector2Tensor(record: Vector): Array[BDM[Double]] = { 67 | val mapSize = new Scale(28, 28) 68 | val m = new BDM[Double](mapSize.x, mapSize.y) 69 | var i: Int = 0 70 | while (i < mapSize.x) { 71 | var j: Int = 0 72 | while (j < mapSize.y) { 73 | m(i, j) = record(mapSize.x * i + j) 74 | j += 1 75 | } 76 | i += 1 77 | } 78 | Array(m) 79 | } 80 | 81 | 82 | } 83 | 84 | 85 | -------------------------------------------------------------------------------- /src/main/MaxPoolingLayer.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package hhbyyh.mCNN 18 | 19 | import breeze.linalg.{DenseMatrix => BDM, kron} 20 | 21 | class MaxPoolingLayer private[mCNN](scaleSize: Scale) extends CNNLayer{ 22 | 23 | def getScaleSize: Scale = scaleSize 24 | 25 | override def forward(input: Array[BDM[Double]]): Array[BDM[Double]] = { 26 | 27 | 28 | val lastMapNum: Int = input.length 29 | val output = new Array[BDM[Double]](lastMapNum) 30 | var i: Int = 0 31 | while (i < lastMapNum) { 32 | val lastMap: BDM[Double] = input(i) 33 | val scaleSize: Scale = this.getScaleSize 34 | output(i) = maxPooling(lastMap, scaleSize) // MeanPoolingLayer.scaleMatrix(lastMap, scaleSize) 35 | i += 1 36 | } 37 | output 38 | } 39 | 40 | override def prevDelta(nextDelta: Array[BDM[Double]], 41 | layerInput: Array[BDM[Double]]): Array[BDM[Double]] = { 42 | val mapNum: Int = layerInput.length 43 | val errors = new Array[BDM[Double]](mapNum) 44 | var m: Int = 0 45 | val scale: Scale = this.getScaleSize 46 | while (m < mapNum) { 47 | val nextError: BDM[Double] = nextDelta(m) 48 | val map: BDM[Double] = layerInput(m) 49 | var outMatrix: BDM[Double] = (1.0 - map) 50 | outMatrix = map :* outMatrix 51 | val backMaxMatrix = backFilling(map, nextError) 52 | outMatrix = outMatrix :* backMaxMatrix 53 | errors(m) = outMatrix 54 | m += 1 55 | } 56 | errors 57 | } 58 | 59 | private[mCNN] def maxPooling(matrix: BDM[Double], scale: Scale): BDM[Double] = { 60 | val m: Int = matrix.rows 61 | val n: Int = matrix.cols 62 | val scaleX = scale.x 63 | val scaleY = scale.y 64 | val sm: Int = m / scaleX 65 | val sn: Int = n / scaleY 66 | val outMatrix = new BDM[Double](sm, sn) 67 | 68 | var i = 0 // iterate through blocks 69 | while (i < sm) { 70 | var j = 0 71 | while (j < sn) { 72 | var max = matrix(i * scaleX, j * scaleY) // initial to left up corner of the block 73 | var bi = i * scaleX // block i 74 | val biMax = (i + 1) * scaleX 75 | val bjMax = (j + 1) * scaleY 76 | while (bi < biMax) { 77 | var bj = j * scaleY // block j 78 | while (bj < bjMax) { 79 | max = if(matrix(bi, bj) > max) matrix(bi, bj) else max 80 | bj += 1 81 | } 82 | bi += 1 83 | } 84 | outMatrix(i, j) = max 85 | j += 1 86 | } 87 | i += 1 88 | } 89 | outMatrix 90 | } 91 | 92 | private[mCNN] def backFilling(inputMap: BDM[Double], nextError: BDM[Double]): BDM[Double] ={ 93 | val scale = this.getScaleSize 94 | val backMaxMatrix = BDM.zeros[Double](inputMap.rows, inputMap.cols) 95 | val scaleX = scale.x 96 | val scaleY = scale.y 97 | val sm: Int = inputMap.rows / scaleX 98 | val sn: Int = inputMap.cols / scaleY 99 | var i = 0 // iterate through blocks 100 | while (i < sm) { 101 | var j = 0 102 | while (j < sn) { 103 | var max = inputMap(i * scaleX, j * scaleY) // initial to left up corner of the block 104 | var maxi = i * scaleX 105 | var maxj = j * scaleY 106 | var bi = i * scaleX // block i 107 | val biMax = (i + 1) * scaleX 108 | val bjMax = (j + 1) * scaleY 109 | while (bi < biMax) { 110 | var bj = j * scaleY // block j 111 | while (bj < bjMax) { 112 | if(inputMap(bi, bj) > max) { 113 | max = inputMap(bi, bj) 114 | maxi = bi 115 | maxj = bj 116 | } 117 | bj += 1 118 | } 119 | bi += 1 120 | } 121 | backMaxMatrix(maxi, maxj) = nextError(i, j) * scaleX * scaleY 122 | j += 1 123 | } 124 | i += 1 125 | } 126 | backMaxMatrix 127 | } 128 | 129 | } 130 | -------------------------------------------------------------------------------- /src/main/MeanPoolingLayer.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package hhbyyh.mCNN 18 | 19 | import breeze.linalg.{DenseMatrix => BDM, kron} 20 | 21 | class MeanPoolingLayer private[mCNN](scaleSize: Scale) 22 | extends CNNLayer{ 23 | 24 | def getScaleSize: Scale = scaleSize 25 | 26 | override def forward(input: Array[BDM[Double]]): Array[BDM[Double]] = { 27 | val lastMapNum: Int = input.length 28 | val output = new Array[BDM[Double]](lastMapNum) 29 | var i: Int = 0 30 | while (i < lastMapNum) { 31 | val lastMap: BDM[Double] = input(i) 32 | val scaleSize: Scale = this.getScaleSize 33 | output(i) = MeanPoolingLayer.scaleMatrix(lastMap, scaleSize) 34 | i += 1 35 | } 36 | output 37 | } 38 | 39 | override def prevDelta(nextDelta: Array[BDM[Double]], 40 | layerInput: Array[BDM[Double]]): Array[BDM[Double]] = { 41 | val mapNum: Int = layerInput.length 42 | val errors = new Array[BDM[Double]](mapNum) 43 | var m: Int = 0 44 | val scale: Scale = this.getScaleSize 45 | while (m < mapNum) { 46 | val nextError: BDM[Double] = nextDelta(m) 47 | val map: BDM[Double] = layerInput(m) 48 | var outMatrix: BDM[Double] = (1.0 - map) 49 | outMatrix = map :* outMatrix 50 | outMatrix = outMatrix :* MeanPoolingLayer.kronecker(nextError, scale) 51 | errors(m) = outMatrix 52 | m += 1 53 | } 54 | errors 55 | } 56 | 57 | } 58 | 59 | object MeanPoolingLayer{ 60 | 61 | private[mCNN] def kronecker(matrix: BDM[Double], scale: Scale): BDM[Double] = { 62 | val ones = BDM.ones[Double](scale.x, scale.y) 63 | kron(matrix, ones) 64 | } 65 | 66 | /** 67 | * return a new matrix that has been scaled down 68 | * 69 | * @param matrix 70 | */ 71 | private[mCNN] def scaleMatrix(matrix: BDM[Double], scale: Scale): BDM[Double] = { 72 | val m: Int = matrix.rows 73 | val n: Int = matrix.cols 74 | val sm: Int = m / scale.x 75 | val sn: Int = n / scale.y 76 | val outMatrix = new BDM[Double](sm, sn) 77 | val size = scale.x * scale.y 78 | var i = 0 79 | while (i < sm) { 80 | var j = 0 81 | while (j < sn) { 82 | var sum = 0.0 83 | var si = i * scale.x 84 | while (si < (i + 1) * scale.x) { 85 | var sj = j * scale.y 86 | while (sj < (j + 1) * scale.y) { 87 | sum += matrix(si, sj) 88 | sj += 1 89 | } 90 | si += 1 91 | } 92 | outMatrix(i, j) = sum / size 93 | j += 1 94 | } 95 | i += 1 96 | } 97 | outMatrix 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/main/Scale.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package hhbyyh.mCNN 18 | 19 | import java.io.Serializable 20 | 21 | 22 | /** 23 | * scale size for conv and sampling, can have different x and y 24 | */ 25 | class Scale(var x: Int, var y: Int) extends Serializable { 26 | 27 | /** 28 | * divide a scale with other scale 29 | * 30 | * @param scaleSize 31 | * @return 32 | */ 33 | private[mCNN] def divide(scaleSize: Scale): Scale = { 34 | val x: Int = this.x / scaleSize.x 35 | val y: Int = this.y / scaleSize.y 36 | if (x * scaleSize.x != this.x || y * scaleSize.y != this.y){ 37 | throw new RuntimeException(this + "can not be divided" + scaleSize) 38 | } 39 | new Scale(x, y) 40 | } 41 | 42 | private[mCNN] def multiply(scaleSize: Scale): Scale = { 43 | val x: Int = this.x * scaleSize.x 44 | val y: Int = this.y * scaleSize.y 45 | new Scale(x, y) 46 | } 47 | 48 | /** 49 | * subtract a scale and add append 50 | */ 51 | private[mCNN] def subtract(other: Scale, append: Int): Scale = { 52 | val x: Int = this.x - other.x + append 53 | val y: Int = this.y - other.y + append 54 | new Scale(x, y) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/printMatrix.scala: -------------------------------------------------------------------------------- 1 | package hhbyyh.mCNN 2 | 3 | import org.apache.log4j.{Level, Logger} 4 | import org.apache.spark.mllib.linalg.Vectors 5 | import org.apache.spark.{SparkContext, SparkConf} 6 | import breeze.linalg.{DenseMatrix => BDM, kron} 7 | 8 | /** 9 | * Created by yuhao on 9/22/15. 10 | */ 11 | object printMatrix { 12 | def main(args: Array[String]) { 13 | Logger.getLogger("org").setLevel(Level.WARN) 14 | Logger.getLogger("akka").setLevel(Level.WARN) 15 | val conf = new SparkConf().setMaster("local[8]").setAppName("ttt") 16 | val sc = new SparkContext(conf) 17 | val lines = sc.textFile("dataset/mnist/mnist_train.csv", 8) 18 | val data = lines.map(line => line.split(",")).map(arr => arr.map(_.toDouble)) 19 | .map(arr => (arr(0), Example.Vector2Tensor(Vectors.dense(arr.slice(1, 785).map(v => if(v > 200) 1.0 else 0)))(0))) 20 | 21 | val lines2 = sc.textFile("dataset/train.format", 8) 22 | val data2 = lines2.map(line => line.split(",")).map(arr => arr.map(_.toDouble)) 23 | .map(arr => (arr(784), Example.Vector2Tensor(Vectors.dense(arr.slice(0, 784)))(0))) 24 | 25 | data2.take(10).foreach(record =>{ 26 | println("label: " + record._1) 27 | val intm = new BDM[Int](28, 28, record._2.toArray.map(d => d.toInt)) 28 | val str = intm.toString(1000, 1000).replace('0', '.').replace('0', '*') 29 | println(str) 30 | }) 31 | 32 | } 33 | } 34 | --------------------------------------------------------------------------------