├── .gitignore ├── .gitmodules ├── .scalafmt.conf ├── .travis.yml ├── README.md ├── appveyor.yml ├── build.sbt ├── cover.jpeg ├── local.sbt ├── project ├── build.properties └── plugins.sbt └── src └── main ├── resources ├── get_datasets.sh └── logback.xml └── scala └── com └── github └── izhangzhihao ├── GettingStarted.scala ├── MiniBatchGradientDescent.scala ├── ReadCIFAR10.scala ├── ReadCIFAR10ToNDArray.scala ├── SoftmaxLinearClassifier.scala ├── TwoLayerNet.scala └── Utils.scala /.gitignore: -------------------------------------------------------------------------------- 1 | project/target 2 | target 3 | notebook/.ipynb_checkpoints 4 | *.html 5 | src/main/resources/cifar-10-batches-bin 6 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "notebooks"] 2 | path = notebooks 3 | url = git://github.com/thoughtworksinc/deeplearning.scala-website.git 4 | -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | spaces.neverAroundInfixTypes = ["##"] 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | matrix: 2 | include: 3 | - os: linux 4 | jdk: oraclejdk8 5 | sudo: false 6 | language: scala 7 | addons: 8 | apt: 9 | packages: 10 | - graphviz 11 | - os: osx 12 | osx_image: xcode8 13 | language: java 14 | before_install: 15 | - brew update 16 | - brew install sbt 17 | 18 | before_cache: 19 | - find $HOME/.sbt -name '*.lock' -delete 20 | - find $HOME/.ivy2 -name 'ivydata-*.properties' -delete 21 | 22 | cache: 23 | directories: 24 | - $HOME/.ivy2/cache 25 | - $HOME/.sbt/boot/ 26 | 27 | script: 28 | - sbt compile 29 | - sbt "runMain com.github.izhangzhihao.GettingStarted" 30 | - sbt "runMain com.github.izhangzhihao.SoftmaxLinearClassifier" 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Tutorial of DeepLearning.scala [![Build Status](https://travis-ci.org/izhangzhihao/deeplearning-tutorial.svg?branch=master)](https://travis-ci.org/izhangzhihao/deeplearning-tutorial) [![Build status](https://ci.appveyor.com/api/projects/status/fvxjskxa9oqwqpel/branch/master?svg=true)](https://ci.appveyor.com/project/izhangzhihao/deeplearning-tutorial/branch/master) [![Codacy Badge](https://api.codacy.com/project/badge/Grade/09b2f5a4d06547a38ce425e7c7fae869)](https://www.codacy.com/app/izhangzhihao/deeplearning-tutorial?utm_source=github.com&utm_medium=referral&utm_content=izhangzhihao/deeplearning-tutorial&utm_campaign=badger) 2 | 3 | **DeepLearning.scala** is a DSL for creating complex neural networks. 4 | 5 | With the help of DeepLearning.scala, regular programmers are able to build complex neural networks from simple code. You write code almost as usual, the only difference being that code based on DeepLearning.scala is [differentiable](https://colah.github.io/posts/2015-09-NN-Types-FP/), which enables such code to evolve by modifying its parameters continuously. 6 | 7 |

8 | 9 | ## NOTE: Please do NOT download this repo,because if you download this repo ,you will find the notebooks dir is empty. So ,please use git clone ,after that , use `git submodule init` & `git submodule update` to clone the notebooks(git submodule). 10 | 11 | ### [Jupyter notebook installation](https://github.com/alexarchambault/jupyter-scala) 12 | 13 | ## Tutorial index 14 | 15 | ### 1 - Getting Start 16 | - [getting start](https://github.com/thoughtworksinc/deeplearning.scala-website/blob/v1.0.0-doc/ipynbs/GettingStarted.ipynb) 17 | - [how to debug](https://github.com/thoughtworksinc/deeplearning.scala-website/blob/v1.0.0-doc/ipynbs/Debug.ipynb) 18 | - [softmax](https://github.com/thoughtworksinc/deeplearning.scala-website/blob/v1.0.0-doc/ipynbs/SoftmaxLinearClassifier.ipynb) 19 | - [mini-batch gradient descent](https://github.com/thoughtworksinc/deeplearning.scala-website/blob/v1.0.0-doc/ipynbs/MiniBatchGradientDescent.ipynb) 20 | - [two layer net](https://github.com/thoughtworksinc/deeplearning.scala-website/blob/v1.0.0-doc/ipynbs/TwoLayerNet.ipynb) 21 | - [CNNs](https://github.com/thoughtworksinc/deeplearning.scala-website/blob/v1.0.0-doc/ipynbs/CNNs.ipynb) 22 | 23 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | environment: 2 | global: 3 | COMPILE: compile 4 | TEST_COMPILE: test:compile 5 | IT_TEST_COMPILE: it:compile 6 | TEST_SUITE: test 7 | IT_TEST_SUITE: "it:testOnly -- -l NoWindows" 8 | matrix: 9 | # - SCALA_VERSION: 2.10.6 10 | - SCALA_VERSION: 2.11.11 11 | # - SCALA_VERSION: 2.12.2 12 | install: 13 | - ps: | 14 | Add-Type -AssemblyName System.IO.Compression.FileSystem 15 | if (!(Test-Path -Path "C:\sbt\sbt-launcher-packaging-0.13.13" )) { 16 | (new-object System.Net.WebClient).DownloadFile( 17 | 'https://dl.bintray.com/sbt/native-packages/sbt/0.13.13/sbt-0.13.13.zip', 18 | 'C:\sbt-0.13.13.zip' 19 | ) 20 | [System.IO.Compression.ZipFile]::ExtractToDirectory("C:\sbt-0.13.13.zip", "C:\sbt") 21 | } 22 | - ps: | 23 | if (!(Test-Path -Path "C:\Program Files\Java\jdk1.8.0\src.zip")) { 24 | (new-object System.Net.WebClient).DownloadFile( 25 | 'https://github.com/fommil/openjdk8src/raw/master/src.zip', 26 | 'C:\Program Files\Java\jdk1.8.0\src.zip' 27 | ) 28 | } 29 | - cmd: SET JAVA_HOME=C:\Program Files\Java\jdk1.8.0 30 | - cmd: SET PATH=C:\sbt\sbt-launcher-packaging-0.13.13\bin;%JAVA_HOME%\bin;%PATH% 31 | - cmd: SET COURSIER_VERBOSITY=-1 32 | - cmd: SET SBT_OPTS=-Xss2m -Xms1024m -Xmx4096m -XX:+TieredCompilation -XX:+CMSPermGenSweepingEnabled -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:MaxPermSize=256m -XX:ReservedCodeCacheSize=256M 33 | - cmd: SET SCALATEST_SPAN_SCALE_FACTOR=20 34 | - cmd: SET SBT_TASK_LIMIT=2 35 | - cmd: SET SCALATEST_PARALLEL_TESTS=2 36 | - cmd: SET SCALATEST_SLEEP_SCALE_FACTOR=2.0 37 | build_script: 38 | - sbt ";++ %SCALA_VERSION% %COMPILE%" 39 | # - sbt ";+++ %SCALA_VERSION% %COMPILE% ;+++ %SCALA_VERSION% %TEST_COMPILE% ;+++ %SCALA_VERSION% %IT_TEST_COMPILE%" 40 | # test_script: 41 | # - sbt ";+++ %SCALA_VERSION% %TEST_SUITE% ;+++ %SCALA_VERSION% %IT_TEST_SUITE%" 42 | cache: 43 | - C:\Program Files\Java\jdk1.8.0\src.zip 44 | - C:\sbt\sbt-launcher-packaging-0.13.13 45 | - C:\Users\appveyor\.ivy2 -> appveyor.yml 46 | - C:\Users\appveyor\.coursier -> appveyor.yml 47 | on_finish: 48 | - ps: Get-ChildItem -Path .\* -Include *.log,*.hprof -Recurse | % { Push-AppveyorArtifact $_.FullName -FileName $_.Name } 49 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | name := "deeplearning-tutorial" 2 | 3 | version := "2.0" 4 | 5 | scalaVersion in Global := "2.11.11" 6 | 7 | val deepLearningScalaVersion = "2.0.0-M2" 8 | 9 | libraryDependencies += "com.thoughtworks.deeplearning" %% "differentiable" % deepLearningScalaVersion 10 | 11 | libraryDependencies += "org.plotly-scala" %% "plotly-render" % "0.3.2" 12 | 13 | libraryDependencies += "org.nd4j" % "nd4j-native-platform" % "0.7.2" 14 | 15 | libraryDependencies += "org.nd4j" % "nd4j-cuda-8.0-platform" % "0.7.2" 16 | 17 | libraryDependencies += "org.rauschig" % "jarchivelib" % "0.5.0" 18 | 19 | libraryDependencies += "org.slf4j" % "jul-to-slf4j" % "1.7.25" 20 | 21 | libraryDependencies += "org.slf4j" % "slf4j-api" % "1.7.25" 22 | 23 | libraryDependencies += "ch.qos.logback" % "logback-classic" % "1.2.2" 24 | 25 | libraryDependencies += "ch.qos.logback" % "logback-core" % "1.2.2" 26 | 27 | addCompilerPlugin( 28 | "org.scalamacros" % "paradise" % "2.1.0" cross CrossVersion.full) 29 | 30 | fork := true 31 | -------------------------------------------------------------------------------- /cover.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/izhangzhihao/deeplearning-tutorial/bbce4e1aeeada61f916ce76248caecdbffe9a910/cover.jpeg -------------------------------------------------------------------------------- /local.sbt: -------------------------------------------------------------------------------- 1 | scalaVersion in Global := "2.11.11" 2 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 0.13.15 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | logLevel := Level.Warn -------------------------------------------------------------------------------- /src/main/resources/get_datasets.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Get CIFAR10 3 | wget http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz 4 | tar -xzvf cifar-10-binary.tar.gz 5 | rm cifar-10-binary.tar.gz 6 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /src/main/scala/com/github/izhangzhihao/GettingStarted.scala: -------------------------------------------------------------------------------- 1 | package com.github.izhangzhihao 2 | 3 | import com.thoughtworks.deeplearning.math._ 4 | import com.thoughtworks.deeplearning.differentiable.Any._ 5 | import com.thoughtworks.deeplearning.differentiable.INDArray.{ 6 | Optimizer => INDArrayOptimizer 7 | } 8 | import INDArrayOptimizer.LearningRate 9 | import com.thoughtworks.deeplearning.differentiable.INDArray.implicits._ 10 | import com.thoughtworks.each.Monadic._ 11 | import com.thoughtworks.raii.asynchronous.Do 12 | import com.thoughtworks.deeplearning.differentiable.Double._ 13 | import com.thoughtworks.deeplearning.differentiable.Double.implicits._ 14 | import com.thoughtworks.deeplearning.{Tape, differentiable} 15 | import org.nd4j.linalg.api.ndarray.INDArray 16 | import org.nd4j.linalg.factory.Nd4j 17 | import org.nd4s.Implicits._ 18 | 19 | import scala.concurrent.ExecutionContext.Implicits.global 20 | import scalaz.concurrent.Task 21 | import scalaz.{-\/, \/, \/-} 22 | import scalaz.std.vector._ 23 | 24 | import plotly.Scatter 25 | import plotly.Plotly._ 26 | 27 | object GettingStarted extends App { 28 | 29 | implicit def optimizer: INDArrayOptimizer = new LearningRate { 30 | def currentLearningRate() = 0.001 31 | } 32 | 33 | val weight = (Nd4j.randn(3, 1) / scala.math.sqrt(3.0)).toWeight 34 | 35 | def myNeuralNetwork(input: INDArray): differentiable.INDArray = { 36 | dot(input, weight) 37 | } 38 | 39 | def lossFunction(input: INDArray, 40 | expectOutput: INDArray): differentiable.Double = { 41 | sumT(abs(myNeuralNetwork(input) - expectOutput)) 42 | } 43 | 44 | val input: INDArray = 45 | Array(Array(0, 1, 2), Array(3, 6, 9), Array(13, 15, 17)).toNDArray 46 | 47 | val expectedOutput: INDArray = Array(Array(1), Array(3), Array(2)).toNDArray 48 | 49 | @monadic[Task] 50 | val trainTask: Task[Unit] = { 51 | 52 | val lossSeq = for (_ <- (1 to 400).toVector) yield { 53 | train(lossFunction(input, expectedOutput)).each 54 | } 55 | 56 | polyLoss(lossSeq) 57 | 58 | } 59 | 60 | def polyLoss(lossSeq: IndexedSeq[Double]) = { 61 | val plot = Seq( 62 | Scatter(lossSeq.indices, lossSeq) 63 | ) 64 | 65 | plot.plot( 66 | title = "loss by time" 67 | ) 68 | } 69 | 70 | val predictResult = throwableMonadic[Task] { 71 | trainTask.each 72 | predict(myNeuralNetwork(input)).each 73 | } 74 | 75 | predictResult.unsafePerformSyncAttempt match { 76 | case -\/(e) => throw e 77 | case \/-(result) => 78 | println(result) 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /src/main/scala/com/github/izhangzhihao/MiniBatchGradientDescent.scala: -------------------------------------------------------------------------------- 1 | package com.github.izhangzhihao 2 | 3 | import com.thoughtworks.deeplearning.math._ 4 | import com.thoughtworks.deeplearning.differentiable.Any._ 5 | import com.thoughtworks.deeplearning.differentiable.INDArray.{ 6 | Optimizer => INDArrayOptimizer 7 | } 8 | import INDArrayOptimizer.LearningRate 9 | import com.github.izhangzhihao.GettingStarted.polyLoss 10 | import com.thoughtworks.deeplearning.differentiable.INDArray.implicits._ 11 | import com.thoughtworks.each.Monadic._ 12 | import com.thoughtworks.raii.asynchronous.Do 13 | import com.thoughtworks.deeplearning.differentiable.Double._ 14 | import com.thoughtworks.deeplearning.differentiable.Double.implicits._ 15 | import com.thoughtworks.deeplearning.{Tape, differentiable} 16 | import org.nd4j.linalg.api.ndarray.INDArray 17 | import org.nd4j.linalg.factory.Nd4j 18 | import org.nd4s.Implicits._ 19 | 20 | import scala.concurrent.ExecutionContext.Implicits.global 21 | import scalaz.concurrent.Task 22 | import scalaz.{-\/, \/, \/-} 23 | import scalaz.std.vector._ 24 | import scalaz.std.list._ 25 | import scalaz.std.map._ 26 | import scalaz.std.iterable._ 27 | import plotly.Scatter 28 | import plotly.Plotly._ 29 | import shapeless._ 30 | 31 | import scala.util.Random 32 | 33 | /** 34 | * @author 张志豪 (izhangzhihao) <izhangzhihao@hotmail.com> 35 | */ 36 | object MiniBatchGradientDescent extends App { 37 | //CIFAR10中的图片共有10个分类(airplane,automobile,bird,cat,deer,dog,frog,horse,ship,truck) 38 | val NumberOfClasses: Int = 10 39 | val NumberOfPixels: Int = 3072 40 | 41 | //加载测试数据，我们读取100条作为测试数据 42 | val testNDArray = 43 | ReadCIFAR10ToNDArray.readFromResource( 44 | "/cifar-10-batches-bin/test_batch.bin", 45 | 100) 46 | 47 | val testData = testNDArray.head 48 | val testExpectResult = testNDArray.tail.head 49 | 50 | val vectorizedTestExpectResult = 51 | Utils.makeVectorized(testExpectResult, NumberOfClasses) 52 | 53 | def softmax(scores: differentiable.INDArray): differentiable.INDArray = { 54 | val expScores = exp(scores) 55 | expScores / sum(expScores, 1) 56 | } 57 | 58 | implicit def optimizer: INDArrayOptimizer = new LearningRate { 59 | def currentLearningRate() = 0.00001 60 | } 61 | 62 | val weight: differentiable.INDArray = 63 | (Nd4j.randn(NumberOfPixels, NumberOfClasses) * 0.001).toWeight 64 | 65 | def myNeuralNetwork(input: INDArray): differentiable.INDArray = { 66 | softmax(dot(input, weight)) 67 | } 68 | 69 | def lossFunction(input: INDArray, 70 | expectOutput: INDArray): differentiable.Double = { 71 | val probabilities = myNeuralNetwork(input) 72 | -mean(log(probabilities) * expectOutput) 73 | } 74 | 75 | @monadic[Task] 76 | val trainTask: Task[Unit] = { 77 | val random = new Random 78 | 79 | val MiniBatchSize = 256 80 | 81 | val lossSeq = 82 | ( 83 | for (_ <- (0 to 50).toVector) yield { 84 | val randomIndex = random 85 | .shuffle[Int, IndexedSeq](0 until 10000) //https://issues.scala-lang.org/browse/SI-6948 86 | .toArray 87 | for (times <- (0 until 10000 / MiniBatchSize).toVector) yield { 88 | val randomIndexArray = 89 | randomIndex.slice(times * MiniBatchSize, 90 | (times + 1) * MiniBatchSize) 91 | val trainNDArray :: expectLabel :: shapeless.HNil = 92 | ReadCIFAR10ToNDArray.getSGDTrainNDArray(randomIndexArray) 93 | val input = 94 | trainNDArray.reshape(MiniBatchSize, 3072) 95 | 96 | val expectLabelVectorized = 97 | Utils.makeVectorized(expectLabel, NumberOfClasses) 98 | val loss = train(lossFunction(input, expectLabelVectorized)).each 99 | println(loss) 100 | loss 101 | } 102 | } 103 | ).flatten 104 | 105 | polyLoss(lossSeq) 106 | } 107 | 108 | val predictResult = throwableMonadic[Task] { 109 | trainTask.each 110 | predict(myNeuralNetwork(testData)).each 111 | } 112 | 113 | predictResult.unsafePerformSyncAttempt match { 114 | case -\/(e) => throw e 115 | case \/-(result) => 116 | println( 117 | "The accuracy is " + Utils.getAccuracy(result, testExpectResult) + "%") 118 | } 119 | 120 | } 121 | -------------------------------------------------------------------------------- /src/main/scala/com/github/izhangzhihao/ReadCIFAR10.scala: -------------------------------------------------------------------------------- 1 | package com.github.izhangzhihao 2 | 3 | import java.awt.Color 4 | import java.awt.image.BufferedImage 5 | import java.io.FileOutputStream 6 | import javax.imageio.ImageIO 7 | 8 | /** 9 | * Created by 张志豪 on 1/16/17. 10 | * 显示一个CIFAR10中的图片 11 | */ 12 | object ReadCIFAR10 extends App { 13 | private val inputStream = 14 | getClass.getResourceAsStream("/cifar-10-batches-bin/data_batch_1.bin") 15 | private val bytes = Array.range(0, 3073).map(_.toByte) 16 | inputStream.read(bytes) 17 | 18 | private val bufferedImage = 19 | new BufferedImage(32, 32, BufferedImage.TYPE_INT_RGB) 20 | 21 | for (row <- 0 until 32) { 22 | for (col <- 0 until 32) { 23 | val color = new Color(bytes(1 + 1024 * 0 + row * 32 + col) & 0xFF, 24 | bytes(1 + 1024 * 1 + row * 32 + col) & 0xFF, 25 | bytes(1 + 1024 * 2 + row * 32 + col) & 0xFF) 26 | bufferedImage.setRGB(col, row, color.getRGB) 27 | } 28 | } 29 | 30 | ImageIO.write(bufferedImage, "jpeg", new FileOutputStream("./out.jpg")) 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/scala/com/github/izhangzhihao/ReadCIFAR10ToNDArray.scala: -------------------------------------------------------------------------------- 1 | package com.github.izhangzhihao 2 | 3 | import java.io._ 4 | import java.net.URL 5 | import java.util.zip.GZIPInputStream 6 | 7 | import sys.process._ 8 | import org.nd4j.linalg.api.ndarray.INDArray 9 | import org.nd4j.linalg.factory.Nd4j 10 | import org.nd4s.Implicits._ 11 | import org.rauschig.jarchivelib.{Archiver, ArchiverFactory} 12 | import shapeless._ 13 | 14 | import scala.collection.immutable.IndexedSeq 15 | 16 | /** 17 | * Created by 张志豪 on 1/16/17. 18 | */ 19 | object ReadCIFAR10ToNDArray { 20 | 21 | /** 22 | * 原始文件字节 23 | */ 24 | lazy val originalFileBytesArray: Array[Array[Byte]] = { 25 | for (fileIndex <- 1 to 5) yield { 26 | 27 | val currentPath = new java.io.File(".").getCanonicalPath + "/src/main/resources/" 28 | 29 | val fileName = currentPath + "/cifar-10-batches-bin" + "/data_batch_" + fileIndex + ".bin" 30 | 31 | if (!new File(fileName).exists()) { 32 | downloadDataAndUnzipIfNotExist( 33 | "http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz", 34 | currentPath, 35 | "cifar-10-binary.tar.gz" 36 | ) 37 | } 38 | val inputStream = new FileInputStream(fileName) 39 | readFromInputStream(inputStream) 40 | } 41 | }.toArray 42 | 43 | /** 44 | * 中心化过后的图片 45 | */ 46 | lazy val pixelBytesArray: Array[Array[Array[Double]]] = { 47 | for (fileIndex <- 0 until 5) yield { 48 | val originalFileBytes = originalFileBytesArray(fileIndex) 49 | for (index <- 0 until 10000) yield { 50 | val beginIndex = index * 3073 + 1 51 | normalizePixel(originalFileBytes.slice(beginIndex, beginIndex + 3072)) 52 | } 53 | }.toArray 54 | }.toArray 55 | 56 | /** 57 | * 图片对应的label 58 | */ 59 | lazy val labelBytesArray: Array[Array[Int]] = { 60 | for (fileIndex <- 0 until 5) yield { 61 | val originalFileBytes = originalFileBytesArray(fileIndex) 62 | for (index <- 0 until 10000) yield { 63 | val beginIndex = index * 3073 64 | originalFileBytes(beginIndex).toInt 65 | } 66 | }.toArray 67 | }.toArray 68 | 69 | val random = new util.Random 70 | 71 | /** 72 | * 从inputStream中读取byte 73 | * 74 | * @param inputStream 75 | * @return 76 | */ 77 | def readFromInputStream(inputStream: InputStream): Array[Byte] = { 78 | try { 79 | val bytes = Array.range(0, 3073 * 10000).map(_.toByte) 80 | inputStream.read(bytes) 81 | bytes 82 | } finally { 83 | inputStream.close() 84 | } 85 | } 86 | 87 | /** 88 | * 数据集不存在的话下载并解压 89 | * 90 | * @param path 91 | * @param url 92 | * @param fileName 93 | */ 94 | //noinspection ScalaUnusedSymbol 95 | def downloadDataAndUnzipIfNotExist(url: String, 96 | path: String, 97 | fileName: String): Unit = { 98 | println("downloading CIFAR10 database...") 99 | val result = new URL(url) #> new File(path + fileName) !! 100 | 101 | println("unzip CIFAR10 database...") 102 | val archiver: Archiver = ArchiverFactory.createArchiver("tar", "gz") 103 | archiver.extract(new File(path + fileName), new File(path)) 104 | println("download and unzip done.") 105 | } 106 | 107 | /** 108 | * 从CIFAR10文件中读图片和其对应的标签 109 | * 110 | * @param fileName CIFAR10文件名 111 | * @param count 要读取多少个图片和其标签 112 | * @return input :: expectedOutput :: HNil 113 | */ 114 | def readFromResource(fileName: String, 115 | count: Int): INDArray :: INDArray :: HNil = { 116 | 117 | val currentPath = new java.io.File(".").getCanonicalPath + "/src/main/resources/" 118 | 119 | val filePathName = currentPath + fileName 120 | 121 | if (!new File(filePathName).exists()) { 122 | downloadDataAndUnzipIfNotExist( 123 | "http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz", 124 | currentPath, 125 | "cifar-10-binary.tar.gz" 126 | ) 127 | } 128 | 129 | val inputStream = new FileInputStream(filePathName) 130 | try { 131 | val bytes = Array.range(0, 3073 * count).map(_.toByte) 132 | inputStream.read(bytes) 133 | 134 | val labels: Seq[Double] = 135 | for (index <- 0 until count) yield { 136 | bytes(index * 3073).toDouble 137 | } 138 | 139 | val pixels: Seq[Seq[Double]] = 140 | for (index <- 0 until count) yield { 141 | for (item <- 1 until 3073) yield { 142 | normalizePixel(bytes(index * 3073 + item).toDouble) 143 | } 144 | } 145 | 146 | val labelsArray = labels.toNDArray.reshape(count, 1) 147 | val pixelsArray = pixels.toNDArray 148 | 149 | pixelsArray :: labelsArray :: HNil 150 | } finally { 151 | inputStream.close() 152 | } 153 | } 154 | 155 | /** 156 | * 归一化pixel数据 157 | * 158 | * @param pixel 159 | * @return 160 | */ 161 | def normalizePixel(pixel: Double): Double = { 162 | (if (pixel < 0) { 163 | pixel + 256 164 | } else { 165 | pixel 166 | }) / 256 167 | } 168 | 169 | /** 170 | * 归一化数组的pixel数据 171 | * 172 | * @param original 173 | * @return 174 | */ 175 | def normalizePixel(original: Array[Byte]): Array[Double] = { 176 | for (pixel <- original) yield { 177 | normalizePixel(pixel) 178 | } 179 | } 180 | 181 | /** 182 | * 随机获取count个train数据 183 | * 184 | * @return 185 | */ 186 | def getSGDTrainNDArray( 187 | randomIndexArray: Array[Int]): INDArray :: INDArray :: HNil = { 188 | //生成0到4的随机数 189 | val randomIndex = random.nextInt(5) 190 | val labelBytes = labelBytesArray(randomIndex) 191 | val pixelBytes = pixelBytesArray(randomIndex) 192 | val count = randomIndexArray.length 193 | 194 | val labels: Seq[Int] = 195 | for (index <- 0 until count) yield { 196 | labelBytes(randomIndexArray(index)) 197 | } 198 | 199 | val pixels: Seq[Seq[Double]] = 200 | for (index <- 0 until count) yield { 201 | pixelBytes(randomIndexArray(index)).toList 202 | } 203 | 204 | val labelsNDArray = labels.toNDArray.reshape(count, 1) 205 | val pixelsNDArray = pixels.toNDArray 206 | 207 | pixelsNDArray :: labelsNDArray :: HNil 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /src/main/scala/com/github/izhangzhihao/SoftmaxLinearClassifier.scala: -------------------------------------------------------------------------------- 1 | package com.github.izhangzhihao 2 | 3 | import com.thoughtworks.deeplearning.math._ 4 | import com.thoughtworks.deeplearning.differentiable.Any._ 5 | import com.thoughtworks.deeplearning.differentiable.INDArray.{ 6 | Optimizer => INDArrayOptimizer 7 | } 8 | import INDArrayOptimizer.LearningRate 9 | import com.github.izhangzhihao.GettingStarted.polyLoss 10 | import com.thoughtworks.deeplearning.differentiable.INDArray.implicits._ 11 | import com.thoughtworks.each.Monadic._ 12 | import com.thoughtworks.raii.asynchronous.Do 13 | import com.thoughtworks.deeplearning.differentiable.Double._ 14 | import com.thoughtworks.deeplearning.differentiable.Double.implicits._ 15 | import com.thoughtworks.deeplearning.{Tape, differentiable} 16 | import org.nd4j.linalg.api.ndarray.INDArray 17 | import org.nd4j.linalg.factory.Nd4j 18 | import org.nd4s.Implicits._ 19 | 20 | import scala.concurrent.ExecutionContext.Implicits.global 21 | import scalaz.concurrent.Task 22 | import scalaz.{-\/, \/, \/-} 23 | import scalaz.std.vector._ 24 | import plotly.Scatter 25 | import plotly.Plotly._ 26 | 27 | /** 28 | * @author 张志豪 (izhangzhihao) <izhangzhihao@hotmail.com> 29 | */ 30 | object SoftmaxLinearClassifier extends App { 31 | 32 | //CIFAR10中的图片共有10个分类(airplane,automobile,bird,cat,deer,dog,frog,horse,ship,truck) 33 | val NumberOfClasses: Int = 10 34 | val NumberOfPixels: Int = 3072 35 | 36 | //加载train数据,我们读取1000条数据作为训练数据 37 | val trainNDArray = 38 | ReadCIFAR10ToNDArray.readFromResource( 39 | "/cifar-10-batches-bin/data_batch_1.bin", 40 | 1000) 41 | 42 | //加载测试数据，我们读取100条作为测试数据 43 | val testNDArray = 44 | ReadCIFAR10ToNDArray.readFromResource( 45 | "/cifar-10-batches-bin/test_batch.bin", 46 | 100) 47 | 48 | val trainData = trainNDArray.head 49 | val testData = testNDArray.head 50 | 51 | val trainExpectResult = trainNDArray.tail.head 52 | val testExpectResult = testNDArray.tail.head 53 | 54 | val vectorizedTrainExpectResult = 55 | Utils.makeVectorized(trainExpectResult, NumberOfClasses) 56 | val vectorizedTestExpectResult = 57 | Utils.makeVectorized(testExpectResult, NumberOfClasses) 58 | 59 | def softmax(scores: differentiable.INDArray): differentiable.INDArray = { 60 | val expScores = exp(scores) 61 | expScores / sum(expScores, 1) 62 | } 63 | 64 | implicit def optimizer: INDArrayOptimizer = new LearningRate { 65 | def currentLearningRate() = 0.00001 66 | } 67 | 68 | val weight: differentiable.INDArray = 69 | (Nd4j.randn(NumberOfPixels, NumberOfClasses) * 0.001).toWeight 70 | 71 | def myNeuralNetwork(input: INDArray): differentiable.INDArray = { 72 | softmax(dot(input, weight)) 73 | } 74 | 75 | def lossFunction(input: INDArray, 76 | expectOutput: INDArray): differentiable.Double = { 77 | val probabilities = myNeuralNetwork(input) 78 | -mean(log(probabilities) * expectOutput) 79 | } 80 | 81 | @monadic[Task] 82 | val trainTask: Task[Unit] = { 83 | 84 | val lossSeq = for (_ <- (1 to 2000).toVector) yield { 85 | val loss = 86 | train(lossFunction(trainData, vectorizedTrainExpectResult)).each 87 | println(loss) 88 | loss 89 | } 90 | 91 | polyLoss(lossSeq) 92 | } 93 | 94 | val predictResult = throwableMonadic[Task] { 95 | trainTask.each 96 | predict(myNeuralNetwork(testData)).each 97 | } 98 | 99 | predictResult.unsafePerformSyncAttempt match { 100 | case -\/(e) => throw e 101 | case \/-(result) => 102 | println( 103 | "The accuracy is " + Utils.getAccuracy(result, testExpectResult) + "%") 104 | } 105 | 106 | } 107 | -------------------------------------------------------------------------------- /src/main/scala/com/github/izhangzhihao/TwoLayerNet.scala: -------------------------------------------------------------------------------- 1 | package com.github.izhangzhihao 2 | 3 | import com.thoughtworks.deeplearning.math._ 4 | import com.thoughtworks.deeplearning.differentiable.Any._ 5 | import com.thoughtworks.deeplearning.differentiable.INDArray.{Optimizer => INDArrayOptimizer, Weight => INDArrayWeight} 6 | import INDArrayOptimizer.{L2Regularization, LearningRate} 7 | import com.github.izhangzhihao.GettingStarted.polyLoss 8 | import com.thoughtworks.deeplearning.Lift 9 | import com.thoughtworks.deeplearning.Tape 10 | import com.thoughtworks.deeplearning.differentiable.INDArray.implicits._ 11 | import com.thoughtworks.each.Monadic._ 12 | import com.thoughtworks.raii.asynchronous 13 | import com.thoughtworks.raii.asynchronous.Do 14 | import com.thoughtworks.raii.ownership.Borrowing 15 | import org.nd4j.linalg.api.ndarray.INDArray 16 | import com.thoughtworks.deeplearning.differentiable.Double.implicits._ 17 | import com.thoughtworks.deeplearning.{Tape, differentiable} 18 | import org.nd4j.linalg.api.ndarray.INDArray 19 | import org.nd4j.linalg.factory.Nd4j 20 | import org.nd4s.Implicits._ 21 | 22 | import scala.concurrent.ExecutionContext.Implicits.global 23 | import scalaz.concurrent.Task 24 | import scalaz.{-\/, \/, \/-} 25 | import scalaz.std.vector._ 26 | import scalaz.std.list._ 27 | import scalaz.std.map._ 28 | import scalaz.std.iterable._ 29 | import plotly.Scatter 30 | import plotly.Plotly._ 31 | import shapeless._ 32 | 33 | import scala.util.Random 34 | 35 | /** 36 | * @author 张志豪 (izhangzhihao) <izhangzhihao@hotmail.com> 37 | */ 38 | object TwoLayerNet extends App { 39 | //CIFAR10中的图片共有10个分类(airplane,automobile,bird,cat,deer,dog,frog,horse,ship,truck) 40 | val NumberOfClasses: Int = 10 41 | 42 | //加载测试数据，我们读取100条作为测试数据 43 | val testNDArray = 44 | ReadCIFAR10ToNDArray.readFromResource( 45 | "/cifar-10-batches-bin/test_batch.bin", 46 | 100) 47 | 48 | val testData = testNDArray.head 49 | val testExpectResult = testNDArray.tail.head 50 | 51 | val vectorizedTestExpectResult = 52 | Utils.makeVectorized(testExpectResult, NumberOfClasses) 53 | 54 | implicit val optimizerFactory = 55 | new differentiable.INDArray.OptimizerFactory { 56 | override def indarrayOptimizer( 57 | weight: INDArrayWeight): INDArrayOptimizer = { 58 | new LearningRate with L2Regularization { 59 | 60 | var learningRate = 0.01 61 | 62 | override protected def currentLearningRate(): Double = { 63 | learningRate *= 0.9995 64 | learningRate 65 | } 66 | 67 | override protected def l2Regularization: Double = 0.03 68 | } 69 | } 70 | } 71 | 72 | def fullyConnectedThenRelu(inputSize: Int, 73 | outputSize: Int, 74 | input: differentiable.INDArray) = { 75 | //TODO 76 | val weight = 77 | (Nd4j.randn(inputSize, outputSize) / math.sqrt(outputSize / 2.0)).toWeight * 0.1 78 | val b = Nd4j.zeros(outputSize).toWeight 79 | max(dot(input, weight) + b, 0.0) 80 | } 81 | 82 | def softmax(scores: differentiable.INDArray): differentiable.INDArray = { 83 | val expScores = exp(scores) 84 | expScores / sum(expScores, 1) 85 | } 86 | 87 | def fullyConnectedThenSoftmax( 88 | inputSize: Int, 89 | outputSize: Int, 90 | input: differentiable.INDArray): differentiable.INDArray = { 91 | //TODO 92 | val weight = 93 | (Nd4j.randn(inputSize, outputSize) / math.sqrt(outputSize)).toWeight 94 | val b = Nd4j.zeros(outputSize).toWeight 95 | softmax(dot(input, weight) + b) 96 | } 97 | 98 | val NumberOfPixels: Int = 3072 99 | 100 | def myNeuralNetwork( 101 | input: differentiable.INDArray): differentiable.INDArray = { 102 | 103 | val layer0 = fullyConnectedThenRelu(NumberOfPixels, 500, input) 104 | 105 | fullyConnectedThenSoftmax(500, 10, layer0) 106 | } 107 | 108 | def crossEntropy(score: differentiable.INDArray, 109 | label: differentiable.INDArray): differentiable.Double = { 110 | -mean( 111 | label * log(score * 0.9 + 0.1) + (1.0 - label) * log(1.0 - score * 0.9)) 112 | } 113 | 114 | def lossFunction(input: INDArray, 115 | expectOutput: INDArray)(implicit liftINDArray: Lift.Aux[INDArray, INDArray, INDArray]): differentiable.Double = { 116 | 117 | val score 118 | : differentiable.INDArray = myNeuralNetwork(1.0 * input) // TODO 119 | crossEntropy(score, 1.0 * expectOutput) //TODO 120 | } 121 | 122 | @monadic[Task] 123 | val trainTask: Task[Unit] = { 124 | val random = new Random 125 | 126 | val MiniBatchSize = 256 127 | 128 | val lossSeq = 129 | ( 130 | for (_ <- (0 to 50).toVector) yield { 131 | val randomIndex = random 132 | .shuffle[Int, IndexedSeq](0 until 10000) //https://issues.scala-lang.org/browse/SI-6948 133 | .toArray 134 | for (times <- (0 until 10000 / MiniBatchSize).toVector) yield { 135 | val randomIndexArray = 136 | randomIndex.slice(times * MiniBatchSize, 137 | (times + 1) * MiniBatchSize) 138 | val trainNDArray :: expectLabel :: shapeless.HNil = 139 | ReadCIFAR10ToNDArray.getSGDTrainNDArray(randomIndexArray) 140 | val input = 141 | trainNDArray.reshape(MiniBatchSize, 3072) 142 | 143 | val expectLabelVectorized = 144 | Utils.makeVectorized(expectLabel, NumberOfClasses) 145 | val loss = train(lossFunction(input, expectLabelVectorized)).each 146 | println(loss) 147 | loss 148 | } 149 | } 150 | ).flatten 151 | 152 | polyLoss(lossSeq) 153 | } 154 | 155 | val predictResult = throwableMonadic[Task] { 156 | trainTask.each 157 | 158 | predict(myNeuralNetwork(1.0 * testData)).each //TODO 159 | } 160 | 161 | predictResult.unsafePerformSyncAttempt match { 162 | case -\/(e) => throw e 163 | case \/-(result) => 164 | println(Utils.getAccuracy(result, testExpectResult)) 165 | } 166 | 167 | } 168 | -------------------------------------------------------------------------------- /src/main/scala/com/github/izhangzhihao/Utils.scala: -------------------------------------------------------------------------------- 1 | package com.github.izhangzhihao 2 | 3 | import org.nd4j.linalg.api.ndarray.INDArray 4 | import org.nd4j.linalg.factory.Nd4j 5 | import org.nd4s.Implicits._ 6 | 7 | import scala.collection.GenTraversable 8 | import scala.collection.generic.GenericTraversableTemplate 9 | import scala.reflect.ClassTag 10 | 11 | /** 12 | * Created by 张志豪 on 2017/2/22. 13 | */ 14 | object Utils { 15 | 16 | /** 17 | * 处理标签数据：将N行一列的NDArray转换为N行NumberOfClasses列的NDArray，每行对应的正确分类的值为1，其它列的值为0 18 | * 19 | * @param ndArray 标签数据 20 | * @return N行NumberOfClasses列的NDArray 21 | */ 22 | def makeVectorized(ndArray: INDArray, numberOfClasses: Int): INDArray = { 23 | val shape = ndArray.shape() 24 | 25 | val p = Nd4j.zeros(shape(0), numberOfClasses) 26 | for (i <- 0 until shape(0)) { 27 | val double = ndArray.getDouble(i, 0) 28 | val column = double.toInt 29 | p.put(i, column, 1) 30 | } 31 | p 32 | } 33 | 34 | /** 35 | * 二维INDArray中获得值最大的元素所在的列组成的INDArray 36 | * 37 | * @param iNDArray 38 | * @return 39 | */ 40 | def findMaxItemIndex(iNDArray: INDArray): INDArray = { 41 | Nd4j.argMax(iNDArray, 1) 42 | } 43 | 44 | /** 45 | * 计算准确率 46 | * @param score 预测结果 47 | * @param testExpectLabel 期望结果 48 | * @return 准确率 49 | */ 50 | def getAccuracy(score: INDArray, testExpectLabel: INDArray): Double = { 51 | val scoreIndex = findMaxItemIndex(score) 52 | if (testExpectLabel.shape().toSeq.last == 1) { //not vectorized 53 | val acc = for (row <- 0 until scoreIndex.shape()(0)) yield { 54 | if (scoreIndex.getDouble(row, 0) == 55 | testExpectLabel.getDouble(row, 0)) { 56 | 1.0 57 | } else 0.0 58 | } 59 | (acc.sum / score.shape()(0)) * 100 60 | } else if (testExpectLabel.shape().toSeq.last == 10) { //vectorized 61 | val expectResultIndex = findMaxItemIndex(testExpectLabel) 62 | val accINDArray = scoreIndex.eq(expectResultIndex) 63 | (accINDArray.sumT / score.shape()(0)) * 100 64 | } else 65 | throw new IllegalArgumentException("Unacceptable testExpectLabel") 66 | } 67 | 68 | class Unzipper[A, CC[X] <: GenTraversable[X]]( 69 | s: GenericTraversableTemplate[A, CC]) { 70 | def unzip4[A1, A2, A3, A4](implicit asQuad: A => (A1, A2, A3, A4)) 71 | : (CC[A1], CC[A2], CC[A3], CC[A4]) = { 72 | val b1 = s.genericBuilder[A1] 73 | val b2 = s.genericBuilder[A2] 74 | val b3 = s.genericBuilder[A3] 75 | val b4 = s.genericBuilder[A4] 76 | for (e <- s) { 77 | val (a, b, c, d) = asQuad(e) 78 | b1 += a 79 | b2 += b 80 | b3 += c 81 | b4 += d 82 | } 83 | (b1.result, b2.result, b3.result, b4.result) 84 | } 85 | } 86 | 87 | implicit def toUnzipper[A, CC[X] <: GenTraversable[X]]( 88 | s: GenericTraversableTemplate[A, CC]): Unzipper[A, CC] = new Unzipper(s) 89 | 90 | class Tuple2ToArray[A: ClassTag](tuple: (A, A)) { 91 | def toArray: Array[A] = { 92 | val (one: A, two: A) = tuple 93 | Array(one, two) 94 | } 95 | } 96 | 97 | implicit def tupleTwoToArray[A: ClassTag](tuple: (A, A)): Tuple2ToArray[A] = 98 | new Tuple2ToArray[A](tuple) 99 | 100 | } 101 | --------------------------------------------------------------------------------