├── .gitignore ├── .jvmopts ├── .sbtopts ├── .travis.yml ├── AUTHORS.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── akka-interface-service └── src │ ├── main │ └── scala │ │ └── glaux │ │ └── interfaces.akka │ │ └── service │ │ ├── Agency.scala │ │ ├── AgentForUser.scala │ │ ├── Protocols.scala │ │ └── main.scala │ └── test │ └── scala │ └── glaux │ └── interfaces.akka │ └── service │ ├── AgencySpec.scala │ ├── AgentSpec.scala │ └── MockPersistence.scala ├── interface-api └── src │ └── main │ └── scala │ └── glaux │ └── interfaces │ └── api │ ├── domain │ ├── models.scala │ └── package.scala │ └── persistence │ └── package.scala ├── linear-algebra └── src │ ├── main │ └── scala │ │ └── glaux │ │ └── linearalgebra │ │ ├── Dimension.scala │ │ ├── Implementation.scala │ │ ├── Tensor.scala │ │ ├── impl │ │ └── nd4j │ │ │ ├── Implicits.scala │ │ │ └── TensorImpl.scala │ │ └── package.scala │ └── test │ └── scala │ └── glaux │ └── linearalgebra │ └── TensorSpec.scala ├── neural-network └── src │ ├── main │ └── scala │ │ └── glaux │ │ └── neuralnetwork │ │ ├── Layer.scala │ │ ├── Net.scala │ │ ├── ParamGradient.scala │ │ ├── Rectangle.scala │ │ ├── layers │ │ ├── Convolution.scala │ │ ├── FullyConnected.scala │ │ ├── MovingFilter.scala │ │ ├── Pool.scala │ │ ├── Regression.scala │ │ ├── Relu.scala │ │ └── Softmax.scala │ │ ├── package.scala │ │ └── trainers │ │ ├── BatchTrainer.scala │ │ └── SGD.scala │ └── test │ ├── resources │ └── kaggle │ │ └── train.csv │ └── scala │ └── glaux │ └── neuralnetwork │ ├── integration │ └── kaggle │ │ └── OttoSpecs.scala │ ├── layers │ ├── ConvolutionSpec.scala │ ├── FullyConnectedSpec.scala │ ├── PoolSpec.scala │ ├── RegressSpec.scala │ ├── ReluSpec.scala │ └── SoftmaxSpec.scala │ └── trainers │ ├── MomentumSGDTrainerSpec.scala │ └── VanillaSGDTrainerSpec.scala ├── persistence-mongodb └── src │ ├── main │ └── scala │ │ └── glaux │ │ └── persistence │ │ └── mongodb │ │ ├── AgentSettingsRepo.scala │ │ ├── GeneralHandlers.scala │ │ ├── GlauxHandlers.scala │ │ ├── InterfaceHandlers.scala │ │ ├── MongoPersistenceImpl.scala │ │ ├── Repository.scala │ │ └── SessionRepoImpl.scala │ └── test │ └── scala │ └── glaux │ └── persistence │ └── mongodb │ ├── GeneralFormatsSpec.scala │ ├── GlauxHandlersSpec.scala │ └── SessionRepoImplIntegration.scala ├── project ├── Common.scala ├── Dependencies.scala ├── Format.scala ├── Helpers.scala ├── Projects.scala ├── Publish.scala ├── Testing.scala ├── build.properties └── plugins.sbt ├── reinforcement-learning └── src │ ├── main │ └── scala │ │ └── glaux │ │ └── reinforcement │ │ ├── DeepMindQLearner.scala │ │ ├── Policy.scala │ │ ├── QAgent.scala │ │ ├── QLearner.scala │ │ └── package.scala │ └── test │ └── scala │ └── glaux │ └── reinforcementlearning │ ├── AnnealingPolicySpec.scala │ ├── ConvolutionBasedSpec.scala │ ├── QAgentSpec.scala │ ├── SimplifiedDeepMindQLearnerSpec.scala │ └── integration │ └── SimplifiedIntegration.scala ├── statistics └── src │ └── main │ └── scala │ └── glaux │ └── statistics │ ├── Distribution.scala │ ├── impl │ └── apache │ │ └── ApacheImplementations.scala │ └── package.scala └── version.sbt /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.class 3 | *.log 4 | .idea 5 | # sbt specific 6 | .cache 7 | .history 8 | .lib/ 9 | dist/* 10 | target/ 11 | lib_managed/ 12 | src_managed/ 13 | project/boot/ 14 | project/plugins/project/ 15 | 16 | # Scala-IDE specific 17 | .scala_dependencies 18 | .worksheet 19 | -------------------------------------------------------------------------------- /.jvmopts: -------------------------------------------------------------------------------- 1 | # see https://weblogs.java.net/blog/kcpeppe/archive/2013/12/11/case-study-jvm-hotspot-flags 2 | -Dfile.encoding=UTF8 3 | -Xms1G 4 | -Xmx3G 5 | -XX:MaxMetaspaceSize=2G 6 | -XX:ReservedCodeCacheSize=250M 7 | -XX:+TieredCompilation 8 | -XX:-UseGCOverheadLimit 9 | # effectively adds GC to Perm space 10 | -XX:+CMSClassUnloadingEnabled 11 | # must be enabled for CMSClassUnloadingEnabled to work 12 | -XX:+UseConcMarkSweepGC 13 | -------------------------------------------------------------------------------- /.sbtopts: -------------------------------------------------------------------------------- 1 | -J-Xmx4G 2 | -J-XX:MaxMetaspaceSize=2G 3 | -J-XX:+CMSClassUnloadingEnabled 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | 3 | scala: 4 | - 2.11.7 5 | 6 | jdk: 7 | - oraclejdk8 8 | 9 | 10 | cache: 11 | directories: 12 | - $HOME/.ivy2/cache 13 | - $HOME/.sbt/boot/ 14 | 15 | script: 16 | - sbt ++$TRAVIS_SCALA_VERSION -jvm-opts .jvmopts clean coverage test 17 | 18 | after_success: 19 | - sbt coverageReport coverageAggregate codacyCoverage 20 | 21 | 22 | -------------------------------------------------------------------------------- /AUTHORS.md: -------------------------------------------------------------------------------- 1 | Maintainers 2 | * Kailuo Wang 3 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typeAI/glaux/28ca593c69c24aafa802b130d8ae526e2850f52e/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright (c) 2015 Kailuo Wang 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/typeAI/glaux.svg)](https://travis-ci.org/typeAI/glaux) 2 | [![Codacy Badge](https://api.codacy.com/project/badge/grade/606a8cedadf9441a8f894bc6e1bf22eb)](https://www.codacy.com/app/kailuo-wang/glaux) 3 | [![Codacy Badge](https://api.codacy.com/project/badge/coverage/606a8cedadf9441a8f894bc6e1bf22eb)](https://www.codacy.com/app/kailuo-wang/glaux) 4 | [![Stories in Ready](https://badge.waffle.io/typeAI/glaux.svg?label=ready&title=Ready)](http://waffle.io/typeAI/glaux) 5 | 6 | 7 | # Glaux - Deep reinforcement learning library in functional scala 8 | 9 | ## This library is still in an experimental phase - no release yet. 10 | 11 | 12 | Glaux is an experiment to code in functional scala some deep reinforcement learning algorithms, generally speaking, i.e. deep neural network applied in reinforcement learning. The first algorithm Glaux is set to implement is [DQN](http://www.readcube.com/articles/10.1038%2Fnature14236?shared_access_token=Lo_2hFdW4MuqEcF3CVBZm9RgN0jAjWel9jnR3ZoTv0P5kedCCNjz3FJ2FhQCgXkApOr3ZSsJAldp-tw3IWgTseRnLpAc9xQq-vTA2Z5Ji9lg16_WvCy4SaOgpK5XXA6ecqo8d8J7l4EJsdjwai53GqKt-7JuioG0r3iV67MQIro74l6IxvmcVNKBgOwiMGi8U0izJStLpmQp6Vmi_8Lw_A%3D%3D) by [DeepMind](http://deepmind.com/). 13 | 14 | 15 | Glaux is modular. As of now glaux consists of 6 modules. 16 | 17 | ### linear-algebra 18 | Linear algebra adaptors for easy exchange of underlying linear algebra library for concrete implementation. Right now there is only one implementation based on [nd4j](http://nd4j.org). 19 | 20 | ### neural-network 21 | A neural network library that is extensible with new types of layers and trainers. 22 | 23 | ### reinforcement-learning 24 | Reinformment learning using neural networks for approximate Q functions. 25 | 26 | ### interface-api 27 | The API for client usage of the reinforcement learning alorithm defined above 28 | 29 | ### akka-interface 30 | An interface application for deep reinforcement learning implemented in AKKA. 31 | 32 | ### persistence-mongodb 33 | A persistence library that can persist reinforcement learning sessions into MongoDB. 34 | 35 | Stores agent settings and data into mongodb 36 | 37 | ## To run tests 38 | For unit tests run 39 | `sbt test` 40 | 41 | For integration tests, start `mongod` and run 42 | 43 | `sbt integration:test` 44 | 45 | -------------------------------------------------------------------------------- /akka-interface-service/src/main/scala/glaux/interfaces.akka/service/Agency.scala: -------------------------------------------------------------------------------- 1 | package glaux.interfaces.akka 2 | package service 3 | 4 | import Protocols.Agency._ 5 | 6 | import akka.actor._ 7 | import glaux.interfaces.akka.service.Agency.AgentCreated 8 | import Protocols.{Confirmed, Rejected} 9 | import glaux.interfaces.api.domain._ 10 | import glaux.interfaces.api.persistence._ 11 | import glaux.reinforcementlearning.{AdvancedQAgent, QAgent, SimpleQAgent} 12 | 13 | import scala.concurrent.Future 14 | 15 | class Agency(persistenceImpl: PersistenceImpl) extends Actor with ActorLogging { 16 | import Agency.AgentActor 17 | import context.dispatcher 18 | 19 | val settingsRepo = persistenceImpl.agentSettingsPersistence 20 | 21 | def receive = running(Map.empty) 22 | 23 | def running(agents: Map[SessionId, AgentActor]): Receive = { 24 | case GetAgentForUser(sessionId) ⇒ 25 | val agentResult = agents.get(sessionId) 26 | if (agentResult.isDefined) 27 | sender ! AgentRef(agentResult.get) 28 | else { 29 | val replyTo = sender 30 | createAgentForUser(sessionId).foreach { 31 | case Some(newAgent) ⇒ self ! AgentCreated(sessionId, newAgent, replyTo) 32 | case None ⇒ replyTo ! Rejected 33 | } 34 | } 35 | 36 | case AgentCreated(sessionId, newAgent, replyTo) ⇒ 37 | val toUse = agents.getOrElse(sessionId, newAgent) //discard the newly created agent if there is already one created at this moment 38 | replyTo ! AgentRef(toUse) 39 | if (toUse == newAgent) 40 | context become running(agents + (sessionId → toUse)) 41 | 42 | case Terminated(agent) ⇒ 43 | val aidResult = agents.collectFirst { 44 | case (aid, `agent`) ⇒ aid 45 | } 46 | aidResult.foreach { agentId ⇒ 47 | context become running(agents - agentId) 48 | } 49 | 50 | case CreateAgentSettings(settings) ⇒ 51 | val replyTo = sender 52 | settingsRepo.upsert(settings).map[Any](_ ⇒ Confirmed).recover { case _ ⇒ Rejected }.foreach { replyMsg ⇒ 53 | replyTo ! replyMsg 54 | } 55 | 56 | } 57 | 58 | def createAgentForUser(sessionId: SessionId): Future[Option[ActorRef]] = { 59 | settingsRepo.get(sessionId.agentName).map(_.map { 60 | case AdvancedAgentSettings(_, na, ls, ts) ⇒ 61 | val agent: AdvancedQAgent = AdvancedQAgent(na, ls, ts) 62 | AgentForUser.props(agent, sessionId)(persistenceImpl.advanceQAgentSessionPersistence) 63 | }.map { agentProps ⇒ 64 | val agent = context.actorOf(agentProps) 65 | context watch agent 66 | agent 67 | }) 68 | } 69 | 70 | } 71 | 72 | object Agency { 73 | type AgentActor = ActorRef 74 | 75 | def props(implicit p: PersistenceImpl): Props = Props(new Agency(p)) 76 | 77 | private case class AgentCreated(sessionId: SessionId, agentActor: AgentActor, replyTo: ActorRef) 78 | } 79 | 80 | -------------------------------------------------------------------------------- /akka-interface-service/src/main/scala/glaux/interfaces.akka/service/AgentForUser.scala: -------------------------------------------------------------------------------- 1 | package glaux.interfaces.akka 2 | package service 3 | 4 | import akka.actor.{Props, Actor, ActorLogging} 5 | import Protocols.{Response, Confirmed} 6 | import glaux.interfaces.api.domain.{SessionId, ProfileId, Reading} 7 | import Protocols.Agent._ 8 | import glaux.interfaces.api.persistence.SessionPersistence 9 | import glaux.reinforcementlearning._ 10 | import scala.concurrent.Future 11 | import scala.util.{Failure, Success} 12 | 13 | class AgentForUser[AT <: QAgent: SessionPersistence](qAgent: AT, sessionId: SessionId) extends Actor with ActorLogging { 14 | 15 | import qAgent.Session 16 | 17 | private val repo = implicitly[SessionPersistence[AT]] //todo: use cached implicit from shapeless here. 18 | import context.dispatcher 19 | 20 | private lazy val previousSessionF: Future[Option[Session]] = repo.get(qAgent, sessionId) 21 | 22 | def receive: Receive = initializing(Vector.empty) 23 | 24 | def initializing(initReadings: Vector[Reading]): Receive = { 25 | case Report(reading, _) ⇒ 26 | val (newContext, response) = tryStart(initReadings :+ reading) 27 | sender ! response 28 | context become newContext 29 | 30 | case m @ (RequestAction | QueryStatus) ⇒ tryStart(initReadings) match { 31 | case (newContext, ActionsAvailable) ⇒ 32 | self forward m 33 | context become newContext 34 | case (newContext, response) ⇒ 35 | sender ! response 36 | context become newContext 37 | } 38 | 39 | } 40 | 41 | def inSession(session: Session): Receive = { 42 | case Report(reading, reward) ⇒ 43 | context become inSession(qAgent.report(reading, reward, session)) 44 | sender ! ActionsAvailable 45 | 46 | case RequestAction ⇒ 47 | val (action, newSession) = qAgent.requestAction(session) 48 | sender ! ActionResult(action) 49 | context become inSession(newSession) 50 | 51 | case ReportTermination(reading, reward) ⇒ 52 | val newS = qAgent.report(reading, reward, session) 53 | val closed = qAgent.close(newS) 54 | val replyTo = sender 55 | storeSession(closed).map { _ ⇒ 56 | replyTo ! Confirmed 57 | }.onFailure { 58 | case e: Throwable ⇒ throw e 59 | } 60 | context stop self 61 | 62 | case QueryStatus ⇒ 63 | sender ! AgentStatus(session.iteration.memory.size, session.iteration.loss) 64 | } 65 | 66 | private def tryStart(readings: Vector[Reading]): (Receive, Response) = { 67 | 68 | def startAgent(previousSession: Option[Session]): (Receive, Response) = 69 | qAgent.start(readings, previousSession) match { 70 | case Left(m) ⇒ 71 | (initializing(readings), PendingMoreReadings) 72 | 73 | case Right(session) ⇒ 74 | (inSession(session), ActionsAvailable) 75 | } 76 | 77 | previousSessionF.value match { 78 | case Some(Success(previousSession)) ⇒ 79 | startAgent(previousSession) 80 | case Some(Failure(e)) ⇒ 81 | throw e 82 | case None ⇒ //future not completed yet 83 | (initializing(readings), Initializing) 84 | } 85 | 86 | } 87 | 88 | private def storeSession(session: Session): Future[Unit] = repo.upsert(qAgent, sessionId)(session) 89 | 90 | } 91 | 92 | object AgentForUser { 93 | def props[AT <: QAgent: SessionPersistence](qAgent: AT, sessionId: SessionId): Props = 94 | 95 | Props(new AgentForUser(qAgent, sessionId)) 96 | 97 | } 98 | -------------------------------------------------------------------------------- /akka-interface-service/src/main/scala/glaux/interfaces.akka/service/Protocols.scala: -------------------------------------------------------------------------------- 1 | package glaux.interfaces.akka.service 2 | 3 | import akka.actor.ActorRef 4 | import glaux.interfaces.api.domain._ 5 | 6 | object Protocols { 7 | 8 | sealed trait Response 9 | sealed trait Request[T <: Response] 10 | sealed trait Request2[T1 <: Response, T2 <: Response] 11 | sealed trait Request3[T1 <: Response, T2 <: Response, T3 <: Response] 12 | 13 | case object Confirmed extends Response 14 | case class Rejected(reason: String) extends Response 15 | 16 | trait RequestWithConfirmation extends Request2[Confirmed.type, Rejected] 17 | 18 | object Agency { 19 | 20 | case class GetAgentForUser(sessionId: SessionId) extends Request2[AgentRef, Rejected] 21 | 22 | case class AgentRef(ref: ActorRef) extends Response 23 | 24 | case class CreateAgentSettings(settings: AgentSettings) extends RequestWithConfirmation 25 | 26 | } 27 | 28 | object Agent { 29 | 30 | case class Report(reading: Reading, reward: Reward) extends Request3[ActionsAvailable.type, PendingMoreReadings.type, Initializing.type] 31 | 32 | case class ReportTermination(reading: Reading, reward: Reward) extends RequestWithConfirmation 33 | 34 | case object RequestAction extends Request3[ActionResult, PendingMoreReadings.type, Initializing.type] 35 | 36 | case class ActionResult(action: Action) extends Response 37 | 38 | case object QueryStatus extends Request2[AgentStatus, Initializing.type] 39 | 40 | case class AgentStatus(memorySize: Int, currentLoss: Double) extends Response 41 | 42 | case object PendingMoreReadings extends Response 43 | case object Initializing extends Response 44 | case object ActionsAvailable extends Response 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /akka-interface-service/src/main/scala/glaux/interfaces.akka/service/main.scala: -------------------------------------------------------------------------------- 1 | package glaux.interfaces.akka.service 2 | 3 | import akka.actor.ActorSystem 4 | 5 | object AkkaApp extends App { 6 | val system = ActorSystem("glaux") 7 | } 8 | -------------------------------------------------------------------------------- /akka-interface-service/src/test/scala/glaux/interfaces.akka/service/AgencySpec.scala: -------------------------------------------------------------------------------- 1 | package glaux.interfaces.akka.service 2 | 3 | import akka.actor._ 4 | import akka.testkit.{ImplicitSender, TestKit} 5 | import Protocols.Agency.{CreateAgentSettings, AgentRef, GetAgentForUser} 6 | import Protocols.{Rejected, Confirmed} 7 | import glaux.interfaces.api.persistence.{SessionPersistence, PersistenceImpl, AgentSettingsPersistence, Persistence} 8 | import glaux.neuralnetwork.trainers.SGD.SGDSettings 9 | import glaux.reinforcementlearning.DeepMindQLearner.ConvolutionBased 10 | import glaux.reinforcementlearning.{AdvancedQAgent, SimpleQAgent} 11 | import org.specs2.mutable.Specification 12 | import org.specs2.specification.{AfterEach, Scope, AfterAll} 13 | import glaux.interfaces.api.domain.{AdvancedAgentSettings, AgentName, AgentSettings, SessionId} 14 | 15 | import scala.concurrent.{Future, Await} 16 | import scala.concurrent.duration._ 17 | 18 | class AgencySpec extends Specification with AfterAll { 19 | sequential 20 | implicit lazy val system = ActorSystem() 21 | "GetAgent" >> { 22 | "starts new agent " in new AgencyScope { 23 | agency ! GetAgentForUser(SessionId("A Test", "testUser")) 24 | expectMsgType[AgentRef] 25 | } 26 | 27 | "returns existing agent if found" in new AgencyScope { 28 | val agentId = SessionId("A Test", "testUser") 29 | 30 | agency ! GetAgentForUser(agentId) 31 | val agent = expectMsgType[AgentRef].ref 32 | agency ! GetAgentForUser(agentId) 33 | expectMsgType[AgentRef].ref === agent 34 | } 35 | 36 | "returns existing agent if hit multiple times" in new AgencyScope { 37 | val agentId = SessionId("A Test", "testUser") 38 | 39 | agency ! GetAgentForUser(agentId) 40 | agency ! GetAgentForUser(agentId) 41 | val agent = expectMsgType[AgentRef].ref 42 | expectMsgType[AgentRef].ref === agent 43 | } 44 | 45 | "create new agent id is new" in new AgencyScope { 46 | val agentId = SessionId("A Test", "testUser") 47 | 48 | agency ! GetAgentForUser(agentId) 49 | val agent = expectMsgType[AgentRef].ref 50 | 51 | agency ! GetAgentForUser(SessionId("A Test", "anotherUser")) 52 | expectMsgType[AgentRef].ref !== agent 53 | } 54 | 55 | "recreate new agent after last agent is terminated " in new AgencyScope { 56 | val agentId = SessionId("A Test", "testUser") 57 | 58 | agency ! GetAgentForUser(agentId) 59 | val agent = expectMsgType[AgentRef].ref 60 | watch(agent) 61 | agent ! PoisonPill 62 | expectTerminated(agent) 63 | 64 | { 65 | agency ! GetAgentForUser(agentId) 66 | expectMsgType[AgentRef].ref 67 | } must be_!=(agent).eventually 68 | } 69 | 70 | } 71 | 72 | "CreateAgentSettings" >> { 73 | 74 | "creates one with confirmation" in new AgencyScope { 75 | val agency1 = system.actorOf(Agency.props(mockRepoWith())) 76 | agency1 ! CreateAgentSettings(AdvancedAgentSettings("test", 5, ConvolutionBased.Settings(), SGDSettings())) 77 | expectMsg(Confirmed) 78 | 79 | } 80 | 81 | "fails with rejection" in new AgencyScope { 82 | val agency1 = system.actorOf(Agency.props(mockRepoWith(Future.failed(new Exception())))) 83 | agency1 ! CreateAgentSettings(AdvancedAgentSettings("test", 5, ConvolutionBased.Settings(), SGDSettings())) 84 | expectMsg(Rejected) 85 | } 86 | } 87 | 88 | def afterAll(): Unit = { 89 | system.terminate() 90 | } 91 | } 92 | 93 | class AgencyScope(implicit system: ActorSystem) extends TestKit(system) with ImplicitSender with Scope { 94 | def mockRepoWith( 95 | upsertResult: Future[Unit] = Future.successful(()), 96 | getSettingResult: Option[AgentSettings] = None 97 | ) = new MockPersistence { 98 | override implicit def agentSettingsPersistence: AgentSettingsPersistence = new AgentSettingsPersistence { 99 | def get(name: AgentName): Future[Option[AgentSettings]] = Future.successful(getSettingResult) 100 | def upsert(settings: AgentSettings): Future[Unit] = upsertResult 101 | } 102 | } 103 | 104 | val testAgentSettings = Some(AdvancedAgentSettings("test", 8, ConvolutionBased.Settings(), SGDSettings())) 105 | 106 | lazy val agency = system.actorOf(Agency.props(mockRepoWith(getSettingResult = testAgentSettings))) 107 | 108 | } 109 | -------------------------------------------------------------------------------- /akka-interface-service/src/test/scala/glaux/interfaces.akka/service/AgentSpec.scala: -------------------------------------------------------------------------------- 1 | package glaux.interfaces.akka.service 2 | 3 | import java.time.ZonedDateTime 4 | 5 | import akka.actor._ 6 | import akka.testkit.{TestProbe, ImplicitSender, TestKit} 7 | import Protocols.Agent._ 8 | import Protocols.{Confirmed, Response} 9 | import glaux.interfaces.api.domain.SessionId 10 | import glaux.reinforcementlearning.{QAgent, Reading, Reward, SimpleQAgent} 11 | import org.specs2.mutable.Specification 12 | import org.specs2.specification.{AfterEach, Scope, AfterAll} 13 | 14 | import scala.concurrent.{Future, Await} 15 | import scala.util.Random 16 | import scala.concurrent.duration._ 17 | 18 | class AgentSpec extends Specification with AfterAll { 19 | sequential 20 | implicit lazy val system = ActorSystem() 21 | 22 | "start from new session" >> { 23 | "does not allow action until getting enough data" in new withAkka { 24 | report() 25 | report() 26 | waitForInitialization() 27 | agent ! RequestAction 28 | expectMsg(PendingMoreReadings) 29 | } 30 | 31 | "does allow action after getting enough data" in new withAkka { 32 | report() 33 | report() 34 | report() 35 | 36 | waitForInitialization() 37 | 38 | agent ! RequestAction 39 | expectMsgType[ActionResult] 40 | } 41 | 42 | "reports action available after initialized" in new withAkka { 43 | initialize() 44 | agent ! Report(randomReading, 3) 45 | expectMsg(ActionsAvailable) 46 | } 47 | 48 | "reports action result after initialized" in new withAkka { 49 | initialize() 50 | agent ! RequestAction 51 | expectMsgType[ActionResult] 52 | } 53 | 54 | "increase memory after one action feedback loop" in new withAkka { 55 | initialize() 56 | requestAction() 57 | agent ! QueryStatus 58 | val status = expectMsgType[AgentStatus] 59 | status.memorySize === 0 60 | 61 | report() 62 | report() 63 | requestAction() 64 | agent ! QueryStatus 65 | val newStatus = expectMsgType[AgentStatus] 66 | newStatus.memorySize === 1 67 | 68 | } 69 | 70 | "confirm after Report Termination" in new withAkka { 71 | initialize() 72 | agent ! ReportTermination(randomReading, 0) 73 | 74 | expectMsg(Confirmed) 75 | } 76 | 77 | "terminates after Report Termination" in new withAkka { 78 | initialize() 79 | watch(agent) 80 | ignoreMsg { case Confirmed ⇒ true } 81 | agent ! ReportTermination(randomReading, 0) 82 | expectTerminated(agent) 83 | } 84 | 85 | "start from exiting previous session" >> { 86 | "pick up previous session" in new withAkka { 87 | initialize() 88 | requestAction() 89 | agent ! ReportTermination(randomReading, 1) 90 | expectMsg(Confirmed) 91 | 92 | val newAgent = system.actorOf(agentProps) 93 | 94 | initialize(newAgent) 95 | newAgent ! QueryStatus 96 | val status = expectMsgType[AgentStatus] 97 | status.memorySize === 1 98 | } 99 | 100 | } 101 | } 102 | 103 | def afterAll(): Unit = { 104 | system.terminate() 105 | } 106 | } 107 | 108 | class withAkka(implicit system: ActorSystem) extends TestKit(system) with ImplicitSender with Scope { 109 | 110 | import glaux.interfaces.api.domain.SessionId 111 | 112 | def randomReading = (Seq[Double](Random.nextDouble(), Random.nextDouble(), Random.nextDouble()), ZonedDateTime.now) 113 | val sa = SimpleQAgent(8, historyLength = 3) 114 | import MockPersistence._ 115 | lazy val agentProps = AgentForUser.props(sa, SessionId("atest", Random.nextString(3))) 116 | 117 | lazy val agent = system.actorOf(agentProps) 118 | 119 | def waitForInitialization(a: ActorRef = agent): Unit = { 120 | a ! QueryStatus 121 | fishForMessage(3.seconds) { 122 | case Initializing ⇒ 123 | a ! QueryStatus 124 | false 125 | case _ ⇒ true 126 | } 127 | } 128 | 129 | def report(a: ActorRef = agent, reading: Reading = randomReading, reward: Reward = Random.nextDouble()): Any = { 130 | a ! Report(reading, reward) 131 | expectReportResponse(a) 132 | } 133 | 134 | def expectReportResponse(a: ActorRef = agent): Any = 135 | expectMsgAnyOf(30.seconds, ActionsAvailable, Initializing, PendingMoreReadings) 136 | 137 | def requestAction(a: ActorRef = agent): ActionResult = { 138 | a ! RequestAction 139 | expectMsgType[ActionResult] 140 | } 141 | 142 | def initialize(a: ActorRef = agent): Unit = { 143 | report(a) 144 | report(a) 145 | report(a) 146 | waitForInitialization(a) 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /akka-interface-service/src/test/scala/glaux/interfaces.akka/service/MockPersistence.scala: -------------------------------------------------------------------------------- 1 | package glaux.interfaces.akka.service 2 | 3 | import glaux.interfaces.api.domain.{SessionId, AgentName, AgentSettings} 4 | import glaux.interfaces.api.persistence.{Persistence, SessionPersistence, PersistenceImpl} 5 | import glaux.reinforcementlearning.{QAgent, AdvancedQAgent, SimpleQAgent} 6 | 7 | import scala.concurrent.Future 8 | 9 | trait MockPersistence extends PersistenceImpl { 10 | case class MapBasedMockPersistence[T, K](getKey: T ⇒ K) extends Persistence[T, K] { 11 | @volatile 12 | var store: Map[K, T] = Map() 13 | 14 | def get(id: K): Future[Option[T]] = Future.successful(store.get(id)) 15 | 16 | def upsert(t: T): Future[Unit] = { 17 | store += getKey(t) → t 18 | Future.successful(()) 19 | } 20 | } 21 | 22 | class MockSessionPersistence[A <: QAgent] extends SessionPersistence[A] { 23 | @volatile 24 | var store: Map[SessionId, QAgent.Session[_, _]] = Map() 25 | 26 | def get(agent: A, id: SessionId): Future[Option[agent.Session]] = 27 | Future.successful(store.get(id).map(_.asInstanceOf[agent.Session])) 28 | 29 | def upsert(agent: A, id: SessionId)(session: agent.Session): Future[Unit] = { 30 | store += id → session 31 | Future.successful(()) 32 | } 33 | } 34 | 35 | implicit def advanceQAgentSessionPersistence: SessionPersistence[AdvancedQAgent] = new MockSessionPersistence[AdvancedQAgent] 36 | 37 | implicit def agentSettingsPersistence: Persistence[AgentSettings, AgentName] = new MapBasedMockPersistence[AgentSettings, AgentName](_.name) 38 | 39 | implicit def simpleQAgentSessionPersistence: SessionPersistence[SimpleQAgent] = new MockSessionPersistence[SimpleQAgent] 40 | } 41 | 42 | object MockPersistence extends MockPersistence 43 | -------------------------------------------------------------------------------- /interface-api/src/main/scala/glaux/interfaces/api/domain/models.scala: -------------------------------------------------------------------------------- 1 | package glaux.interfaces.api 2 | package domain 3 | 4 | import glaux.neuralnetwork.trainers.SGD.SGDSettings 5 | import glaux.reinforcementlearning.DeepMindQLearner.ConvolutionBased 6 | 7 | case class SessionId(agentName: AgentName, profileId: ProfileId) 8 | 9 | sealed trait AgentSettings { 10 | def name: AgentName 11 | } 12 | 13 | case class AdvancedAgentSettings( 14 | name: AgentName, 15 | numOfActions: Int, 16 | learnerSettings: ConvolutionBased.Settings, 17 | trainerSettings: SGDSettings 18 | ) extends AgentSettings 19 | -------------------------------------------------------------------------------- /interface-api/src/main/scala/glaux/interfaces/api/domain/package.scala: -------------------------------------------------------------------------------- 1 | package glaux.interfaces.api 2 | 3 | package object domain { 4 | 5 | type ProfileId = String 6 | type AgentName = String 7 | 8 | type Reward = glaux.reinforcementlearning.Reward 9 | type Action = glaux.reinforcementlearning.Action 10 | type Time = glaux.reinforcementlearning.Time 11 | type Reading = glaux.reinforcementlearning.Reading 12 | } 13 | -------------------------------------------------------------------------------- /interface-api/src/main/scala/glaux/interfaces/api/persistence/package.scala: -------------------------------------------------------------------------------- 1 | package glaux.interfaces.api 2 | 3 | import glaux.interfaces.api.domain.{SessionId, AgentSettings, AgentName} 4 | import glaux.reinforcementlearning.{SimpleQAgent, AdvancedQAgent, QAgent} 5 | 6 | import scala.concurrent.Future 7 | 8 | package object persistence { 9 | 10 | type AgentSettingsPersistence = Persistence[AgentSettings, AgentName] 11 | 12 | } 13 | 14 | package persistence { 15 | 16 | trait Persistence[T, Key] { 17 | def get(k: Key): Future[Option[T]] 18 | def upsert(t: T): Future[Unit] 19 | } 20 | trait SessionPersistence[A <: QAgent] { 21 | def get(agent: A, id: SessionId): Future[Option[agent.Session]] 22 | def upsert(agent: A, id: SessionId)(session: agent.Session): Future[Unit] 23 | } 24 | 25 | trait PersistenceImpl { 26 | implicit def advanceQAgentSessionPersistence: SessionPersistence[AdvancedQAgent] 27 | implicit def simpleQAgentSessionPersistence: SessionPersistence[SimpleQAgent] 28 | implicit def agentSettingsPersistence: AgentSettingsPersistence 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /linear-algebra/src/main/scala/glaux/linearalgebra/Dimension.scala: -------------------------------------------------------------------------------- 1 | package glaux.linearalgebra 2 | 3 | import Dimension.Shape 4 | 5 | sealed trait Dimension { 6 | lazy val totalSize: Int = shape.reduce(_ * _) 7 | def shape: Shape 8 | 9 | def ranges: Array[Range] = shape.map(Range(0, _)) 10 | } 11 | 12 | object Dimension { 13 | type Shape = Array[Int] 14 | 15 | trait DimensionFactory[DT <: Dimension] { 16 | val createFunction: PartialFunction[Shape, DT] 17 | lazy val tryCreate: Shape ⇒ Option[DT] = createFunction.lift 18 | def create(shape: Shape): DT = tryCreate(shape).getOrElse(throw unsupported(shape)) 19 | } 20 | 21 | private def unsupported(shape: Shape): Exception = new UnsupportedOperationException(s"unrecognized INDArray shape (${shape.mkString(",")})") 22 | 23 | def ofSpecific[DT <: Dimension](shape: Shape)(implicit df: DimensionFactory[DT]): DT = { 24 | df.tryCreate(shape).getOrElse(throw unsupported(shape)) 25 | } 26 | 27 | def of(shape: Shape): Dimension = { 28 | val df4d = implicitly[DimensionFactory[FourD]] 29 | val df3d = implicitly[DimensionFactory[ThreeD]] 30 | val df2d = implicitly[DimensionFactory[TwoD]] 31 | val df1d = implicitly[DimensionFactory[Row]] 32 | (Seq(df4d, df3d, df2d, df1d).foldLeft[Option[Dimension]](None) { (lr, f) ⇒ lr orElse f.tryCreate(shape) }).getOrElse(throw unsupported(shape)) 33 | } 34 | 35 | case class FourD(x: Int, y: Int, z: Int, f: Int) extends Dimension { 36 | assert(x > 0 && y > 0 && z > 0 && f > 0) 37 | def shape: Shape = Array(x, y, z, f) 38 | } 39 | 40 | case class ThreeD(x: Int, y: Int, z: Int) extends Dimension { 41 | assert(x > 0 && y > 0 && z > 0) 42 | def shape: Shape = Array(x, y, z) 43 | } 44 | 45 | case class TwoD(x: Int, y: Int) extends Dimension { 46 | assert(x > 0 && y > 0) 47 | def shape: Shape = Array(x, y) 48 | } 49 | 50 | case class Row(size: Int) extends Dimension { 51 | assert(size > 0) 52 | def shape: Shape = Array(1, size) 53 | } 54 | 55 | implicit object Row extends DimensionFactory[Row] { 56 | val dimIndexOfData: Int = 1 57 | 58 | val createFunction: PartialFunction[Shape, Row] = { 59 | case Array(1, size) ⇒ Row(size) 60 | } 61 | } 62 | 63 | implicit object TwoD extends DimensionFactory[TwoD] { 64 | val createFunction: PartialFunction[Shape, TwoD] = { 65 | case Array(x, y) if x > 1 ⇒ TwoD(x, y) 66 | } 67 | } 68 | 69 | implicit object ThreeD extends DimensionFactory[ThreeD] { 70 | val createFunction: PartialFunction[Shape, ThreeD] = { 71 | case Array(x, y, z) ⇒ ThreeD(x, y, z) 72 | } 73 | } 74 | 75 | implicit object FourD extends DimensionFactory[FourD] { 76 | val createFunction: PartialFunction[Shape, FourD] = { 77 | case Array(x, y, z, f) ⇒ FourD(x, y, z, f) 78 | } 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /linear-algebra/src/main/scala/glaux/linearalgebra/Implementation.scala: -------------------------------------------------------------------------------- 1 | package glaux.linearalgebra 2 | 3 | import glaux.linearalgebra.Tensor._ 4 | 5 | // Implement this trait to provide an implementation of the linarg interfaces. 6 | trait Implementation { 7 | implicit val rBuilder: RowBuilder 8 | implicit val mBuilder: MatrixBuilder 9 | implicit val vBuilder: VolBuilder 10 | implicit val t4Builder: Tensor4Builder 11 | implicit val genBuilder: GenTensorBuilder[Tensor] 12 | } 13 | -------------------------------------------------------------------------------- /linear-algebra/src/main/scala/glaux/linearalgebra/Tensor.scala: -------------------------------------------------------------------------------- 1 | package glaux.linearalgebra 2 | 3 | import Dimension._ 4 | import Tensor._ 5 | import glaux.statistics.{RealDistribution, Distribution} 6 | 7 | trait Tensor extends TensorOperations { 8 | 9 | type Dimensionality <: Dimension 10 | def dimension: Dimensionality 11 | def sumAll: Double = seqView.sum 12 | def seqView: Seq[Double] 13 | def head: Double = seqView.head 14 | def toArray: Array[Double] = seqView.toArray 15 | def toRowVector: RowVector = 16 | if (isInstanceOf[RowVector]) this.asInstanceOf[RowVector] else (Row(dimension.totalSize), seqView) 17 | def getSalar(indices: Seq[Int]): Double 18 | } 19 | 20 | trait TensorOperations { 21 | 22 | def +(that: Tensor): Tensor 23 | 24 | def -(that: Tensor): Tensor 25 | 26 | /** element-by-element multiplication */ 27 | def *(that: Tensor): Tensor 28 | 29 | /** matrix multiplication */ 30 | def **(that: Tensor): Tensor 31 | 32 | def /(that: Tensor): Tensor 33 | 34 | /** right division ... is this the correct symbol? */ 35 | def \(that: Tensor): Tensor 36 | 37 | def +(that: Number): Tensor 38 | 39 | def -(that: Number): Tensor 40 | 41 | def *(that: Number): Tensor 42 | 43 | def /(that: Number): Tensor 44 | 45 | def \(that: Number): Tensor 46 | 47 | def T: Tensor 48 | 49 | } 50 | 51 | object Tensor { 52 | 53 | type CanBuildFrom[From, V <: Tensor] = From ⇒ V 54 | type TensorBuilder[V <: Tensor] = CanBuildFrom[(V#Dimensionality, Seq[Double]), V] //Seq instead of Iterable for performance concerns 55 | type GenTensorBuilder[V <: Tensor] = CanBuildFrom[(Dimension, Seq[Double]), V] 56 | type RowBuilder = TensorBuilder[RowVector] 57 | type MatrixBuilder = TensorBuilder[Matrix] 58 | type VolBuilder = TensorBuilder[Vol] 59 | type Tensor4Builder = TensorBuilder[Tensor4] 60 | 61 | implicit val toRow: CanBuildFrom[Tensor, RowVector] = v ⇒ v.asInstanceOf[RowVector] 62 | implicit val toMatrix: CanBuildFrom[Tensor, Matrix] = v ⇒ v.asInstanceOf[Matrix] 63 | 64 | implicit def toGen[V <: Tensor](implicit gb: GenTensorBuilder[V]): TensorBuilder[V] = gb 65 | implicit class TensorOps[V <: Tensor: TensorBuilder](self: V) { 66 | 67 | def map(f: Double ⇒ Double): V = (self.dimension, self.seqView.map(f)) 68 | 69 | def fill(value: Double): V = map(_ ⇒ 0) 70 | 71 | def merge(v2: V)(f: (Double, Double) ⇒ Double): V = { 72 | assert(self.dimension == v2.dimension) 73 | (self.dimension, self.seqView.zip(v2.seqView).map(f.tupled)) 74 | } 75 | 76 | } 77 | 78 | def apply(dimension: Dimension, data: Seq[Double]): Tensor = (dimension, data) 79 | 80 | } 81 | 82 | trait RowVector extends Tensor { 83 | type Dimensionality = Row 84 | def apply(index: Int): Double 85 | def dot(that: RowVector): Double = { 86 | assert(dimension == that.dimension, "can only dot vector with the same dimension") 87 | (this ** that.T).head 88 | } 89 | 90 | def update(index: Int, value: Double): RowVector = (dimension, seqView.updated(index, value)) 91 | } 92 | 93 | trait Matrix extends Tensor { 94 | type Dimensionality = TwoD 95 | def apply(x: Int, y: Int) = getSalar(Seq(x, y)) 96 | } 97 | 98 | trait Vol extends Tensor { 99 | type Dimensionality = ThreeD 100 | def apply(x: Int, y: Int, z: Int) = getSalar(Seq(x, y, z)) 101 | 102 | } 103 | 104 | trait Tensor4 extends Tensor { 105 | type Dimensionality = FourD 106 | def apply(x: Int, y: Int, z: Int, f: Int) = getSalar(Seq(x, y, z, f)) 107 | } 108 | 109 | trait TensorFactory[V <: Tensor] { 110 | def apply(dimension: V#Dimensionality, data: Seq[Double])(implicit b: TensorBuilder[V]): V = b((dimension, data)) 111 | 112 | def fill(dimension: V#Dimensionality, value: Double)(implicit b: TensorBuilder[V]): V = apply(dimension, Array.fill(dimension.totalSize)(value)) 113 | 114 | def sampleOf(dimension: V#Dimensionality, dist: RealDistribution, size: Int)(implicit b: TensorBuilder[V]): Iterable[V] = 115 | 1.until(size).map(_ ⇒ sampleOf(dimension, dist)) 116 | 117 | def sampleOf(dimension: V#Dimensionality, dist: RealDistribution)(implicit b: TensorBuilder[V]): V = 118 | apply(dimension, dist.sample(dimension.totalSize).toSeq) 119 | 120 | def normalized(dimension: V#Dimensionality, size: Int)(implicit b: TensorBuilder[V]): V = { 121 | import glaux.statistics.distributions.normal 122 | sampleOf(dimension, normal(0, Math.sqrt(1.0d / size))) 123 | } 124 | 125 | } 126 | 127 | object RowVector extends TensorFactory[RowVector] { 128 | def apply(values: Double*): RowVector = apply(Dimension.Row(values.length), values) 129 | } 130 | 131 | object Matrix extends TensorFactory[Matrix] { 132 | def apply(x: Int, y: Int, data: Seq[Double]): Matrix = apply(Dimension.TwoD(x, y), data) 133 | } 134 | 135 | object Vol extends TensorFactory[Vol] { 136 | def apply(x: Int, y: Int, z: Int, data: Seq[Double]): Vol = apply(Dimension.ThreeD(x, y, z), data) 137 | } 138 | 139 | object Tensor4 extends TensorFactory[Tensor4] { 140 | def apply(x: Int, y: Int, z: Int, f: Int, data: Seq[Double]): Tensor4 = apply(Dimension.FourD(x, y, z, f), data) 141 | } 142 | 143 | -------------------------------------------------------------------------------- /linear-algebra/src/main/scala/glaux/linearalgebra/impl/nd4j/Implicits.scala: -------------------------------------------------------------------------------- 1 | package glaux.linearalgebra.impl.nd4j 2 | 3 | import glaux.linearalgebra.Dimension.{FourD, Row, TwoD, ThreeD} 4 | import glaux.linearalgebra._ 5 | import glaux.linearalgebra.Tensor._ 6 | import org.nd4j.linalg.api.ndarray.INDArray 7 | import org.nd4j.linalg.factory.Nd4j 8 | 9 | object Implicits { 10 | implicit val tensorBuilder: CanBuildFrom[INDArray, Tensor] = indArray ⇒ 11 | Dimension.of(indArray.shape) match { 12 | case d @ FourD(_, _, _, _) ⇒ Tensor4Imp(indArray) 13 | case d @ ThreeD(_, _, _) ⇒ VolImp(indArray) 14 | case d @ TwoD(_, _) ⇒ MatrixImp(indArray) 15 | case d @ Row(_) ⇒ RowVectorImp(indArray) 16 | } 17 | 18 | type IndArrayVolBuilder[V <: Tensor] = CanBuildFrom[INDArray, V] 19 | 20 | implicit val indArrayRowBuilder: IndArrayVolBuilder[RowVector] = RowVectorImp.apply 21 | implicit val indArrayMatrixBuilder: IndArrayVolBuilder[Matrix] = MatrixImp.apply 22 | implicit val indArrayVolBuilder: IndArrayVolBuilder[Vol] = VolImp.apply 23 | implicit val indArrayTensor4Builder: IndArrayVolBuilder[Tensor4] = Tensor4Imp.apply 24 | 25 | implicit def builder[V <: Tensor](implicit indArrayVolBuilder: IndArrayVolBuilder[V]): TensorBuilder[V] = (createINDArray _).tupled.andThen(indArrayVolBuilder) 26 | implicit val rBuilder: RowBuilder = builder[RowVector] 27 | implicit val mBuilder: MatrixBuilder = builder[Matrix] 28 | implicit val vBuilder: VolBuilder = builder[Vol] 29 | implicit val t4Builder: Tensor4Builder = builder[Tensor4] 30 | implicit val genBuilder: GenTensorBuilder[Tensor] = (createINDArray _).tupled.andThen(tensorBuilder) 31 | 32 | implicit def toWithIndArray(v: Tensor): WithIndArray = new WithIndArray { val indArray = v.asInstanceOf[ND4JBackedTensor].indArray } 33 | 34 | private def createINDArray(dimension: Dimension, data: Seq[Double]): INDArray = { 35 | assert(dimension.totalSize == data.length, s"data length ${data.length} does not conform to $dimension") 36 | Nd4j.create(data.toArray, dimension.shape) 37 | } 38 | 39 | } 40 | 41 | class ND4JImplementation extends Implementation { 42 | implicit val rBuilder = Implicits.rBuilder 43 | implicit val mBuilder = Implicits.mBuilder 44 | implicit val vBuilder = Implicits.vBuilder 45 | implicit val t4Builder = Implicits.t4Builder 46 | implicit val genBuilder = Implicits.genBuilder 47 | } 48 | 49 | -------------------------------------------------------------------------------- /linear-algebra/src/main/scala/glaux/linearalgebra/impl/nd4j/TensorImpl.scala: -------------------------------------------------------------------------------- 1 | package glaux.linearalgebra.impl.nd4j 2 | 3 | import glaux.linearalgebra.Dimension._ 4 | import glaux.linearalgebra._ 5 | import org.nd4j.linalg.api.ndarray.INDArray 6 | 7 | trait TensorImpl extends Tensor with TensorOperationsImpl with WithIndArray { 8 | def getSalar(indices: Seq[Int]): Double = indArray.getDouble(indices: _*) 9 | } 10 | 11 | trait WithIndArray { 12 | val indArray: INDArray 13 | } 14 | 15 | trait TensorOperationsImpl extends TensorOperations { 16 | this: WithIndArray ⇒ 17 | 18 | import Implicits.tensorBuilder 19 | import Implicits.toWithIndArray 20 | def +(that: Tensor): Tensor = this.indArray.add(that.indArray) 21 | 22 | def -(that: Tensor): Tensor = indArray.sub(that.indArray) 23 | 24 | /** element-by-element multiplication */ 25 | def *(that: Tensor): Tensor = indArray.mul(that.indArray) 26 | 27 | /** matrix multiplication */ 28 | def **(that: Tensor): Tensor = indArray.mmul(that.indArray) 29 | 30 | def /(that: Tensor): Tensor = indArray.div(that.indArray) 31 | 32 | /** right division ... is this the correct symbol? */ 33 | def \(that: Tensor): Tensor = indArray.rdiv(that.indArray) 34 | 35 | def +(that: Number): Tensor = indArray.add(that) 36 | def -(that: Number): Tensor = indArray.sub(that) 37 | def *(that: Number): Tensor = indArray.mul(that) 38 | def /(that: Number): Tensor = indArray.div(that) 39 | def \(that: Number): Tensor = indArray.rdiv(that) 40 | 41 | def T: Tensor = indArray.transpose 42 | 43 | } 44 | 45 | protected sealed abstract class ND4JBackedTensor(val indArray: INDArray) extends TensorImpl { 46 | val dimensionFactory: DimensionFactory[Dimensionality] 47 | 48 | lazy val dimension: Dimensionality = dimensionFactory.create(indArray.shape()) 49 | def seqView: Seq[Double] = { 50 | 51 | new Seq[Double] { 52 | outer ⇒ 53 | def length = dimension.totalSize 54 | def apply(idx: Int) = indArray.getDouble(idx) 55 | //todo: although it's trivial to implement, I think there should be an existing iterator for seq somewhere. 56 | def iterator = new Iterator[Double] { 57 | var index: Int = 0 58 | override def hasNext: Boolean = index < outer.length 59 | override def next(): Double = { 60 | index += 1 61 | apply(index - 1) 62 | } 63 | } 64 | } 65 | } 66 | 67 | } 68 | 69 | case class RowVectorImp(override val indArray: INDArray) extends ND4JBackedTensor(indArray) with RowVector { 70 | val dimensionFactory = Row 71 | def apply(index: Int): Double = indArray.getDouble(index) 72 | } 73 | 74 | case class MatrixImp(override val indArray: INDArray) extends ND4JBackedTensor(indArray) with Matrix { 75 | val dimensionFactory = TwoD 76 | } 77 | 78 | case class VolImp(override val indArray: INDArray) extends ND4JBackedTensor(indArray) with Vol { 79 | val dimensionFactory = ThreeD 80 | } 81 | 82 | case class Tensor4Imp(override val indArray: INDArray) extends ND4JBackedTensor(indArray) with Tensor4 { 83 | val dimensionFactory = FourD 84 | } 85 | 86 | -------------------------------------------------------------------------------- /linear-algebra/src/main/scala/glaux/linearalgebra/package.scala: -------------------------------------------------------------------------------- 1 | package glaux 2 | 3 | import glaux.linearalgebra.impl.nd4j.ND4JImplementation 4 | 5 | //The single link towards the implementation linalg data structure. 6 | package object linearalgebra extends ND4JImplementation with Implementation { 7 | 8 | } 9 | -------------------------------------------------------------------------------- /linear-algebra/src/test/scala/glaux/linearalgebra/TensorSpec.scala: -------------------------------------------------------------------------------- 1 | package glaux.linearalgebra 2 | 3 | import glaux.linearalgebra.Dimension.{Row, ThreeD, TwoD} 4 | import org.specs2.mutable.Specification 5 | 6 | class TensorSpec 7 | extends Specification { 8 | 9 | "Normal Vols" >> { 10 | val dim = ThreeD(1, 1, 3) 11 | 12 | "* op keeps dimension" >> { 13 | val vol = Tensor(dim, Seq(2, 3, 4)) 14 | val vol2: Tensor = vol * vol 15 | vol2 must_== Tensor(dim, Seq(4, 9, 16)) 16 | } 17 | 18 | "* scalar keeps dimension" >> { 19 | val vol = Tensor(dim, Seq(2, 3, 4)) 20 | val vol2: Tensor = vol * 0.5 21 | vol2 must_== Tensor(dim, Seq(1, 1.5, 2)) 22 | } 23 | 24 | "sum correctly" >> { 25 | val m = Matrix(2, 2, Seq(1, 2, 3, 4)) 26 | m.sumAll must_== 10 27 | } 28 | 29 | "map" >> { 30 | "with immutability" >> { 31 | val m = Matrix(2, 2, Seq(1, 2, 3, 4)) 32 | m.map(_ * 2) 33 | m must_== Matrix(2, 2, Seq(1, 2, 3, 4)) 34 | } 35 | 36 | "correctly" >> { 37 | val m = Matrix(2, 2, Seq(1, 2, 3, 4)) 38 | val result = m.map(_ * 2) 39 | result must_== Matrix(2, 2, Seq(2, 4, 6, 8)) 40 | } 41 | } 42 | 43 | "toRow" >> { 44 | "matrix" >> { 45 | val m = Matrix(2, 2, Seq(1, 2, 3, 4)) 46 | m.toRowVector must_== RowVector(1, 2, 3, 4) 47 | } 48 | "row vector" >> { 49 | val r = RowVector(1, 2, 3, 4) 50 | r.toRowVector must_== RowVector(1, 2, 3, 4) 51 | } 52 | 53 | } 54 | 55 | "merge" >> { 56 | 57 | "correctly" >> { 58 | val m = Vol(ThreeD(2, 2, 2), 1.until(9).map(_.toDouble)) 59 | val m2 = Vol(ThreeD(2, 2, 2), 9.until(1, -1).map(_.toDouble)) 60 | val result = m.merge(m2)(_ + _) 61 | result must_== Vol(ThreeD(2, 2, 2), Seq.fill(8)(10d)) 62 | } 63 | } 64 | 65 | "transpose" >> { 66 | RowVector(0, 1, 0, 0).T must_== Matrix(4, 1, Seq(0, 1, 0, 0)) 67 | } 68 | 69 | "dot" >> { 70 | val r1 = RowVector(3, 3, 1, 2) 71 | val r2 = RowVector(0, 1, 0, 2) 72 | (r1 dot r2) === 7 73 | } 74 | 75 | "uniform" >> { 76 | val m = Vol.fill(ThreeD(2, 2, 2), 10) 77 | m must_== Vol(2, 2, 2, Seq.fill(8)(10d)) 78 | 79 | } 80 | 81 | "normal" >> { 82 | import glaux.statistics.distributions.normal 83 | val data = RowVector.sampleOf(Row(3), normal(10, 3), 100).toSeq.flatMap(_.seqView) 84 | val avg = data.sum / data.size 85 | val devs = data.map(value ⇒ (value - avg) * (value - avg)) 86 | val std = Math.sqrt(devs.sum / data.size) 87 | avg must beCloseTo(10.0 within 2.significantFigures) 88 | std must beCloseTo(3.0 within 1.significantFigures) 89 | } 90 | 91 | } 92 | 93 | "Generic VolOps " >> { 94 | "map correctly" >> { 95 | val d: TwoD = TwoD(3, 1) 96 | val m: Tensor = Matrix.fill(d, 0.5) 97 | val result = m.map(_ * 2) 98 | result must_== Matrix.fill(d, 1) 99 | } 100 | } 101 | 102 | "RowVector Vols" >> { 103 | "sum works correctly" >> { 104 | RowVector(1, 3, 4, 5).sumAll === 13 105 | } 106 | "* works correctly" >> { 107 | val result: Tensor = RowVector(1, 3, 4, 5) * 3 108 | result must_== (RowVector(3, 9, 12, 15)) 109 | } 110 | 111 | } 112 | 113 | } 114 | -------------------------------------------------------------------------------- /neural-network/src/main/scala/glaux/neuralnetwork/Layer.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork 2 | 3 | import glaux.linearalgebra.{RowVector, Tensor} 4 | 5 | trait Layer { 6 | 7 | type Input <: Tensor 8 | type Output <: Tensor 9 | type InDimension = Input#Dimensionality 10 | type OutDimension = Output#Dimensionality 11 | 12 | type InGradient = Input 13 | type OutGradient = Gradient[Output] 14 | def inDimension: InDimension 15 | def outDimension: OutDimension 16 | 17 | def forward(input: Input, isTraining: Boolean): Output 18 | 19 | } 20 | 21 | case class Gradient[T <: Tensor](data: T, gradient: T) { 22 | assert(data.dimension == gradient.dimension) 23 | } 24 | 25 | trait HiddenLayer extends Layer { 26 | def id: String //should be unique within a network 27 | def backward(input: Input, outGradient: OutGradient): (InGradient, Seq[ParamGradient]) 28 | def updateParams(params: Iterable[LayerParam]): HiddenLayer //This should not change the unique id of the layer 29 | def params: Seq[LayerParam] 30 | } 31 | 32 | object HiddenLayer { 33 | def newId(): String = java.util.UUID.randomUUID.toString 34 | } 35 | 36 | trait LossLayer extends Layer { 37 | type Input = RowVector 38 | type Output = Input 39 | def loss(target: Output, actual: Output): (Loss, InGradient) 40 | } 41 | 42 | case class InputLayer[I <: Tensor](inDimension: I#Dimensionality) extends Layer { 43 | type Input = I 44 | type Output = Input 45 | def outDimension: OutDimension = inDimension 46 | def forward(input: Input, isTraining: Boolean) = input 47 | } 48 | 49 | case class LayerParam(id: String, value: Tensor, regularizationSetting: RegularizationSetting) 50 | case class LayerData[L <: Layer](in: L#Input, out: L#Output, layer: L) 51 | 52 | case class RegularizationSetting(l1DM: DecayMultiplier, l2DM: DecayMultiplier) 53 | -------------------------------------------------------------------------------- /neural-network/src/main/scala/glaux/neuralnetwork/Net.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork 2 | 3 | import glaux.linearalgebra.{RowVector, Tensor} 4 | 5 | trait Net { 6 | type Input <: Tensor 7 | final type Output = LossLayer#Output 8 | def inputLayer: InputLayer[Input] 9 | def hiddenLayers: Seq[HiddenLayer] 10 | def lossLayer: LossLayer 11 | def inputDimension: Input#Dimensionality = inputLayer.inDimension 12 | def outputDimension: Output#Dimensionality = lossLayer.outDimension 13 | def allLayers: Seq[Layer] = inputLayer +: hiddenLayers :+ lossLayer 14 | 15 | //throws assertion exceptions 16 | def validate(): Unit = { 17 | def assertUniqueness[T](seq: Seq[T], message: String): Unit = assert(seq.distinct.size == seq.size, message) 18 | allLayers.reduce { (lastLayer, thisLayer) ⇒ 19 | assert(lastLayer.outDimension == thisLayer.inDimension, s"${lastLayer.getClass.getSimpleName}'s output ${lastLayer.outDimension} doesn't match ${thisLayer.getClass.getSimpleName}'s input ${thisLayer.inDimension}") 20 | thisLayer 21 | } 22 | assertUniqueness(hiddenLayers.map(_.id), "Some hidden layers share the same id") 23 | hiddenLayers.foreach { l ⇒ 24 | assertUniqueness(l.params.map(_.id), "Some layers have params that share the same id") 25 | } 26 | } 27 | 28 | def forward(input: Input): DataFlow = { 29 | val (_, dataFlow) = allLayers.foldLeft[(Tensor, DataFlow)]((input, Vector[LayerData[_]]())) { (pair, layer) ⇒ 30 | val (lastOutput, dataFlow) = pair 31 | val in = lastOutput.asInstanceOf[layer.Input] //cast to this input, hard to have compiler check for dynamically constructed layers. 32 | val out = layer.forward(in, true) 33 | (out, dataFlow :+ LayerData(in, out, layer)) 34 | } 35 | dataFlow 36 | } 37 | 38 | def predict(input: Input): Output = finalOutput(forward(input)) 39 | 40 | def backward(target: Output, dataFlow: DataFlow): (Loss, NetParamGradients) = { 41 | val (loss, lossLayerInGrad) = lossLayer.loss(target, finalOutput(dataFlow)) 42 | val (_, netParamGrads) = hiddenLayers.foldRight[(Tensor, NetParamGradients)]((lossLayerInGrad, Map())) { (layer, pair) ⇒ 43 | val (outGradientValue, accuParamGrads) = pair 44 | val layerData = findData[layer.type](dataFlow, layer) 45 | val (inGradient, layerParamGrads) = layer.backward(layerData.in, Gradient(layerData.out, outGradientValue.asInstanceOf[layer.Output])) 46 | (inGradient, accuParamGrads + (layer → layerParamGrads)) 47 | } 48 | (loss, netParamGrads) 49 | } 50 | 51 | private def finalOutput(dataFlow: DataFlow): Output = dataFlow.last.out.asInstanceOf[Output] 52 | private def findData[L <: Layer](dataFlow: DataFlow, layer: L): LayerData[L] = 53 | dataFlow.find(_.layer == layer).get.asInstanceOf[LayerData[L]] 54 | 55 | } 56 | 57 | object Net { 58 | type Updater[N <: Net] = (N, Iterable[HiddenLayer]) ⇒ N 59 | 60 | case class DefaultNet[InputT <: Tensor](inputLayer: InputLayer[InputT], hiddenLayers: Seq[HiddenLayer], lossLayer: LossLayer) extends Net { 61 | type Input = InputT 62 | validate() 63 | } 64 | 65 | implicit def simpleUpdater[Input <: Tensor]: Updater[DefaultNet[Input]] = (net, newLayers) ⇒ { 66 | net.hiddenLayers.map(_.id).zip(newLayers.map(_.id)).foreach { 67 | case (id1, id2) ⇒ assert(id1 == id2, "update layer cannot change layer ids and sequence") 68 | } 69 | net.copy(hiddenLayers = newLayers.toSeq) 70 | } 71 | 72 | def apply[Input <: Tensor](inputDimension: Input#Dimensionality, hiddenLayers: Seq[HiddenLayer], lossLayer: LossLayer): Net = DefaultNet( 73 | InputLayer[Input](inputDimension), hiddenLayers, lossLayer 74 | ) 75 | } 76 | -------------------------------------------------------------------------------- /neural-network/src/main/scala/glaux/neuralnetwork/ParamGradient.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork 2 | 3 | import glaux.linearalgebra.Tensor 4 | 5 | case class ParamGradient(param: LayerParam, value: Tensor) 6 | 7 | -------------------------------------------------------------------------------- /neural-network/src/main/scala/glaux/neuralnetwork/Rectangle.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork 2 | 3 | import glaux.linearalgebra.Dimension.ThreeD 4 | 5 | case class Rectangle(x: Int, y: Int) { 6 | def *(factor: Double) = Rectangle((x * factor).toInt, (y * factor).toInt) 7 | def /(factor: Double) = *(1 / factor) 8 | def +(toAdd: Int) = Rectangle(x + toAdd, y + toAdd) 9 | def -(toSubtract: Int): Rectangle = this.+(-toSubtract) 10 | def +(p: Rectangle) = Rectangle(x + p.x, y + p.y) 11 | def contains(tx: Int, ty: Int): Boolean = tx >= 0 && tx < x && ty >= 0 && ty < y 12 | } 13 | 14 | object Rectangle { 15 | def planeSize(threeD: ThreeD) = Rectangle(threeD.x, threeD.y) 16 | } 17 | 18 | case class RectangleRange(xs: Range, ys: Range) { 19 | def contains(tx: Int, ty: Int): Boolean = xs.contains(tx) && ys.contains(ty) 20 | } 21 | -------------------------------------------------------------------------------- /neural-network/src/main/scala/glaux/neuralnetwork/layers/Convolution.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.layers 2 | 3 | import glaux.linearalgebra.Dimension.{FourD, ThreeD, TwoD} 4 | import glaux.linearalgebra.{Tensor4, RowVector, Vol, Matrix} 5 | import glaux.neuralnetwork._ 6 | 7 | /** 8 | * 9 | * @param filters a {@code Tensor4} to represent all filters 10 | * @param bias 11 | * @param stride 12 | * @param pad 13 | * @param inputSize 14 | * @param filterRegularization 15 | * @param id 16 | */ 17 | case class Convolution( 18 | filters: Tensor4, 19 | bias: Vol, 20 | stride: Int, 21 | pad: Rectangle, 22 | inputSize: Rectangle, 23 | filterRegularization: RegularizationSetting, 24 | id: String 25 | ) extends HiddenLayer with MovingFilter { 26 | 27 | private val biasRegularization = RegularizationSetting(0, 0) 28 | assert(bias.dimension == ThreeD(1, 1, filters.dimension.f), "bias size matches number of filters") 29 | val filterSize = Rectangle(filters.dimension.x, filters.dimension.y) 30 | val inputDepth = filters.dimension.z 31 | lazy val filtersParam: LayerParam = LayerParam("filters", filters, filterRegularization) 32 | lazy val biasParam: LayerParam = LayerParam("bias", bias, biasRegularization) 33 | 34 | type Output = Vol 35 | type Input = Vol 36 | private lazy val Array(inputXs, inputYs, inputZs) = inDimension.ranges 37 | private lazy val Array(filterXs, filterYs, filterZs, filterFs) = filters.dimension.ranges 38 | private lazy val Array(outXs, outYs, outZs) = outDimension.ranges 39 | 40 | def outDimension: OutDimension = ThreeD(outSize.x, outSize.y, filters.dimension.f) 41 | 42 | def inDimension: InDimension = ThreeD(inputSize.x, inputSize.y, inputDepth) 43 | 44 | def params: Seq[LayerParam] = Seq(filtersParam, biasParam) 45 | 46 | def forward(input: Input, isTraining: Boolean): Output = { 47 | val values = for { 48 | f ← filterFs 49 | offsetYs ← inputPlaneRanges.ys 50 | offsetXs ← inputPlaneRanges.xs 51 | } yield (for (x ← filterXs; y ← filterYs; z ← filterZs) yield { 52 | val (ix, iy) = (x + offsetXs, y + offsetYs) 53 | filters(x, y, z, f) * (if (inPaddedArea(ix, iy)) 0 else input(ix, iy, z)) 54 | }).sum + bias(0, 0, f) 55 | Vol(outDimension, values) 56 | } 57 | 58 | def backward(input: Input, outGradient: OutGradient): (InGradient, Seq[ParamGradient]) = { 59 | def outGradValue(x: Int, filterX: Int, y: Int, filterY: Int, filterF: Int): Double = 60 | mappedOutCoordinate(x, filterX, y, filterY).map { 61 | case (outX, outY) ⇒ outGradient.gradient(outX, outY, filterF) 62 | }.getOrElse(0) 63 | 64 | val inGradValues = for (z ← inputZs; y ← inputYs; x ← inputXs) 65 | yield (for (fx ← filterXs; fy ← filterYs; ff ← filterFs) 66 | yield outGradValue(x, fx, y, fy, ff) * filters(fx, fy, z, ff)).sum 67 | 68 | val inGrad = Vol(inDimension, inGradValues) 69 | 70 | val filtersGradValues = for (ff ← filterFs; z ← filterZs; fy ← filterYs; fx ← filterXs) 71 | yield (for (y ← inputYs; x ← inputXs) 72 | yield outGradValue(x, fx, y, fy, ff) * input(x, y, z)).sum 73 | val filtersGrad = Tensor4(filters.dimension, filtersGradValues) 74 | val biasGrad = Vol( 75 | bias.dimension, 76 | for (f ← outZs) yield (for (x ← outXs; y ← outYs) yield outGradient.gradient(x, y, f)).sum 77 | ) 78 | (inGrad, Seq(ParamGradient(biasParam, biasGrad), ParamGradient(filtersParam, filtersGrad))) 79 | } 80 | 81 | def updateParams(params: Iterable[LayerParam]): HiddenLayer = copy( 82 | filters = params.find(_.id == "filters").get.value.asInstanceOf[Tensor4], 83 | bias = params.find(_.id == "bias").get.value.asInstanceOf[Vol] 84 | ) 85 | 86 | } 87 | 88 | object Convolution { 89 | /** 90 | * 91 | * @param numOfFilters 92 | * @param filterSize 93 | * @param inputDimension 94 | * @param stride 95 | * @param padding whether to use padding to ensure the same output size (the x*y dimension of the vol) 96 | * @return 97 | */ 98 | def apply( 99 | numOfFilters: Int, 100 | filterSize: Rectangle, 101 | inputDimension: ThreeD, 102 | stride: Int = 1, 103 | padding: Boolean = false, 104 | filterRegularization: RegularizationSetting = RegularizationSetting(0, 1) 105 | ): Convolution = { 106 | 107 | val inputPlain = Rectangle(inputDimension.x, inputDimension.y) 108 | val pad: Rectangle = if (padding) (filterSize + (inputPlain * (stride - 1)) - stride) / 2 else Rectangle(0, 0) 109 | Convolution( 110 | Tensor4.normalized(FourD(filterSize.x, filterSize.y, inputDimension.z, numOfFilters), filterSize.x * filterSize.y * inputDimension.z), 111 | Vol.fill(ThreeD(1, 1, numOfFilters), 0), 112 | stride, 113 | pad, 114 | Rectangle(inputDimension.x, inputDimension.y), 115 | filterRegularization, 116 | HiddenLayer.newId() 117 | ) 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /neural-network/src/main/scala/glaux/neuralnetwork/layers/FullyConnected.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.layers 2 | 3 | import glaux.linearalgebra.Dimension.{Row, TwoD} 4 | import glaux.linearalgebra.Tensor.TensorBuilder 5 | import glaux.linearalgebra.{Tensor, RowVector, Matrix, Dimension} 6 | import glaux.neuralnetwork._ 7 | import glaux.neuralnetwork.layers.FullyConnected.{Bias, Filter} 8 | 9 | case class FullyConnected[IT <: Tensor: TensorBuilder]( 10 | filter: Filter, 11 | bias: Bias, 12 | filterRegularization: RegularizationSetting, 13 | inDimension: IT#Dimensionality, 14 | id: String 15 | ) extends HiddenLayer { 16 | private val biasRegularization = RegularizationSetting(0, 0) 17 | lazy val filterParam: LayerParam = LayerParam("filter", filter, filterRegularization) 18 | lazy val biasParam: LayerParam = LayerParam("bias", bias, biasRegularization) 19 | type Output = RowVector 20 | type Input = IT 21 | 22 | val outDimension: OutDimension = Dimension.Row(filter.dimension.y) 23 | 24 | assert(bias.dimension == outDimension, s"bias dimension ${bias.dimension} must match out dimension $outDimension") 25 | 26 | def params = Seq(filterParam, biasParam) 27 | 28 | def updateParams(params: Iterable[LayerParam]): HiddenLayer = { 29 | val f = params.find(_.id == "filter").get.value.asInstanceOf[Matrix] 30 | val b = params.find(_.id == "bias").get.value.asInstanceOf[RowVector] 31 | copy(f, b) 32 | } 33 | 34 | def backward(input: Input, outGradient: OutGradient): (InGradient, Seq[ParamGradient]) = { 35 | val og = outGradient.gradient 36 | val filterGradient: Matrix = input.toRowVector.T ** og 37 | val biasGradient: RowVector = og 38 | val inGradient: Input = toInputFormat(og ** filter.T) 39 | (inGradient, Seq[ParamGradient]( 40 | ParamGradient(filterParam, filterGradient), 41 | ParamGradient(biasParam, biasGradient) 42 | )) 43 | } 44 | 45 | def forward(input: Input, isTraining: Boolean = false): Output = { 46 | assert(input.dimension == inDimension, s"incorrect input dimension ${input.dimension} vs ${inDimension}") 47 | (input.toRowVector ** filter) + bias 48 | } 49 | 50 | def updateFilterBias(filter: Filter, bias: Bias): HiddenLayer = copy(filter, bias) 51 | 52 | private def toInputFormat(vector: RowVector): Input = { 53 | if (vector.dimension == inDimension) vector.asInstanceOf[Input] else (inDimension, vector.seqView) 54 | } 55 | } 56 | 57 | object FullyConnected { 58 | type Filter = Matrix 59 | type Bias = RowVector 60 | 61 | private def bias(numOfNeurons: Int): Bias = RowVector.fill(Row(numOfNeurons), 0) 62 | private def filter(numOfFeatures: Int, numOfNeurons: Int): Filter = Matrix.normalized(TwoD(numOfFeatures, numOfNeurons), numOfFeatures) 63 | 64 | def apply(numOfFeatures: Int, numOfNeurons: Int): FullyConnected[RowVector] = 65 | FullyConnected(filter(numOfFeatures, numOfNeurons), bias(numOfNeurons)) 66 | 67 | def apply[T <: Tensor: TensorBuilder](inputDimension: T#Dimensionality, numOfNeurons: Int): FullyConnected[T] = 68 | FullyConnected[T](filter(inputDimension.totalSize, numOfNeurons), bias(numOfNeurons), RegularizationSetting(0, 1), inputDimension, HiddenLayer.newId()) 69 | 70 | private[glaux] def apply(filter: Filter, bias: Bias): FullyConnected[RowVector] = 71 | FullyConnected[RowVector](filter, bias, RegularizationSetting(0, 1), Row(filter.dimension.x), HiddenLayer.newId()) 72 | 73 | } 74 | -------------------------------------------------------------------------------- /neural-network/src/main/scala/glaux/neuralnetwork/layers/MovingFilter.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.layers 2 | 3 | import glaux.linearalgebra.Dimension.ThreeD 4 | import glaux.linearalgebra.{Vol, Tensor4} 5 | import glaux.neuralnetwork._ 6 | 7 | trait MovingFilter { 8 | 9 | def inputSize: Rectangle 10 | def stride: Int 11 | def pad: Rectangle 12 | def filterSize: Rectangle 13 | 14 | def inputPlaneRanges = RectangleRange( 15 | Range.inclusive(-pad.x, inputSize.x + pad.x - filterSize.x, stride), 16 | Range.inclusive(-pad.y, inputSize.y + pad.y - filterSize.y, stride) 17 | ) 18 | 19 | def outSize: Rectangle = Rectangle( 20 | (inputSize.x + (pad.x * 2) - filterSize.x) / stride + 1, 21 | (inputSize.y + (pad.y * 2) - filterSize.y) / stride + 1 22 | ) 23 | 24 | def inPaddedArea(x: Int, y: Int) = !inputSize.contains(x, y) 25 | 26 | def mappedOutCoordinate(inputX: Int, filterX: Int, inputY: Int, filterY: Int): Option[(Int, Int)] = { 27 | val outX = (inputX - filterX + pad.x) / stride 28 | val outY = (inputY - filterY + pad.y) / stride 29 | if (outSize.contains(outX, outY)) 30 | Some((outX, outY)) 31 | else 32 | None 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /neural-network/src/main/scala/glaux/neuralnetwork/layers/Pool.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.layers 2 | 3 | import glaux.linearalgebra.Dimension.ThreeD 4 | import glaux.linearalgebra.Vol 5 | import glaux.neuralnetwork.{ParamGradient, LayerParam, Rectangle, HiddenLayer} 6 | 7 | case class Pool( 8 | filterSize: Rectangle, 9 | stride: Int, 10 | pad: Rectangle, 11 | inDimension: ThreeD, 12 | id: String 13 | ) extends HiddenLayer with MovingFilter { 14 | type Output = Vol 15 | type Input = Vol 16 | def inputSize = Rectangle.planeSize(inDimension) 17 | def outDimension: OutDimension = ThreeD(outSize.x, outSize.y, inDimension.z) 18 | def params: Seq[LayerParam] = Nil 19 | def updateParams(params: Iterable[LayerParam]): HiddenLayer = this 20 | 21 | private lazy val Array(inputXs, inputYs, inputZs) = inDimension.ranges 22 | private lazy val (filterXs, filterYs) = (0 until filterSize.x, 0 until filterSize.y) 23 | 24 | def forward(input: Vol, isTraining: Boolean): Vol = { 25 | val values = for { 26 | z ← 0 until inDimension.z 27 | offsetY ← inputPlaneRanges.ys 28 | offsetX ← inputPlaneRanges.xs 29 | } yield (for (x ← 0 until filterSize.x; y ← 0 until filterSize.y) yield { 30 | val (ix, iy) = (x + offsetX, y + offsetY) 31 | if (inPaddedArea(ix, iy)) 0 else input(ix, iy, z) 32 | }).max 33 | Vol(outDimension, values) 34 | } 35 | 36 | def backward(input: Vol, outGradient: OutGradient): (InGradient, Seq[ParamGradient]) = { 37 | val inGradValues: Seq[Double] = for (z ← inputZs; y ← inputYs; x ← inputXs) 38 | yield (for (filterX ← filterXs; filterY ← filterYs) yield { 39 | mappedOutCoordinate(x, filterX, y, filterY).map { 40 | case (outX, outY) ⇒ (outGradient.data(outX, outY, z), outGradient.gradient(outX, outY, z)) 41 | } 42 | }).find { 43 | case Some((outValue, outGrad)) ⇒ outValue == input(x, y, z) 44 | case None ⇒ false 45 | }.flatten.map(_._2).getOrElse(0d) 46 | 47 | val inGrad = Vol(inDimension, inGradValues) 48 | (inGrad, Nil) 49 | } 50 | 51 | } 52 | 53 | object Pool { 54 | def apply( 55 | inDimension: ThreeD, 56 | filterSize: Rectangle, 57 | stride: Int, 58 | padding: Boolean = false 59 | ): Pool = { 60 | val pad: Rectangle = if (padding) { 61 | val x = ((stride - (inDimension.x - filterSize.x) % stride) % stride) / 2 62 | val y = ((stride - (inDimension.y - filterSize.y) % stride) % stride) / 2 63 | Rectangle(x, y) 64 | } else Rectangle(0, 0) 65 | 66 | Pool(filterSize, stride, pad, inDimension, HiddenLayer.newId()) 67 | 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /neural-network/src/main/scala/glaux/neuralnetwork/layers/Regression.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.layers 2 | 3 | import glaux.linearalgebra.Dimension 4 | import glaux.neuralnetwork._ 5 | 6 | case class Regression(inDimension: Dimension.Row) extends LossLayer { 7 | def loss(target: Output, actual: Output): (Loss, InGradient) = { 8 | assert(target.dimension == outDimension && actual.dimension == outDimension) 9 | val gradient: Output = actual - target 10 | val losses: Output = (gradient * gradient) * 0.5 11 | (losses.sumAll, gradient) 12 | } 13 | 14 | val outDimension: OutDimension = inDimension 15 | 16 | def forward(input: Input, isTraining: Boolean = false): Output = input 17 | } 18 | 19 | object Regression { 20 | def apply(numOfOutputs: Int): Regression = Regression(Dimension.Row(numOfOutputs)) 21 | } 22 | -------------------------------------------------------------------------------- /neural-network/src/main/scala/glaux/neuralnetwork/layers/Relu.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.layers 2 | 3 | import glaux.linearalgebra.Tensor 4 | import Tensor.TensorBuilder 5 | import Tensor.TensorOps 6 | 7 | import scala.math.max 8 | import glaux.neuralnetwork._ 9 | 10 | case class Relu[DataType <: Tensor: TensorBuilder](dimension: DataType#Dimensionality, id: String = HiddenLayer.newId()) extends HiddenLayer { 11 | 12 | type Output = DataType 13 | type Input = DataType 14 | 15 | def backward(input: Input, outGradient: OutGradient): (InGradient, Seq[ParamGradient]) = { 16 | val inGradient = outGradient.data.merge(outGradient.gradient)((o, g) ⇒ if (o <= 0) 0 else g) 17 | (inGradient, Nil) 18 | } 19 | 20 | def updateParams(params: Iterable[LayerParam]): HiddenLayer = this 21 | 22 | def params: Seq[LayerParam] = Nil 23 | 24 | def outDimension: OutDimension = dimension 25 | 26 | def inDimension: InDimension = dimension 27 | 28 | def forward(input: Input, isTraining: Boolean = false): Output = input.map(max(_, 0)) 29 | 30 | } 31 | -------------------------------------------------------------------------------- /neural-network/src/main/scala/glaux/neuralnetwork/layers/Softmax.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.layers 2 | 3 | import glaux.linearalgebra._ 4 | import Tensor._ 5 | import glaux.neuralnetwork.{Loss, LossLayer} 6 | 7 | case class Softmax(inDimension: Dimension.Row) extends LossLayer { 8 | 9 | def loss(target: RowVector, actual: RowVector): (Loss, InGradient) = { 10 | assert( 11 | target.dimension == actual.dimension && 12 | target.sumAll == 1 && 13 | target.seqView.forall(v ⇒ v == 0 || v == 1), 14 | "target must be same dimension as output with exact one 1 value and the rest 0s" 15 | ) 16 | 17 | val inGradient = actual - target 18 | val missMatch: Double = actual dot target 19 | 20 | val loss = if (missMatch > 0) -Math.log(missMatch) else 99999 21 | (loss, inGradient) 22 | } 23 | 24 | val outDimension: OutDimension = inDimension 25 | 26 | def forward(input: RowVector, isTraining: Boolean = false): RowVector = { 27 | val maxV = input.seqView.max 28 | val exp = input.map(v ⇒ Math.exp(v - maxV)) 29 | val sum = exp.sumAll 30 | exp.map(_ / sum) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /neural-network/src/main/scala/glaux/neuralnetwork/package.scala: -------------------------------------------------------------------------------- 1 | package glaux 2 | 3 | package object neuralnetwork { 4 | type DecayMultiplier = Double 5 | type Loss = Double 6 | type Decay = Double 7 | type DataFlow = Vector[LayerData[_]] 8 | type NetParamGradients = Map[HiddenLayer, Seq[ParamGradient]] 9 | type NetParams = Map[HiddenLayer, Seq[LayerParam]] 10 | } 11 | -------------------------------------------------------------------------------- /neural-network/src/main/scala/glaux/neuralnetwork/trainers/BatchTrainer.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.trainers 2 | 3 | import glaux.neuralnetwork._ 4 | import glaux.neuralnetwork.trainers.BatchTrainer.LossInfo 5 | 6 | trait BatchTrainer[Trainee <: Net, CalculationContext] { 7 | type Output = Trainee#Output 8 | 9 | val build: Net.Updater[Trainee] 10 | 11 | type BatchResult = BatchTrainer.BatchResult[Trainee, CalculationContext] 12 | 13 | def initialCalculationContext(net: Trainee): CalculationContext 14 | 15 | val initialLossInfo = LossInfo(0, 0, 0) 16 | def init(net: Trainee): BatchResult = BatchTrainer.BatchResult(initialLossInfo, net, 0, initialCalculationContext(net)) 17 | 18 | type ScalarOutputInfo = (Double, Int) //(Value, Index) 19 | 20 | //training based on only partial correction about output - a scalar value at an index, this helps us with regression on a single scalar value 21 | def trainBatchWithScalaOutputInfo(lastResult: BatchResult)(batch: Iterable[(lastResult.net.Input, ScalarOutputInfo)]): BatchResult = { 22 | val dataFlowBatch = batch.map { 23 | case (input, scalarOutputInfo) ⇒ { 24 | val dataFlow = lastResult.net.forward(input) 25 | val (targetScalar, index) = scalarOutputInfo 26 | (dataFlow, dataFlow.last.out.update(index, targetScalar)) 27 | } 28 | } 29 | trainBatchWithFullDataFlow(dataFlowBatch, lastResult) 30 | } 31 | 32 | def trainBatch(lastResult: BatchResult)(batch: Iterable[(lastResult.net.Input, Output)]): BatchResult = { 33 | val dataFlowBatch = batch.map { 34 | case (input, output) ⇒ (lastResult.net.forward(input), output) 35 | } 36 | trainBatchWithFullDataFlow(dataFlowBatch, lastResult) 37 | } 38 | 39 | private def trainBatchWithFullDataFlow(batch: Iterable[(DataFlow, Output)], lastResult: BatchResult): BatchResult = { 40 | val net: Trainee = lastResult.net 41 | val pairs = batch.map { 42 | case (dataFlow, target) ⇒ net.backward(target, dataFlow) 43 | } 44 | val batchLoss = pairs.last._1 //todo: confirm on this 45 | val batchSize = pairs.size 46 | val paramsGrads = accumulate(pairs.map(_._2)) 47 | val (newParams, newContext, lossInfo) = calculate(paramsGrads, lastResult, batchLoss, batchSize) 48 | 49 | val newLayers = newParams.map { 50 | case (l, ps) ⇒ l.updateParams(ps) 51 | } 52 | val newLayersSorted = net.hiddenLayers.map { oldLayer ⇒ 53 | newLayers.find(_.id == oldLayer.id).getOrElse(oldLayer) //it is possible that some layer didn't get updated and thus use the old layer instead 54 | } 55 | 56 | val newNet: Trainee = build(net, newLayersSorted) 57 | BatchTrainer.BatchResult(lossInfo, newNet, batchSize, newContext) 58 | } 59 | 60 | def accumulate(netParamGradients: Iterable[NetParamGradients]): NetParamGradients = 61 | netParamGradients.reduce { (npg1, npg2) ⇒ 62 | (npg1, npg2).zipped.map { 63 | case ((layer1, paramGradients1), (layer2, paramGradients2)) ⇒ 64 | assert(layer1 == layer2) //assume sequence of the two NetParamGradients are the same, but assert the match here 65 | val newParamGradient = (paramGradients1, paramGradients2).zipped.map { (paramGradient1, paramGradient2) ⇒ 66 | assert(paramGradient1.param == paramGradient2.param) //assume the sequence remain the same, but assert the match here 67 | paramGradient1.copy(value = paramGradient1.value + paramGradient2.value) 68 | } 69 | (layer1, newParamGradient) 70 | } 71 | } 72 | 73 | def calculate(paramGrads: NetParamGradients, lastIterationResult: BatchResult, loss: Loss, batchSize: Int): (NetParams, CalculationContext, LossInfo) 74 | } 75 | 76 | object BatchTrainer { 77 | case class LossInfo(l1Decay: Loss, l2Decay: Loss, cost: Loss) { 78 | val total: Loss = l1Decay + l2Decay + cost 79 | } 80 | 81 | case class BatchResult[Trainee <: Net, CalculationContext](lossInfo: LossInfo, net: Trainee, batchSize: Int, calcContext: CalculationContext) 82 | 83 | } 84 | 85 | -------------------------------------------------------------------------------- /neural-network/src/main/scala/glaux/neuralnetwork/trainers/SGD.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.trainers 2 | 3 | import glaux.linearalgebra.Tensor 4 | import glaux.neuralnetwork._ 5 | import glaux.neuralnetwork.trainers.BatchTrainer._ 6 | import glaux.neuralnetwork.trainers.MomentumSGD.{Settings, ParamGSum, IterationContext} 7 | import glaux.neuralnetwork.trainers.SGD._ 8 | 9 | abstract class SGD[NT <: Net: Net.Updater, CalculationContext](options: SGDSettings) extends BatchTrainer[NT, CalculationContext] { 10 | type Trainee = NT 11 | 12 | val build: Net.Updater[Trainee] = implicitly[Net.Updater[Trainee]] 13 | 14 | def calculateParamAdjustment(layer: HiddenLayer, param: LayerParam, rawBatchGradient: Tensor, lastContext: CalculationContext): Tensor 15 | 16 | def calculate(netParamGradients: NetParamGradients, lastIterationResult: BatchResult, loss: Loss, batchSize: Int): (NetParams, CalculationContext, LossInfo) = { 17 | val lastContext = lastIterationResult.calcContext 18 | 19 | def calcNewParam(paramGrad: ParamGradient, layer: HiddenLayer): NewParamResult = { 20 | val l1Decay = options.l1Decay * paramGrad.param.regularizationSetting.l1DM 21 | val l2Decay = options.l2Decay * paramGrad.param.regularizationSetting.l2DM 22 | val p2DL: Tensor = (paramGrad.param.value * paramGrad.param.value) * l2Decay / 2 23 | val p1DL: Tensor = paramGrad.param.value.map(Math.abs(_)) * l1Decay 24 | val l2DecayLoss: Loss = p2DL.sumAll 25 | val l1DecayLoss: Loss = p1DL.sumAll 26 | val param = paramGrad.param 27 | val pValue = param.value 28 | val l1grad: Tensor = pValue.map((v: Double) ⇒ if (v > 0) 1 else -1) * l1Decay 29 | val l2grad: Tensor = pValue * l2Decay 30 | val rawBatchGradient: Tensor = (l1grad + l2grad + paramGrad.value) / batchSize 31 | 32 | val paramAdjustment = calculateParamAdjustment(layer, param, rawBatchGradient, lastContext) 33 | val newParam = param.copy(value = pValue + paramAdjustment) 34 | 35 | NewParamResult(newParam, l1DecayLoss, l2DecayLoss, paramAdjustment) 36 | } 37 | 38 | //note here if there is no paramGrad, the layer won't be calculated and will be missing 39 | val results: Results = (for { 40 | (layer, paramGrads) ← netParamGradients.toSeq 41 | paramGrad ← paramGrads 42 | newParamResult = calcNewParam(paramGrad, layer) 43 | } yield (layer, newParamResult)).groupBy(_._1).mapValues(_.map(_._2)) 44 | 45 | val newNetParams: NetParams = results.mapValues(_.map(_.newParam)) 46 | 47 | val newContext = updateContext(lastContext, results) 48 | val l1DecayLoss = results.values.flatten.map(_.l1DecayLoss).sum 49 | val l2DecayLoss = results.values.flatten.map(_.l2DecayLoss).sum 50 | val lossInfo = LossInfo(l1DecayLoss, l2DecayLoss, loss) 51 | (newNetParams, newContext, lossInfo) 52 | } 53 | 54 | def updateContext(lastContext: CalculationContext, results: Results): CalculationContext 55 | } 56 | 57 | object SGD { 58 | case class NewParamResult(newParam: LayerParam, l1DecayLoss: Loss, l2DecayLoss: Loss, adjustment: Tensor) 59 | type Results = Map[HiddenLayer, Seq[NewParamResult]] 60 | 61 | case class SGDSettings(learningRate: Double = 0.01, l1Decay: Decay = 0, l2Decay: Decay = 0) 62 | } 63 | 64 | case class VanillaSGD[NT <: Net: Net.Updater](options: SGDSettings) extends SGD[NT, Unit](options) { 65 | 66 | def initialCalculationContext(net: Trainee): Unit = () 67 | 68 | def updateContext(lastContext: Unit, results: Results) = () 69 | 70 | def calculateParamAdjustment(layer: HiddenLayer, param: LayerParam, rawBatchGradient: Tensor, lastContext: Unit): Tensor = 71 | rawBatchGradient * (-options.learningRate) 72 | 73 | } 74 | 75 | case class MomentumSGD[NT <: Net: Net.Updater](settings: Settings) extends SGD[NT, IterationContext](settings.sgdSettings) { 76 | 77 | def initialCalculationContext(net: Trainee): IterationContext = { 78 | val paramGSums = net.hiddenLayers.flatMap { layer ⇒ 79 | layer.params.map { p ⇒ ParamGSum(layer, p, p.value.fill(0)) 80 | } 81 | } 82 | IterationContext(paramGSums) 83 | } 84 | 85 | def updateContext(lastContext: IterationContext, results: Results): IterationContext = { 86 | val gsums = results.flatMap { 87 | case (layer, paramResults) ⇒ paramResults.map(pr ⇒ ParamGSum(layer, pr.newParam, pr.adjustment)) 88 | }.toSeq 89 | lastContext.copy(gSums = gsums) 90 | } 91 | 92 | def calculateParamAdjustment(layer: HiddenLayer, param: LayerParam, rawBatchGradient: Tensor, lastContext: IterationContext): Tensor = { 93 | val lastGsum = lastContext.gSums.find(gs ⇒ gs.param.id == param.id && gs.layer.id == layer.id).get 94 | (lastGsum.value * settings.momentum) - (rawBatchGradient * settings.sgdSettings.learningRate) 95 | } 96 | } 97 | 98 | object MomentumSGD { 99 | case class ParamGSum(layer: HiddenLayer, param: LayerParam, value: Tensor) 100 | 101 | case class IterationContext(gSums: Seq[ParamGSum]) 102 | 103 | case class Settings(sgdSettings: SGDSettings, momentum: Double) 104 | 105 | } 106 | -------------------------------------------------------------------------------- /neural-network/src/test/scala/glaux/neuralnetwork/integration/kaggle/OttoSpecs.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.integration.kaggle 2 | 3 | import glaux.linearalgebra.Dimension.{TwoD, Row} 4 | import glaux.linearalgebra.{Matrix, RowVector} 5 | import glaux.neuralnetwork.InputLayer 6 | import glaux.neuralnetwork.Net.DefaultNet 7 | import glaux.neuralnetwork.layers.{Relu, Softmax, FullyConnected} 8 | import glaux.neuralnetwork.trainers.MomentumSGD.Settings 9 | import glaux.neuralnetwork.trainers.{SGD, MomentumSGD} 10 | import SGD.SGDSettings 11 | import org.specs2.mutable.Specification 12 | 13 | import scala.io.Source 14 | import scala.util.Random 15 | 16 | class OttoSpecs extends Specification { 17 | val runIntegration = false //todo make this configurable 18 | if (runIntegration) { 19 | "integration test with Otto data" >> { 20 | val source = Source.fromURL(getClass.getResource("/kaggle/train.csv")) 21 | val pairs = source.getLines().drop(1).map { line ⇒ 22 | val values = line.split(",") 23 | val inputValues = values.drop(1).dropRight(1).map(_.toDouble) 24 | val input = RowVector(inputValues: _*) 25 | val target = values.takeRight(1).head.replace("Class_", "").toInt 26 | (input, target) 27 | }.toSeq 28 | 29 | val (trainning, test) = Random.shuffle(pairs).splitAt(50000) 30 | 31 | "read resource correctly" >> { 32 | pairs.length === 61878 33 | val allTargets = pairs.map(_._2) 34 | val allInputs = pairs.map(_._1) 35 | allTargets.distinct must contain(exactly(1, 2, 3, 4, 5, 6, 7, 8, 9)) 36 | allInputs.map(_.dimension.totalSize).distinct must be_==(Seq(93)) 37 | } 38 | 39 | val numOfFeatures = 93 40 | val numOfMidNeurons = 45 41 | val numOfClass = 9 42 | val dim: Row = Row(numOfFeatures) 43 | val inputLayer = InputLayer[RowVector](dim) 44 | import glaux.statistics.distributions.normal 45 | val fc1 = FullyConnected(Matrix.sampleOf(TwoD(numOfFeatures, numOfMidNeurons), normal(0, 2)), RowVector.fill(Row(numOfMidNeurons), 0.1)) 46 | val relu = Relu[RowVector](Row(numOfMidNeurons)) 47 | val fc2 = FullyConnected(Matrix.sampleOf(TwoD(numOfMidNeurons, numOfClass), normal(0, 2)), RowVector.fill(Row(numOfClass), 0)) 48 | val lossLayer = Softmax(Row(numOfClass)) 49 | val initNet: DefaultNet[RowVector] = DefaultNet(inputLayer, Seq(fc1, relu, fc2), lossLayer) 50 | 51 | val trainer = MomentumSGD[DefaultNet[RowVector]](Settings(SGDSettings(learningRate = 0.001), 0.9)) 52 | val initResult = trainer.init(initNet) 53 | 54 | val batchSize: Int = 20 55 | val batches = trainning.grouped(batchSize) //two batches first 56 | 57 | def classificationVector(classification: Int, numOfClassification: Int): RowVector = { 58 | val seqValues = (Seq.fill(classification - 1)(0) ++ Seq(1) ++ Seq.fill(9 - classification)(0)).map(_.toDouble) 59 | RowVector(seqValues: _*) 60 | } 61 | var track = 0 62 | val result = batches.foldLeft(initResult) { (lastResult, batch) ⇒ 63 | val processedBatch = batch.map { 64 | case (input, target) ⇒ (input, classificationVector(target, 9)) 65 | } 66 | track += 1 67 | println(s"training batch $track / ${trainning.length / batchSize}") 68 | println(s"last lost was ${lastResult.lossInfo.cost}") 69 | trainer.trainBatch(lastResult)(processedBatch) 70 | } 71 | 72 | "reach 65% correct rate on test data" >> { 73 | def classification(vector: RowVector): Int = { 74 | val max = vector.seqView.max 75 | val maximized = vector.map((v: Double) ⇒ if (v < max) 0 else 1) 76 | (maximized dot RowVector(1, 2, 3, 4, 5, 6, 7, 8, 9)).toInt 77 | } 78 | val predictions: Iterable[Boolean] = test.map { 79 | case (input, target: Int) ⇒ 80 | val prediction = classification(result.net.predict(input)) 81 | prediction == target 82 | } 83 | 84 | predictions.count((b: Boolean) ⇒ b).toDouble / test.length must be_>(0.65) 85 | } 86 | 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /neural-network/src/test/scala/glaux/neuralnetwork/layers/ConvolutionSpec.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.layers 2 | 3 | import glaux.linearalgebra._ 4 | import Dimension._ 5 | import glaux.neuralnetwork.{Gradient, Rectangle} 6 | import org.specs2.mutable.Specification 7 | class ConvolutionSpec extends Specification { 8 | "creation" >> { 9 | "without padding" >> { 10 | "reduced out dimension" >> { 11 | val inputDim = ThreeD(4, 4, 2) 12 | val layer = Convolution(3, Rectangle(3, 3), inputDim, 1, false) 13 | layer.outDimension === ThreeD(2, 2, 3) 14 | } 15 | } 16 | "with padding" >> { 17 | "same output dimension as input" >> { 18 | val inputDim = ThreeD(4, 4, 2) 19 | val layer = Convolution(3, Rectangle(3, 3), inputDim, 1, true) 20 | layer.outDimension === ThreeD(4, 4, 3) 21 | } 22 | } 23 | 24 | } 25 | 26 | "backward/forward" >> { 27 | val inputDim = ThreeD(4, 4, 2) 28 | val layer = Convolution(3, Rectangle(3, 3), inputDim, 1, true).copy( 29 | filters = Tensor4(3, 3, 2, 3, Seq(0.3, 0.1, -0.1, -0.1, 0.3, 0.1, 0.3, 0.4, -0.1, -0.4, 0.2, -0.03, 0.2, 0.4, 0.3, 0.1, 0.2, 0.1, 0, 0.2, -0.2, -0.1, -0.2, -0.1, -0.1, 0.2, 0.3, -0.2, 0.2, 0.1, 0.3, -0.3, 0.3, -0.1, 0.1, -0.3, -0.4, -0.2, -0.2, -0.1, -0.1, 0.1, 0.3, 0.3, -0.1, -0.5, -0.1, 0.1, 0.1, -0.1, 0.1, 0.1, -0.1, 0.1)) 30 | ) 31 | val input = Vol(inputDim, Seq(1, 2, 5, 3, 9, 1, 5, 1, 4, 11, 7, 3, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 8, 9, 1)) 32 | 33 | "forward" >> { 34 | "without bias" >> { 35 | val output = layer.forward(input, false) 36 | output === Vol(4, 4, 3, Seq(5, 4.9, 5.1, 3.2, 4.37, 5.87, 9.17, 5.6, 7.27, 11.27, 5.97, 4, 4.17, 7.27, 9.27, 4.6, 1.5, -0.2, -0.3, -1.4, 2.1, 1.8, 0.5, -0.2, -0.8, -5.8, -1.4, -1.8, -0.3, 0.6, 0.5, 2.7, 2.7, 2.9, 1.5, 1, -1.3, 1.2, 1.9, -0.8, 1.1, -3.8, -3.0, -2.4, -2.8, -6.1, -7.0, -3.4)) 37 | } 38 | 39 | "with bias" >> { 40 | val output = layer.copy(bias = Vol(1, 1, 3, Seq(0, 1, 2))).forward(input, false) 41 | output === Vol(4, 4, 3, Seq(5, 4.9, 5.1, 3.2, 4.37, 5.87, 9.17, 5.6, 7.27, 11.27, 5.97, 4, 4.17, 7.27, 9.27, 4.6, 2.5, 0.8, 0.7, -0.4, 3.1, 2.8, 1.5, 0.8, 0.2, -4.8, -0.4, -0.8, 0.7, 1.6, 1.5, 3.7, 4.7, 4.9, 3.5, 3, 0.7, 3.2, 3.9, 1.2, 3.1, -1.8, -1.0, -0.4, -0.8, -4.1, -5.0, -1.4)) 42 | } 43 | } 44 | 45 | "backward" >> { 46 | val output = layer.forward(input, false) 47 | 48 | val outGrad = Vol(4, 4, 3, Seq(0, 0, 0, 0, 0.9, 0, 3, 0, 0, 0, 0, -1, 0, 0, 0.2, 0.3, 0.4, 0.1, 0, 0, 0, 2, 0.3, 0.09, 0, 0, 0, -0.3, 0.4, 0.7, 0.1, 0.5, 0.2, -0.1, -0.01, 0.3, 1, 0, 0, -1, 0, 0, 3, 0, 0, 0, 0, 0)) 49 | val (inGrad, paraGrads) = layer.backward(input, Gradient(output, outGrad)) 50 | 51 | "in gradient" >> { 52 | val expectedInGrad = Vol(inputDim, Seq(-0.21, 0.981, 0.311, -0.173, 0.07, -1.653, -0.042, -0.317, 0.54, 0.90, 1.371, -0.342, -0.15, 0.69, 0.52, -0.76, -0.44, -0.668, 1.393, 0.027, 0.86, -1.121, 1.788, 1.072, -0.18, 1.04, -0.759, 0.473, 0.09, 0.28, 0.1, 0.13)) 53 | inGrad === expectedInGrad 54 | } 55 | 56 | "bias gradient" >> { 57 | paraGrads.find(_.param.id == "bias").get.value === Vol(1, 1, 3, Seq(3.4, 4.29, 3.39)) 58 | } 59 | 60 | "filter gradient" >> { 61 | val expected = Tensor4(3, 3, 2, 3, Seq(5.3, 17.2, 11.4, -3.5, 20.6, 4.1, 32, 23.6, 18.9, 2.5, 3.4, 4.1, 6.3, 5, 4.1, -6, 2.9, 3.9, 8.95, 16.97, 20.5, 21.55, 6.99, 12.8, 12.53, 27.77, 15.8, 3.39, 3.79, 3.5, 8.19, 9.99, 12.4, -0.21, 2.59, 2.8, -2, 13, 5, 29.38, 29.85, 9.87, -3.41, 5.95, 13.69, 2, 3, 4, 2.19, 3.39, 4.09, 23.19, 27.39, 4.09)) 62 | paraGrads.find(_.param.id == "filters").get.value === expected 63 | } 64 | } 65 | } 66 | 67 | } 68 | 69 | /* convnetjs test code 70 | 71 | 72 | function linear(vol){ 73 | var rv = [] 74 | for(z = 0; z < vol.depth ; z ++ ) { 75 | for(y = 0; y < vol.sy ; y ++ ) { 76 | for(x = 0; x < vol.sx ; x ++ ) { 77 | rv.push(vol.get(x, y, z)); 78 | } 79 | } 80 | } 81 | return rv; 82 | } 83 | 84 | function linearGrad(vol){ 85 | var rv = [] 86 | for(z = 0; z < vol.depth ; z ++ ) { 87 | for(y = 0; y < vol.sy ; y ++ ) { 88 | for(x = 0; x < vol.sx ; x ++ ) { 89 | rv.push(vol.get_grad(x, y, z)); 90 | } 91 | } 92 | } 93 | return rv; 94 | } 95 | 96 | 97 | var l = new convnetjs.ConvLayer({in_sx: 4, in_sy: 4, in_depth: 2, filters: 3, sx: 3, sy: 3, stride: 1, pad: 1}); 98 | var input = new convnetjs.Vol(4, 4, 2, 1) 99 | input.w = [1, 1, 2, 1, 5, 1, 3, 1, 9, 1, 1, 1, 5, 1, 1, 1, 4, 1, 11, 1, 7, 1, 3, 1, 6, 1, 1, 8, 1, 9, 1, 1] 100 | linear(input) 101 | 102 | l.filters[0].w = [0.3, -0.4, 0.1, 0.2, -0.1, -0.03, -0.1, 0.2, 0.3, 0.4, 0.1, 0.3, 0.3, 0.1, 0.4, 0.2, -0.1, 0.1] 103 | l.filters[1].w = [0, -0.2, 0.2, 0.2, -0.2, 0.1, -0.1, 0.3, -0.2, -0.3, -0.1, 0.3, -0.1, -0.1, 0.2, 0.1, 0.3, -0.3] 104 | l.filters[2].w = [-0.4, -0.5, -0.2, -0.1, -0.2, 0.1, -0.1, 0.1, -0.1, -0.1, 0.1, 0.1, 0.3, 0.1, 0.3, -0.1, -0.1, 0.1] 105 | 106 | var f = [] //filter in linear array 107 | for(d = 0; d < 3 ; d ++ ) { 108 | f = f.concat(linear(l.filters[d])) 109 | } 110 | f 111 | 112 | var output = l.forward(input); 113 | 114 | linear(output) 115 | 116 | 117 | l.out_act.dw = [0, 0.4, 0.2, 0, 0.1, -0.1, 0, 0, -0.01, 0, 0, 0.3, 0.9, 0, 1, 0, 2, 0, 3, 0.3, 0, 0, 0.09, -1.0, 0, 0, 0, 0, 0, 0, 0, 0, 3, -1, -0.3, 0, 0, 0.4, 0, 0, 0.7, 0, 0.2, 0.1, 0, 0.3, 0.5, 0] 118 | 119 | linearGrad(l.out_act) 120 | 121 | l.backward() 122 | 123 | linearGrad(l.in_act) 124 | 125 | l.biases.dw 126 | 127 | var fd = [] //filter gradient in linear array 128 | for(d = 0; d < 3 ; d ++ ) { 129 | fd = fd.concat(linearGrad(l.filters[d])) 130 | } 131 | */ 132 | -------------------------------------------------------------------------------- /neural-network/src/test/scala/glaux/neuralnetwork/layers/FullyConnectedSpec.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.layers 2 | 3 | import glaux.linearalgebra.Dimension.{TwoD, Row, ThreeD} 4 | import glaux.linearalgebra.{Vol, Matrix, RowVector} 5 | import glaux.neuralnetwork.{HiddenLayer, RegularizationSetting, Gradient} 6 | import org.specs2.mutable.Specification 7 | 8 | class FullyConnectedSpec extends Specification { 9 | 10 | "RowVector as input" >> { 11 | val fc = FullyConnected(Matrix(3, 2, Seq(1d, 2d, 3d, 4d, 5d, 6d)), RowVector(100, 200)) 12 | val input = RowVector(1, 1, 1) 13 | "forward correctly" >> { 14 | val output = fc.forward(input) 15 | output must_== RowVector(106, 215) 16 | } 17 | 18 | "backward returns correctly" >> { 19 | val (inGraident, paramGradients) = fc.backward(input, Gradient(RowVector(106, 215), RowVector(3, 4))) 20 | "the input gradient" >> { 21 | inGraident must_== RowVector(19, 26, 33) 22 | } 23 | 24 | "the filter gradient" >> { 25 | val fg = paramGradients.find(_.param.id == "filter").get 26 | fg.value must_== Matrix(3, 2, Seq(3, 3, 3, 4, 4, 4)) 27 | } 28 | 29 | "the bias gradient" >> { 30 | val bg = paramGradients.find(_.param.id == "bias").get 31 | bg.value must_== RowVector(3, 4) 32 | } 33 | } 34 | 35 | } 36 | 37 | "Vol as input" >> { 38 | val fc = FullyConnected[Vol](ThreeD(2, 2, 2), 2).copy(filter = Matrix.fill(TwoD(8, 2), 1)) 39 | val input = Vol(2, 2, 2, Seq(1, 2, 3, 4, 5, 6, 7, 8)) 40 | "forward correctly" >> { 41 | val output = fc.forward(input) 42 | output must_== RowVector(36, 36) 43 | } 44 | 45 | "backward returns correctly" >> { 46 | val (inGraident, paramGradients) = fc.backward(input, Gradient(RowVector(36, 36), RowVector(1, 2))) 47 | "the input gradient" >> { 48 | inGraident must_== Vol.fill(ThreeD(2, 2, 2), 3) 49 | } 50 | } 51 | } 52 | 53 | "initialize with random normalized filters" >> { 54 | val created = FullyConnected(100, 100) 55 | val weights = created.filter.seqView 56 | val within3Std = weights.count(w ⇒ Math.abs(w) < 0.3) 57 | (within3Std.toDouble / weights.length) must be_>(0.9) 58 | 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /neural-network/src/test/scala/glaux/neuralnetwork/layers/PoolSpec.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.layers 2 | 3 | import glaux.linearalgebra.Dimension.ThreeD 4 | import glaux.linearalgebra.Vol 5 | import glaux.neuralnetwork.{Gradient, Rectangle} 6 | import org.specs2.mutable.Specification 7 | 8 | class PoolSpec extends Specification { 9 | "creation" >> { 10 | "padding" >> { 11 | Pool( 12 | ThreeD(4, 4, 1), 13 | Rectangle(3, 3), 14 | 3, 15 | true 16 | ).pad === Rectangle(1, 1) 17 | 18 | Pool( 19 | ThreeD(4, 4, 1), 20 | Rectangle(2, 2), 21 | 2, 22 | true 23 | ).pad === Rectangle(0, 0) 24 | 25 | } 26 | 27 | } 28 | 29 | val expectedOutput = Vol(2, 2, 1, Seq(6, 8, 9, 7)) 30 | val input = Vol(4, 4, 1, Seq(1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7)) 31 | 32 | "forward" >> { 33 | val l = Pool(ThreeD(4, 4, 1), Rectangle(2, 2), 2) 34 | l.forward(input, false) === expectedOutput 35 | } 36 | 37 | "backward" >> { 38 | val expected = Vol(4, 4, 1, Seq(0, 0, 0, 0, 0, 1, 0, 4, -1, 0, 0, 0, 0, 0, 0, 2)) 39 | val l = Pool(ThreeD(4, 4, 1), Rectangle(2, 2), 2) 40 | 41 | val (inGrad, _) = l.backward(input, Gradient(expectedOutput, Vol(2, 2, 1, Seq(1, 4, -1, 2)))) 42 | inGrad === Vol(4, 4, 1, Seq(0, 0, 0, 0, 0, 1, 0, 4, -1, 0, 0, 0, 0, 0, 0, 2)) 43 | } 44 | 45 | } 46 | 47 | /* convnet js test code 48 | 49 | function linear(vol){ 50 | var rv = [] 51 | for(z = 0; z < vol.depth ; z ++ ) { 52 | for(y = 0; y < vol.sy ; y ++ ) { 53 | for(x = 0; x < vol.sx ; x ++ ) { 54 | rv.push(vol.get(x, y, z)); 55 | } 56 | } 57 | } 58 | return rv; 59 | } 60 | 61 | function linearGrad(vol){ 62 | var rv = [] 63 | for(z = 0; z < vol.depth ; z ++ ) { 64 | for(y = 0; y < vol.sy ; y ++ ) { 65 | for(x = 0; x < vol.sx ; x ++ ) { 66 | rv.push(vol.get_grad(x, y, z)); 67 | } 68 | } 69 | } 70 | return rv; 71 | } 72 | 73 | var l = new convnetjs.PoolLayer({in_sx: 4, in_sy: 4, in_depth: 1, stride: 2, sx: 2, sy: 2, pad: 0}); 74 | var input = new convnetjs.Vol(4, 4, 1, 1) 75 | input.w = [1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6, 7] 76 | linear(input) 77 | var output = l.forward(input) 78 | 79 | output.w 80 | 81 | 82 | output.dw = [1,4,-1,2] 83 | l.backward() 84 | l.in_act.dw 85 | */ 86 | -------------------------------------------------------------------------------- /neural-network/src/test/scala/glaux/neuralnetwork/layers/RegressSpec.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.layers 2 | 3 | import glaux.linearalgebra.{RowVector, Dimension} 4 | import org.specs2.mutable.Specification 5 | 6 | class RegressSpec extends Specification { 7 | "RegressionLayer" >> { 8 | "Dimension" >> { 9 | Regression(2).inDimension must_== Dimension.Row(2) 10 | Regression(2).outDimension must_== Dimension.Row(2) 11 | } 12 | 13 | "forward" >> { 14 | Regression(2).forward(RowVector(1, 2)) must_== RowVector(1, 2) 15 | } 16 | 17 | "Loss" >> { 18 | val rl = Regression(3) 19 | val (loss, inGradient) = rl.loss(RowVector(2, 3, 4), RowVector(1, 2, 3)) 20 | loss must_== 1.5 21 | inGradient must_== RowVector(-1, -1, -1) 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /neural-network/src/test/scala/glaux/neuralnetwork/layers/ReluSpec.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.layers 2 | 3 | import glaux.linearalgebra.{RowVector, Dimension} 4 | import glaux.neuralnetwork.{Gradient, ParamGradient} 5 | import org.specs2.mutable.Specification 6 | 7 | class ReluSpec extends Specification { 8 | 9 | val rl = Relu[RowVector](Dimension.Row(3)) 10 | val input = RowVector(1, -2, 3) 11 | 12 | "forward correctly" >> { 13 | val output = rl.forward(input) 14 | output must_== RowVector(1, 0, 3) 15 | } 16 | 17 | "backward returns correctly" >> { 18 | val (inGraident, paramGradients: Seq[_]) = rl.backward(input, Gradient(RowVector(1, 0, 3), RowVector(3, 4, 6))) 19 | "the input gradient" >> { 20 | inGraident must_== RowVector(3, 0, 6) 21 | } 22 | 23 | "no parameters" >> { 24 | paramGradients.isEmpty must beTrue 25 | } 26 | 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /neural-network/src/test/scala/glaux/neuralnetwork/layers/SoftmaxSpec.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.layers 2 | 3 | import glaux.linearalgebra.Dimension.Row 4 | import glaux.linearalgebra.{RowVector, Matrix} 5 | import org.specs2.mutable.Specification 6 | 7 | class SoftmaxSpec extends Specification { 8 | val layer = Softmax(Row(3)) 9 | val input = RowVector(1, 3, 2) 10 | val expectedOutput = RowVector(0.09003057, 0.66524096, 0.24472847) 11 | "forward correctly" >> { 12 | val output = layer.forward(input) 13 | output must_== expectedOutput 14 | } 15 | 16 | "loss returns correctly" >> { 17 | val (loss, inGradient) = layer.loss(RowVector(1, 0, 0), expectedOutput) 18 | loss must beCloseTo(2.40760596 within 6.significantFigures) 19 | inGradient must_== RowVector(-0.909969427, 0.665240956, 0.24472847) 20 | } 21 | 22 | "loss returns correctly for incorrect guess" >> { 23 | val (loss, inGradient) = layer.loss(RowVector(0, 1, 0), expectedOutput) 24 | loss must beCloseTo(0.40760596 within 6.significantFigures) 25 | inGradient must_== RowVector(0.09003057, -0.33475904, 0.24472847) 26 | } 27 | 28 | "loss returns correctly for complete missmatch" >> { 29 | val (loss, inGradient) = layer.loss(RowVector(1, 0, 0), RowVector(0, 1, 0)) 30 | loss must be_>(1000d) 31 | inGradient must_== RowVector(-1, 1, 0) 32 | } 33 | } 34 | 35 | /** 36 | * * 37 | * 38 | * 39 | * var l = new convnetjs.SoftmaxLayer({in_sx:1, in_sy: 1, in_depth: 3}) 40 | * var input = new convnetjs.Vol([1,3,2]) 41 | * l.forward(input) 42 | * l.backward(1) 43 | * l.backward(2) 44 | */ 45 | 46 | -------------------------------------------------------------------------------- /neural-network/src/test/scala/glaux/neuralnetwork/trainers/MomentumSGDTrainerSpec.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.trainers 2 | 3 | import glaux.linearalgebra.Dimension.{Row, TwoD} 4 | import glaux.linearalgebra.{Dimension, Matrix, RowVector} 5 | import glaux.neuralnetwork.InputLayer 6 | import glaux.neuralnetwork.Net.DefaultNet 7 | import glaux.neuralnetwork.layers.{FullyConnected, Regression} 8 | import glaux.neuralnetwork.trainers.MomentumSGD.Settings 9 | import glaux.neuralnetwork.trainers.SGD.SGDSettings 10 | import glaux.statistics 11 | import org.specs2.mutable.Specification 12 | 13 | class MomentumSGDTrainerSpec extends Specification { 14 | 15 | val dim: Row = Row(3) 16 | val inputLayer = InputLayer[RowVector](dim) 17 | val hiddenLayer = FullyConnected(Matrix(3, 1, Seq(0.5, -0.7, 1.5)), RowVector(0)) 18 | val lossLayer = Regression(1) 19 | val initNet: DefaultNet[RowVector] = DefaultNet(inputLayer, Seq(hiddenLayer), lossLayer) 20 | 21 | val trainer = MomentumSGD[DefaultNet[RowVector]](Settings(SGDSettings(learningRate = 0.01), 0.9)) 22 | val initResult = trainer.init(initNet) 23 | 24 | "init context" >> { 25 | initResult.calcContext.gSums.size === hiddenLayer.params.size 26 | initResult.calcContext.gSums.find(gs ⇒ gs.param.id == hiddenLayer.params.head.id && gs.layer == hiddenLayer) must beSome[MomentumSGD.ParamGSum] 27 | } 28 | 29 | "net consistent with convnetjs" >> { 30 | val df = initNet.forward(RowVector(3, 2, 1)) 31 | df.last.out must_== RowVector(1.6) 32 | } 33 | 34 | "trainer consistent with convnetjs" >> { 35 | val result = trainer.trainBatch(initResult)(Seq((RowVector(3, 2, 1), RowVector(7)))) 36 | result.lossInfo.cost must beCloseTo(14.58 within 4.significantFigures) 37 | 38 | result.calcContext.gSums.head.value must_== Matrix(3, 1, Seq(0.162, 0.108, 0.054)) 39 | result.calcContext.gSums.last.value must_== RowVector(0.054) 40 | 41 | val result2 = trainer.trainBatch(result)(Seq((RowVector(4, 4, 5), RowVector(14)))) 42 | result2.lossInfo.cost must beCloseTo(17.381408 within 4.significantFigures) 43 | 44 | val result3 = trainer.trainBatch(result2)(Seq((RowVector(-1, 1, 3), RowVector(4)))) 45 | result3.lossInfo.cost must beCloseTo(0.15188 within 4.significantFigures) 46 | } 47 | 48 | val dist = statistics.distributions.normal(0, 3) 49 | val noise = statistics.distributions.normal(0, 0.001) 50 | def randomSample(): (initNet.Input, initNet.Output) = { 51 | val input = RowVector.sampleOf(dim, dist) 52 | val output = RowVector(input.sumAll + 1 + noise.sample) 53 | (input, output) 54 | } 55 | 56 | "train summation" >> { 57 | val batches = 0.until(600).map(_ ⇒ 1.until(3).map(_ ⇒ randomSample())) 58 | val finalResult = batches.foldLeft(initResult) { (lastResult, batch) ⇒ 59 | trainer.trainBatch(lastResult)(batch) 60 | } 61 | 62 | val result = finalResult.net.predict(RowVector(2, 3, 4)) 63 | result.dimension.size === 1 64 | result(0) must beCloseTo(10.0 within 2.significantFigures) 65 | } 66 | 67 | } 68 | 69 | /** 70 | * convnetjs test code 71 | * 72 | * var layer_defs = []; 73 | * layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:3}); 74 | * layer_defs.push({type:'regression', num_neurons: 1}); 75 | * 76 | * var net = new convnetjs.Net(); 77 | * net.makeLayers(layer_defs); 78 | * var trainer = new convnetjs.Trainer(net, {method: 'sgd', learning_rate: 0.01, 79 | * l2_decay: 0, momentum: 0.9, batch_size: 1, 80 | * l1_decay: 1}); 81 | * net.layers[1].filters[0].w = [0.5,-0.7,1.5] 82 | * 83 | * trainer.train(new convnetjs.Vol([3,2,1]), 7) 84 | * trainer.train(new convnetjs.Vol([4,4,5]), 14) 85 | * trainer.train(new convnetjs.Vol([-1,1,3]), 4) 86 | */ 87 | -------------------------------------------------------------------------------- /neural-network/src/test/scala/glaux/neuralnetwork/trainers/VanillaSGDTrainerSpec.scala: -------------------------------------------------------------------------------- 1 | package glaux.neuralnetwork.trainers 2 | 3 | import glaux.linearalgebra.{Matrix, RowVector, Dimension} 4 | import Dimension.{TwoD, Row} 5 | import glaux.neuralnetwork.Net.DefaultNet 6 | import glaux.neuralnetwork.layers.{Regression, FullyConnected} 7 | import glaux.neuralnetwork.{InputLayer, Net} 8 | import glaux.statistics 9 | import org.specs2.mutable.Specification 10 | import glaux.neuralnetwork.trainers.SGD.SGDSettings 11 | 12 | class VanillaSGDTrainerSpec extends Specification { 13 | val dim: Row = Row(3) 14 | val inputLayer = InputLayer[RowVector](Row(3)) 15 | val hiddenLayer = FullyConnected(Matrix(3, 1, Seq(0.5, -0.7, 1.5)), RowVector(0)) 16 | val lossLayer = Regression(1) 17 | val initNet: DefaultNet[RowVector] = DefaultNet(inputLayer, Seq(hiddenLayer), lossLayer) 18 | 19 | val trainer = VanillaSGD[DefaultNet[RowVector]](SGDSettings(learningRate = 0.05)) 20 | val initResult = trainer.init(initNet) 21 | 22 | val dist = statistics.distributions.normal(0, 3) 23 | val noise = statistics.distributions.normal(0, 0.01) 24 | def randomSample(): (initNet.Input, initNet.Output) = { 25 | val input = RowVector.sampleOf(dim, dist) 26 | val output = RowVector(input.sumAll + 1 + noise.sample) 27 | (input, output) 28 | } 29 | 30 | "net consistent with convnetjs" >> { 31 | val df = initNet.forward(RowVector(3, 2, 1)) 32 | df.last.out must_== RowVector(1.6) 33 | } 34 | 35 | "trainer consistent with convnetjs" >> { 36 | val result = trainer.trainBatch(initResult)(Seq((RowVector(3, 2, 1), RowVector(5)))) 37 | result.lossInfo.cost must beCloseTo(5.78 within 4.significantFigures) 38 | } 39 | 40 | "train summation" >> { 41 | val batches = 0.until(50).map(_ ⇒ 1.until(3).map(_ ⇒ randomSample())) 42 | val finalResult = batches.foldLeft(initResult) { (lastResult, batch) ⇒ 43 | trainer.trainBatch(lastResult)(batch) 44 | } 45 | 46 | val result = finalResult.net.predict(RowVector(2, 3, 4)) 47 | result.dimension.size === 1 48 | result(0) must beCloseTo(10.0 within 2.significantFigures) 49 | } 50 | 51 | } 52 | 53 | /* convnetjs test code 54 | 55 | var layer_defs = []; 56 | layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:3}); 57 | layer_defs.push({type:'regression', num_neurons: 1}); 58 | 59 | var net = new convnetjs.Net(); 60 | net.makeLayers(layer_defs); 61 | var trainer = new convnetjs.Trainer(net, {method: 'sgd', learning_rate: 0.05, 62 | l2_decay: 0, momentum: 0, batch_size: 1, 63 | l1_decay: 1}); 64 | net.layers[1].filters[0].w = [0.5,-0.7,1.5] 65 | //verify net output 66 | net.forward(new convnetjs.Vol([3,2,1])); 67 | trainer.train(new convnetjs.Vol([3,2,1]), 5) 68 | */ 69 | -------------------------------------------------------------------------------- /persistence-mongodb/src/main/scala/glaux/persistence/mongodb/AgentSettingsRepo.scala: -------------------------------------------------------------------------------- 1 | package glaux.persistence.mongodb 2 | 3 | import glaux.interfaces.api.domain.{AgentName, AgentSettings} 4 | import glaux.interfaces.api.persistence.{AgentSettingsPersistence, Persistence} 5 | import reactivemongo.api.collections.bson.BSONCollection 6 | import reactivemongo.api.indexes.{IndexType, Index} 7 | import reactivemongo.bson.BSONDocument 8 | import scala.concurrent.ExecutionContext.Implicits.global 9 | import InterfaceHandlers._ 10 | 11 | import scala.concurrent.Future 12 | 13 | case class AgentSettingsRepoImpl(collection: BSONCollection) extends Persistence[AgentSettings, AgentName] with Repository { 14 | 15 | collection.indexesManager.ensure(Index(Seq(("name", IndexType.Ascending)), name = Some("agentTypeName"))) 16 | 17 | def get(name: AgentName): Future[Option[AgentSettings]] = collection.find(BSONDocument("name" → name)).one[AgentSettings] 18 | 19 | def upsert(settings: AgentSettings): Future[Unit] = 20 | collection.findAndUpdate( 21 | selector = BSONDocument("name" → settings.name), 22 | update = settings, 23 | upsert = true 24 | ) 25 | 26 | } 27 | 28 | object AgentSettingsRepo { 29 | 30 | def apply(): AgentSettingsPersistence = { 31 | AgentSettingsRepoImpl(Repository.collectionOf("session")) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /persistence-mongodb/src/main/scala/glaux/persistence/mongodb/GeneralHandlers.scala: -------------------------------------------------------------------------------- 1 | package glaux.persistence.mongodb 2 | 3 | import java.io.{ObjectOutputStream, ByteArrayOutputStream, ByteArrayInputStream, ObjectInputStream} 4 | import java.time.{ZoneId, Instant, ZonedDateTime} 5 | 6 | import reactivemongo.bson._ 7 | 8 | import scala.reflect.runtime.universe._ 9 | 10 | import play.api.libs.functional._ 11 | 12 | object GeneralHandlers { 13 | type Handler[T] = BSONDocumentReader[T] with BSONDocumentWriter[T] 14 | 15 | trait HandlerTrait[T] extends BSONDocumentReader[T] with BSONDocumentWriter[T] 16 | 17 | object HandlerTrait { 18 | def apply[T](reader: BSONDocumentReader[T], writer: BSONDocumentWriter[T]) = new HandlerTrait[T] { 19 | def write(t: T): BSONDocument = writer.write(t) 20 | def read(bson: BSONDocument): T = reader.read(bson) 21 | } 22 | } 23 | 24 | type Reader[T] = BSONReader[_ <: BSONValue, T] 25 | type Writer[T] = BSONWriter[T, _ <: BSONValue] 26 | 27 | implicit val fcbReader: FunctionalCanBuild[BSONDocumentReader] = new FunctionalCanBuild[BSONDocumentReader] { 28 | def apply[A, B](ma: BSONDocumentReader[A], mb: BSONDocumentReader[B]) = new BSONDocumentReader[~[A, B]] { 29 | def read(b: BSONDocument): ~[A, B] = new ~(ma.read(b), mb.read(b)) 30 | } 31 | } 32 | 33 | implicit val fcbWriter: FunctionalCanBuild[BSONDocumentWriter] = new FunctionalCanBuild[BSONDocumentWriter] { 34 | def apply[A, B](ma: BSONDocumentWriter[A], mb: BSONDocumentWriter[B]) = new BSONDocumentWriter[~[A, B]] { 35 | def write(t: ~[A, B]): BSONDocument = t match { case (a ~ b) ⇒ ma.write(a) ++ mb.write(b) } 36 | } 37 | } 38 | 39 | implicit val fcbHandler: FunctionalCanBuild[Handler] = new FunctionalCanBuild[Handler] { 40 | def apply[A, B](fa: Handler[A], fb: Handler[B]): Handler[A ~ B] = HandlerTrait[A ~ B](fcbReader(fa, fb), fcbWriter(fa, fb)) 41 | } 42 | 43 | implicit val invariantFunctorHandler: InvariantFunctor[Handler] = new InvariantFunctor[Handler] { 44 | def inmap[A, B](m: Handler[A], f1: (A) ⇒ B, f2: (B) ⇒ A): Handler[B] = new HandlerTrait[B] { 45 | def write(b: B): BSONDocument = m.write(f2(b)) 46 | def read(bson: BSONDocument): B = f1(m.read(bson)) 47 | } 48 | } 49 | 50 | def field[T](fieldName: String)(implicit reader: BSONReader[_ <: BSONValue, T], writer: BSONWriter[T, _ <: BSONValue]): Handler[T] = new HandlerTrait[T] { 51 | def read(bson: BSONDocument): T = bson.getAs[T](fieldName).get 52 | def write(t: T): BSONDocument = BSONDocument(fieldName → t) 53 | } 54 | 55 | def nullableField[T](fieldName: String)(implicit reader: BSONReader[_ <: BSONValue, T], writer: BSONWriter[T, _ <: BSONValue]): Handler[Option[T]] = new HandlerTrait[Option[T]] { 56 | def read(bson: BSONDocument): Option[T] = bson.getAs[T](fieldName) 57 | def write(t: Option[T]): BSONDocument = t.map(f ⇒ BSONDocument(fieldName → f)).getOrElse(BSONDocument.empty) 58 | } 59 | 60 | implicit class HandlerOpts[T](self: Handler[T]) { 61 | def cast[To]: Handler[To] = invariantFunctorHandler.inmap(self, (d: T) ⇒ d.asInstanceOf[To], (d: To) ⇒ d.asInstanceOf[T]) 62 | } 63 | 64 | implicit def tuple2F[T1: Reader: Writer, T2: Reader: Writer] = new HandlerTrait[(T1, T2)] { 65 | def read(bson: BSONDocument): (T1, T2) = (bson.getAs[T1]("_1").get, bson.getAs[T2]("_2").get) 66 | 67 | def write(t: (T1, T2)): BSONDocument = BSONDocument( 68 | "_1" → t._1, 69 | "_2" → t._2 70 | ) 71 | } 72 | 73 | implicit def tuple3F[T1: Reader: Writer, T2: Reader: Writer, T3: Reader: Writer] = new HandlerTrait[(T1, T2, T3)] { 74 | def read(bson: BSONDocument): (T1, T2, T3) = ( 75 | bson.getAs[T1]("_1").get, 76 | bson.getAs[T2]("_2").get, 77 | bson.getAs[T3]("_3").get 78 | ) 79 | 80 | def write(t: (T1, T2, T3)): BSONDocument = BSONDocument( 81 | "_1" → t._1, 82 | "_2" → t._2, 83 | "_3" → t._3 84 | ) 85 | } 86 | 87 | def binaryFormat[T]: Handler[T] = new HandlerTrait[T] { 88 | def read(bson: BSONDocument): T = { 89 | val array = bson.get("binary").get.asInstanceOf[BSONBinary].byteArray 90 | val r = new ObjectInputStream(new ByteArrayInputStream(array)).readObject() 91 | r.asInstanceOf[T] 92 | } 93 | 94 | def write(t: T): BSONDocument = { 95 | val bos = new ByteArrayOutputStream() 96 | val oos = new ObjectOutputStream(bos) 97 | try { 98 | oos.writeObject(t) 99 | } catch { 100 | case e: Exception ⇒ 101 | e.printStackTrace() 102 | throw e 103 | } 104 | 105 | oos.close 106 | 107 | BSONDocument( 108 | "class" → t.getClass.getCanonicalName, 109 | "binary" → BSONBinary(bos.toByteArray, Subtype.GenericBinarySubtype) 110 | ) 111 | } 112 | } 113 | 114 | implicit val zdtf = new HandlerTrait[ZonedDateTime] { 115 | def read(bson: BSONDocument): ZonedDateTime = ZonedDateTime.ofInstant( 116 | Instant.ofEpochMilli(bson.getAs[BSONDateTime]("epoch").get.value), 117 | ZoneId.of(bson.getAs[String]("zone").get) 118 | ) 119 | 120 | def write(t: ZonedDateTime): BSONDocument = BSONDocument( 121 | "zone" → t.getZone.getId, 122 | "epoch" → BSONDateTime(t.toInstant.toEpochMilli) 123 | ) 124 | } 125 | 126 | class Partial[ParentT, ChildT <: ParentT: Handler: TypeTag] { 127 | val typeTag = implicitly[TypeTag[ChildT]] 128 | val runtimeClass = typeTag.mirror.runtimeClass(typeTag.tpe) 129 | def name: String = { 130 | typeTag.tpe.typeSymbol.fullName 131 | } 132 | def read(bson: BSONDocument): ParentT = bson.as[ChildT] 133 | def write(t: ParentT) = BSON.write(t.asInstanceOf[ChildT]) 134 | def isDefined(t: ParentT): Boolean = runtimeClass.isInstance(t) 135 | } 136 | 137 | /** 138 | * Use partial function rather than runtimeClass which is prone to erasure. 139 | */ 140 | class GPartial[ParentT, ChildT <: ParentT: Handler: TypeTag](pf: PartialFunction[ParentT, Unit]) extends Partial[ParentT, ChildT] { 141 | override def isDefined(t: ParentT): Boolean = pf.isDefinedAt(t) 142 | } 143 | 144 | def polymorphic[T](partials: Partial[T, _]*) = new HandlerTrait[T] { 145 | 146 | def read(bson: BSONDocument): T = { 147 | val p = partials.find(_.name == bson.getAs[String]("type").get).get 148 | p.read(bson.get("item").get.asInstanceOf[BSONDocument]) 149 | } 150 | 151 | def write(t: T): BSONDocument = { 152 | val p = partials.find(_.isDefined(t)).get 153 | BSONDocument("type" → p.name, "item" → p.write(t)) 154 | } 155 | } 156 | 157 | implicit val unitHandler: Handler[Unit] = new HandlerTrait[Unit] { 158 | override def write(t: Unit): BSONDocument = BSONDocument.empty 159 | 160 | override def read(bson: BSONDocument): Unit = () 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /persistence-mongodb/src/main/scala/glaux/persistence/mongodb/GlauxHandlers.scala: -------------------------------------------------------------------------------- 1 | package glaux.persistence.mongodb 2 | 3 | import glaux.interfaces.api.domain.{SessionId} 4 | import glaux.linearalgebra.Tensor.TensorBuilder 5 | import glaux.linearalgebra._ 6 | import Dimension._ 7 | import glaux.neuralnetwork.Net.DefaultNet 8 | import glaux.neuralnetwork.layers._ 9 | import glaux.neuralnetwork._ 10 | import glaux.neuralnetwork.trainers.BatchTrainer.{LossInfo, BatchResult} 11 | import glaux.neuralnetwork.trainers.SGD.SGDSettings 12 | import glaux.reinforcementlearning.DeepMindQLearner.ConvolutionBased 13 | import glaux.reinforcementlearning.Policy.AnnealingContext 14 | import glaux.reinforcementlearning.QAgent.Session 15 | import glaux.reinforcementlearning._ 16 | import glaux.reinforcementlearning.QLearner.{Transition, History, State, TemporalState} 17 | import glaux.statistics.Probability 18 | import play.api.libs.functional._ 19 | import syntax._ 20 | import reactivemongo.bson.Macros.Options.{AllImplementations, \/, UnionType} 21 | import reactivemongo.bson._ 22 | 23 | import glaux.persistence.mongodb.GeneralHandlers._ 24 | 25 | object GlauxHandlers { 26 | 27 | val d1f = Macros.handler[Row] 28 | val d2f = Macros.handler[TwoD] 29 | val d3f = Macros.handler[ThreeD] 30 | val d4f = Macros.handler[FourD] 31 | 32 | implicit val gdimensionH: Handler[Dimension] = Macros.handlerOpts[Dimension, AllImplementations] 33 | 34 | case class InputDimensionHandler[T <: Tensor](dh: Handler[T#Dimensionality]) extends HandlerTrait[T#Dimensionality] { 35 | def read(bson: BSONDocument): T#Dimensionality = dh.read(bson) 36 | def write(t: T#Dimensionality): BSONDocument = dh.write(t) 37 | } 38 | 39 | implicit val rvih = InputDimensionHandler[RowVector](d1f) 40 | implicit val mih = InputDimensionHandler[Matrix](d2f) 41 | implicit val vih = InputDimensionHandler[Vol](d3f) 42 | implicit val t4ih = InputDimensionHandler[Tensor4](d4f) 43 | 44 | def tf[T <: Tensor: TensorBuilder: InputDimensionHandler]: Handler[T] = new HandlerTrait[T] { 45 | def read(bson: BSONDocument): T = ( 46 | bson.getAs[T#Dimensionality]("dimension").get, 47 | bson.getAs[Seq[Double]]("values").get 48 | ) 49 | 50 | def write(t: T) = BSONDocument( 51 | "dimension" → t.dimension.asInstanceOf[T#Dimensionality], 52 | "values" → t.seqView 53 | ) 54 | } 55 | 56 | implicit val rvf = tf[RowVector] 57 | implicit val mf = tf[Matrix] 58 | implicit val vf = tf[Vol] 59 | implicit val t3f = tf[Tensor4] 60 | 61 | implicit def inputLayerF[Input <: Tensor: InputDimensionHandler] = { 62 | field[Input#Dimensionality]("dimension").inmap(InputLayer.apply[Input], unlift(InputLayer.unapply[Input])) 63 | } 64 | 65 | implicit val rsf = Macros.handler[RegularizationSetting] 66 | implicit val rectanglef = Macros.handler[Rectangle] 67 | implicit val poolF = Macros.handler[Pool] 68 | implicit val convH = Macros.handler[Convolution] 69 | 70 | implicit def fcf[T <: Tensor: TensorBuilder: InputDimensionHandler] = ( 71 | field[Matrix]("filter") ~ 72 | field[RowVector]("bias") ~ 73 | field[RegularizationSetting]("filterRegularization") ~ 74 | field[Dimension]("inDimension").cast[T#Dimensionality] ~ 75 | field[String]("id") 76 | )(FullyConnected.apply[T], unlift(FullyConnected.unapply[T])) 77 | 78 | implicit def reluf[T <: Tensor: TensorBuilder: InputDimensionHandler] = 79 | (field[Dimension]("dimension").cast[T#Dimensionality] //todo: try field[T#Dimensionality] 80 | ~ field[String]("id"))(Relu.apply[T], unlift(Relu.unapply[T])) 81 | 82 | implicit val hiddenLayerH = polymorphic[HiddenLayer]( 83 | new Partial[HiddenLayer, Convolution], 84 | new Partial[HiddenLayer, Pool], 85 | new GPartial[HiddenLayer, FullyConnected[Matrix]]({ case FullyConnected(_, _, _, TwoD(_, _), _) ⇒ }), 86 | new GPartial[HiddenLayer, FullyConnected[RowVector]]({ case FullyConnected(_, _, _, Row(_), _) ⇒ }), 87 | new GPartial[HiddenLayer, FullyConnected[Vol]]({ case FullyConnected(_, _, _, ThreeD(_, _, _), _) ⇒ }), 88 | new GPartial[HiddenLayer, Relu[RowVector]]({ case Relu(Row(_), _) ⇒ }), 89 | new GPartial[HiddenLayer, Relu[Vol]]({ case Relu(ThreeD(_, _, _), _) ⇒ }), 90 | new GPartial[HiddenLayer, Relu[Matrix]]({ case l @ Relu(TwoD(_, _), _) ⇒ }) 91 | ) 92 | 93 | implicit val rlh = Macros.handler[Regression] 94 | implicit val smxh = Macros.handler[Softmax] 95 | implicit val lossh = Macros.handler[LossInfo] 96 | 97 | implicit val lossLayerH = polymorphic[LossLayer]( 98 | new Partial[LossLayer, Regression], 99 | new Partial[LossLayer, Softmax] 100 | ) 101 | 102 | implicit def dftNetH[Input <: Tensor: InputDimensionHandler] = ( 103 | field[InputLayer[Input]]("inputLayer") ~ 104 | field[Seq[HiddenLayer]]("hiddenLayers") ~ 105 | field[LossLayer]("lossLayer") 106 | )(DefaultNet.apply[Input], unlift(DefaultNet.unapply[Input])) 107 | 108 | implicit def dbrh[Input <: Tensor: InputDimensionHandler] = ( 109 | field[LossInfo]("lossInfo") ~ 110 | field[DefaultNet[Input]]("net") ~ 111 | field[Int]("batchSize") ~ 112 | field[Unit]("calculationContext") 113 | )(BatchResult.apply[DefaultNet[Input], Unit], unlift(BatchResult.unapply[DefaultNet[Input], Unit])) 114 | 115 | implicit def temporalStateH[Input <: Tensor: TensorBuilder: Handler] = 116 | (field[Input]("readings") ~ 117 | field[Time]("time"))(TemporalState.apply[Input], unlift(TemporalState.unapply[Input])) 118 | 119 | implicit def qStateH[Input <: Tensor: TensorBuilder: Handler] = 120 | (field[Seq[TemporalState[Input]]]("fullHistory") ~ 121 | field[Boolean]("isTerminal"))(State.apply[Input], unlift(State.unapply[Input])) 122 | 123 | implicit def transitionH[Input <: Tensor: TensorBuilder: Handler] = 124 | (field[State[Input]]("before") ~ 125 | field[Action]("action") ~ 126 | field[Reward]("reward") ~ 127 | field[State[Input]]("after"))(Transition.apply[Input], unlift(Transition.unapply[Input])) 128 | 129 | implicit def learnerIterationH[T <: DeepMindQLearner](learner: T)( 130 | implicit 131 | nh: Handler[learner.Net], trh: Handler[learner.TrainingResult], transitionH: Handler[Transition[learner.Input]], sh: Handler[learner.State] 132 | ) = { 133 | //manual method to function conversion is needed because scala compiler can't convert method with dependent type 134 | import learner.{Net, Memory, TrainingResult, State} 135 | val apply = (n: Net, m: Memory, r: TrainingResult, s: State, it: Boolean, tnhc: Int) ⇒ learner.DeepMindIteration(n, m, r, s, it, tnhc) 136 | val unapply = (i: learner.DeepMindIteration) ⇒ (i.net, i.memory, i.trainingResult, i.state, i.isTerminal, i.targetNetHitCount) 137 | 138 | (field[Net]("targetNet") ~ 139 | field[Memory]("memory") ~ 140 | field[TrainingResult]("trainingResult") ~ 141 | field[State]("state") ~ 142 | field[Boolean]("isTerminal") ~ 143 | field[Int]("targetNetHitCount"))(apply, unapply) 144 | } 145 | 146 | implicit val probabilityR: Reader[Probability] = new BSONReader[BSONDouble, Probability] { 147 | def read(bson: BSONDouble): Probability = Probability(bson.value) 148 | } 149 | 150 | implicit val probabilityW: Writer[Probability] = new BSONWriter[Probability, BSONDouble] { 151 | def write(p: Probability): BSONDouble = BSONDouble(p.value) 152 | } 153 | 154 | implicit val ach = Macros.handler[AnnealingContext] 155 | 156 | implicit def agentSessionH(implicit agent: SimpleQAgent): Handler[agent.Session] = { 157 | implicit val iH = learnerIterationH(agent.qLearner) 158 | sessionH[agent.qLearner.Iteration, agent.policy.Context] 159 | } 160 | 161 | implicit def agentSessionH(implicit agent: AdvancedQAgent): Handler[agent.Session] = { 162 | implicit val iH = learnerIterationH(agent.qLearner) 163 | sessionH[agent.qLearner.Iteration, agent.policy.Context] 164 | } 165 | 166 | def sessionH[Iteration <: QLearner#IterationLike: Handler, Context <: Policy.DecisionContext: Handler]: Handler[Session[Iteration, Context]] = { 167 | val apply = (i: Iteration, cr: Reward, crd: Vector[Reading], dc: Context, la: Option[Action], ic: Boolean) ⇒ Session(i, cr, crd, dc, la, ic) 168 | 169 | val unapply = (s: Session[Iteration, Context]) ⇒ (s.iteration, s.currentReward, s.currentReadings, s.decisionContext, s.lastAction, s.isClosed) 170 | 171 | (field[Iteration]("iteration") ~ 172 | field[Reward]("currentReward") ~ 173 | field[Vector[Reading]]("currentReadings") ~ 174 | field[Context]("decisionContext") ~ 175 | nullableField[Action]("lastAction") ~ 176 | field[Boolean]("isClosed"))(apply, unapply) 177 | } 178 | import ConvolutionBased.Settings 179 | implicit val cblsh = Macros.handler[Settings] 180 | implicit val sgdlsh = Macros.handler[SGDSettings] 181 | 182 | } 183 | -------------------------------------------------------------------------------- /persistence-mongodb/src/main/scala/glaux/persistence/mongodb/InterfaceHandlers.scala: -------------------------------------------------------------------------------- 1 | package glaux.persistence.mongodb 2 | 3 | import glaux.interfaces.api.domain.{AdvancedAgentSettings, AgentSettings, SessionId} 4 | import glaux.persistence.mongodb.GeneralHandlers._ 5 | import glaux.reinforcementlearning.QAgent 6 | import reactivemongo.bson.Macros 7 | import reactivemongo.bson.Macros.Options.AllImplementations 8 | import GlauxHandlers._ 9 | import play.api.libs.functional._ 10 | import syntax._ 11 | 12 | object InterfaceHandlers { 13 | implicit val sessionIdH = Macros.handler[SessionId] 14 | implicit val aashandler = Macros.handler[AdvancedAgentSettings] 15 | implicit val agshandler = Macros.handlerOpts[AgentSettings, AllImplementations] 16 | 17 | implicit def sessionRecordFormat[Session <: QAgent.Session[_, _]: Handler]: Handler[SessionRecord[Session]] = { 18 | (field[SessionId]("id") ~ 19 | field[Session]("session"))(SessionRecord.apply[Session], unlift(SessionRecord.unapply[Session])) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /persistence-mongodb/src/main/scala/glaux/persistence/mongodb/MongoPersistenceImpl.scala: -------------------------------------------------------------------------------- 1 | package glaux.persistence.mongodb 2 | import GeneralHandlers._ 3 | import GlauxHandlers._ 4 | import glaux.interfaces.api.domain.SessionId 5 | import glaux.interfaces.api.persistence.{AgentSettingsPersistence, SessionPersistence, PersistenceImpl} 6 | import glaux.reinforcementlearning.{AdvancedQAgent, QAgent, SimpleQAgent} 7 | import reactivemongo.bson.BSONDocument 8 | 9 | import scala.concurrent.Future 10 | 11 | object MongoPersistenceImpl extends PersistenceImpl { 12 | 13 | implicit def advanceQAgentSessionPersistence: SessionPersistence[AdvancedQAgent] = new QAgentSessionPersistence[AdvancedQAgent] { 14 | implicit def sessionHandler(implicit agent: AdvancedQAgent): Handler[agent.Session] = agentSessionH(agent) 15 | } 16 | 17 | implicit def agentSettingsPersistence: AgentSettingsPersistence = AgentSettingsRepo() 18 | 19 | implicit def simpleQAgentSessionPersistence: SessionPersistence[SimpleQAgent] = new QAgentSessionPersistence[SimpleQAgent] { 20 | implicit def sessionHandler(implicit agent: SimpleQAgent): Handler[agent.Session] = agentSessionH(agent) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /persistence-mongodb/src/main/scala/glaux/persistence/mongodb/Repository.scala: -------------------------------------------------------------------------------- 1 | package glaux.persistence.mongodb 2 | 3 | import glaux.interfaces.api.domain._ 4 | import glaux.persistence.mongodb.GeneralHandlers.Handler 5 | import reactivemongo.api.MongoDriver 6 | import reactivemongo.api.collections.bson.BSONCollection 7 | import reactivemongo.api.commands.WriteResult 8 | import reactivemongo.bson.BSONDocument 9 | 10 | import scala.concurrent.Future 11 | import scala.concurrent.ExecutionContext.Implicits.global 12 | 13 | trait Repository { 14 | val collection: BSONCollection 15 | import collection.BatchCommands.FindAndModifyCommand.FindAndModifyResult 16 | import collection.BatchCommands.FindAndModifyCommand.UpdateLastError 17 | implicit protected def writeResultToUnit(r: Future[WriteResult]): Future[Unit] = r.collect { 18 | case wr: WriteResult if wr.ok ⇒ () 19 | } 20 | 21 | implicit protected def modifyResultToUnit(r: Future[FindAndModifyResult]): Future[Unit] = r.flatMap { 22 | case FindAndModifyResult(Some(le), _) if le.err.isEmpty ⇒ Future.successful(()) 23 | case r: FindAndModifyResult ⇒ Future.failed(new FailedUpdate(r.lastError.get)) 24 | } 25 | 26 | case class FailedUpdate(le: UpdateLastError) extends Exception(le.err.toString) 27 | 28 | def upsert[T: Handler](idSelector: BSONDocument, item: T): Future[Unit] = 29 | collection.findAndUpdate(selector = idSelector, update = item, upsert = true) 30 | 31 | } 32 | 33 | object Repository { 34 | def collectionOf(name: String): BSONCollection = { 35 | val driver = new MongoDriver 36 | val connection = driver.connection(List("localhost")) 37 | val db = connection("glaux") 38 | db(name) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /persistence-mongodb/src/main/scala/glaux/persistence/mongodb/SessionRepoImpl.scala: -------------------------------------------------------------------------------- 1 | package glaux.persistence.mongodb 2 | 3 | import glaux.interfaces.api.domain.SessionId 4 | import glaux.interfaces.api.persistence.SessionPersistence 5 | import glaux.persistence.mongodb.GeneralHandlers.Handler 6 | import glaux.reinforcementlearning.QAgent 7 | import reactivemongo.api.collections.bson.BSONCollection 8 | import reactivemongo.api.commands.WriteResult 9 | import reactivemongo.api.indexes.{IndexType, Index} 10 | import reactivemongo.bson.BSONDocument 11 | 12 | import scala.concurrent.Future 13 | import scala.concurrent.ExecutionContext.Implicits.global 14 | import InterfaceHandlers._ 15 | import shapeless.cachedImplicit 16 | 17 | trait QAgentSessionPersistence[A <: QAgent] extends SessionPersistence[A] { 18 | 19 | implicit def sessionHandler(implicit agent: A): Handler[agent.Session] 20 | 21 | def repo(implicit agent: A) = SessionRepoImpl(agent) //todo: creating an instance every time is not performance friendly - need to bench mark 22 | 23 | def get(agent: A, id: SessionId): Future[Option[agent.Session]] = repo(agent).get(id) 24 | 25 | def upsert(agent: A, id: SessionId)(session: agent.Session): Future[Unit] = repo(agent).upsert(id, session) 26 | } 27 | 28 | case class SessionRepoImpl[Session <: QAgent.Session[_, _]: Handler](collection: BSONCollection) extends Repository { 29 | implicit private val handler: Handler[SessionRecord[Session]] = cachedImplicit 30 | 31 | def get(id: SessionId): Future[Option[Session]] = 32 | collection.find(idSelector(id)).one[SessionRecord[Session]].map(_.map(_.session)) 33 | 34 | def insert(sessionId: SessionId, session: Session): Future[Unit] = collection.insert(SessionRecord(sessionId, session)) 35 | 36 | def upsert(id: SessionId, session: Session): Future[Unit] = upsert(idSelector(id), SessionRecord(id, session)) 37 | 38 | private def idSelector(id: SessionId): BSONDocument = BSONDocument("id.profileId" → id.profileId, "id.agentName" → id.agentName) 39 | 40 | } 41 | 42 | private[mongodb] case class SessionRecord[Session <: QAgent.Session[_, _]: Handler](id: SessionId, session: Session) 43 | 44 | object SessionRepoImpl { 45 | 46 | def apply[AT <: QAgent](qAgent: AT)(implicit h: Handler[qAgent.Session]): SessionRepoImpl[qAgent.Session] = { 47 | SessionRepoImpl[qAgent.Session](collection) 48 | } 49 | 50 | private[glaux] def removeAll(): Future[WriteResult] = collection.remove(BSONDocument.empty) 51 | 52 | private lazy val collection: BSONCollection = { 53 | val col = Repository.collectionOf("session") 54 | col.indexesManager.ensure(Index(Seq( 55 | ("id.profileId", IndexType.Ascending), 56 | ("id.agentName", IndexType.Ascending) 57 | ), name = Some("agentId_index"))) 58 | col 59 | } 60 | 61 | def close() = collection.db.connection.close() 62 | 63 | } 64 | -------------------------------------------------------------------------------- /persistence-mongodb/src/test/scala/glaux/persistence/mongodb/GeneralFormatsSpec.scala: -------------------------------------------------------------------------------- 1 | package glaux.persistence.mongodb 2 | 3 | import java.time.ZonedDateTime 4 | 5 | import glaux.interfaces.api.domain._ 6 | import glaux.persistence.mongodb.GeneralHandlers._ 7 | import org.specs2.mutable.Specification 8 | import reactivemongo.bson._ 9 | import play.api.libs.functional._ 10 | import syntax._ 11 | 12 | case class TestCase(readings: Vector[(Seq[Double], ZonedDateTime)]) 13 | case class TestCase2(readings: Seq[Boolean]) 14 | 15 | class GeneralFormatsSpec extends Specification { 16 | 17 | def canHandle[T: Handler](t: T) = 18 | BSON.write(t).as[T] === t 19 | 20 | "Formats" should { 21 | 22 | "format Tuple with option and vectors" in { 23 | implicit val f = Macros.handler[TestCase] 24 | canHandle(TestCase(Vector((Seq(12d, 2d), ZonedDateTime.now)))) 25 | } 26 | 27 | "format with serialization" in { 28 | implicit val f = binaryFormat[TestCase] 29 | canHandle(TestCase(Vector((Seq(12d, 2d), ZonedDateTime.now)))) 30 | } 31 | 32 | "format time" in { 33 | val t = ZonedDateTime.now 34 | canHandle(t) 35 | } 36 | 37 | "format readings" in { 38 | val reading = (Seq(12d, 2d), ZonedDateTime.now) 39 | canHandle(reading) 40 | } 41 | 42 | "format seq" in { 43 | implicit val f = Macros.handler[TestCase2] 44 | canHandle(TestCase2(Seq(true, false))) 45 | } 46 | 47 | "composing" in { 48 | case class TestADT(a: String, b: Int) 49 | 50 | implicit val f = ( 51 | field[String]("a1") ~ 52 | field[Int]("b") 53 | )(TestADT.apply, unlift(TestADT.unapply)) 54 | 55 | canHandle(TestADT("aa", 4)) 56 | } 57 | 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /persistence-mongodb/src/test/scala/glaux/persistence/mongodb/GlauxHandlersSpec.scala: -------------------------------------------------------------------------------- 1 | package glaux.persistence.mongodb 2 | 3 | import java.time.{Clock, LocalTime, LocalDate, ZonedDateTime} 4 | 5 | import glaux.interfaces.api.domain.Action 6 | import glaux.linearalgebra.Dimension._ 7 | import glaux.linearalgebra._ 8 | import glaux.neuralnetwork.Net.DefaultNet 9 | import glaux.neuralnetwork.trainers.BatchTrainer.{BatchResult, LossInfo} 10 | import glaux.neuralnetwork.trainers.SGD.SGDSettings 11 | import glaux.neuralnetwork.{Rectangle, LossLayer, HiddenLayer, InputLayer} 12 | import glaux.neuralnetwork.layers._ 13 | import glaux.reinforcementlearning.DeepMindQLearner.ConvolutionBased 14 | import glaux.reinforcementlearning.QLearner.{Observation, State, TemporalState} 15 | import glaux.reinforcementlearning._ 16 | import org.specs2.mutable.Specification 17 | import GlauxHandlers._ 18 | import GeneralHandlers._ 19 | import reactivemongo.bson._ 20 | 21 | case class Test(readings: Vector[Reading], lastAction: Option[Action]) 22 | case class TestHiddenLayer(layers: Seq[HiddenLayer]) 23 | case class TestLossLayer(layers: Seq[LossLayer]) 24 | 25 | class GlauxHandlersSpec extends Specification { 26 | 27 | def canHandle[T: Handler](t: T) = 28 | BSON.write(t).as[T] === t 29 | 30 | "Formats" should { 31 | 32 | "format specific dimensions" in { 33 | canHandle(Row(5)) 34 | canHandle(TwoD(5, 3)) 35 | canHandle(ThreeD(5, 2, 1)) 36 | } 37 | 38 | "format general dimension" in { 39 | val d: Dimension = TwoD(5, 3) 40 | canHandle(d) 41 | val d2: Dimension = ThreeD(5, 2, 1) 42 | canHandle(d2) 43 | } 44 | 45 | "format tensors" in { 46 | canHandle(RowVector(2, 3, 4)) 47 | canHandle(Matrix(2, 2, Seq(4d, 3d, 1d, 2d))) 48 | canHandle(Vol(2, 2, 2, Seq(4d, 3d, 1d, 2d, 4d, 3d, 1d, 2d))) 49 | } 50 | 51 | "format InputLayer" in { 52 | canHandle(InputLayer[RowVector](Row(3))) 53 | } 54 | 55 | "format FullyConnected" in { 56 | canHandle(FullyConnected(3, 4)) 57 | } 58 | 59 | "format FullyConnected of Vol" in { 60 | canHandle(FullyConnected[Vol](ThreeD(3, 3, 3), 4)) 61 | } 62 | 63 | "format Convolution" in { 64 | val inputDim = ThreeD(4, 4, 2) 65 | val layer = Convolution(3, Rectangle(3, 3), inputDim, 1) 66 | canHandle(layer) 67 | } 68 | 69 | "format Pool" in { 70 | canHandle(Pool(ThreeD(4, 4, 1), Rectangle(3, 3), 3)) 71 | } 72 | 73 | "format Relu" in { 74 | canHandle(Relu[RowVector](Row(4))) 75 | canHandle(Relu[Matrix](TwoD(2, 3))) 76 | canHandle(Relu[Vol](ThreeD(4, 2, 3))) 77 | } 78 | 79 | "format HiddenLayer" in { 80 | val f: HiddenLayer = FullyConnected(3, 4) 81 | val fv: HiddenLayer = FullyConnected[Vol](ThreeD(3, 3, 3), 4) 82 | val r: Relu[RowVector] = Relu[RowVector](Row(4)) 83 | val p: Pool = Pool(ThreeD(4, 4, 1), Rectangle(3, 3), 3) 84 | val c = Convolution(3, Rectangle(3, 3), ThreeD(3, 3, 3), 1) 85 | val r3: Relu[Vol] = Relu[Vol](ThreeD(3, 2, 3)) 86 | val r2: Relu[Matrix] = Relu[Matrix](TwoD(2, 3)) 87 | implicit val thf = Macros.handler[TestHiddenLayer] 88 | canHandle(TestHiddenLayer(Seq(f, fv, r, p, c, r3, r2))) 89 | } 90 | 91 | "format lossLayer" in { 92 | val r: LossLayer = Regression(Row(3)) 93 | val s: LossLayer = Softmax(Row(3)) 94 | implicit val thf = Macros.handler[TestLossLayer] 95 | canHandle(TestLossLayer(Seq(r, s))) 96 | } 97 | 98 | "format default net" in { 99 | val inputSize = 5 100 | val netInputDimension = Row(inputSize) 101 | val inputLayer = InputLayer[RowVector](netInputDimension) 102 | val fc1 = FullyConnected(inputSize, inputSize) 103 | val relu = Relu[RowVector](netInputDimension) 104 | val fc2 = FullyConnected(inputSize, 5) 105 | val lossLayer = Regression(5) 106 | val net = DefaultNet(inputLayer, Seq(fc1, relu, fc2), lossLayer) 107 | canHandle(net) 108 | } 109 | 110 | "format BatchResult" in { 111 | val loss = LossInfo(1, 3, 4) 112 | 113 | val inputSize = 5 114 | val inputLayer = InputLayer[RowVector](Row(inputSize)) 115 | val fc1 = FullyConnected(inputSize, inputSize) 116 | val lossLayer = Regression(inputSize) 117 | val net = DefaultNet(inputLayer, Seq(fc1), lossLayer) 118 | canHandle(BatchResult(loss, net, 1, ())) 119 | } 120 | 121 | "format temporal state" in { 122 | val t = new TemporalState(Matrix(2, 2, Seq(1d, 1d, 1d, 3d)), ZonedDateTime.now) 123 | canHandle(t) 124 | } 125 | 126 | "format State" in { 127 | val t = new TemporalState(Matrix(2, 2, Seq(1d, 1d, 1d, 3d)), ZonedDateTime.now) 128 | val t2 = new TemporalState(Matrix(2, 2, Seq(1d, 2d, 4d, 3d)), ZonedDateTime.now.plusDays(1)) 129 | canHandle(State(Seq(t, t2), true)) 130 | } 131 | 132 | "format learner iteration" in { 133 | val start = ZonedDateTime.of(LocalDate.of(2015, 2, 14), LocalTime.of(14, 30), Clock.systemDefaultZone().getZone) 134 | val learner = DeepMindQLearner.Simplified(historyLength = 2) 135 | val initHistory = Seq(TemporalState(RowVector(1, 0), start), TemporalState(RowVector(2, 1), start.plusMinutes(1))) 136 | val init = learner.init(initHistory, 2) 137 | val iteration = learner.iterate(init, Observation( 138 | lastAction = 1, 139 | reward = 1, 140 | recentHistory = Seq(TemporalState(RowVector(3, 2), start.plusMinutes(2))), 141 | isTerminal = true 142 | )) 143 | implicit val iterH = learnerIterationH(learner) 144 | canHandle(iteration) 145 | } 146 | 147 | "format simple qAgent Session" in { 148 | val agent = SimpleQAgent(3, 3) 149 | val reading = (Seq(3d, 2d), ZonedDateTime.now) 150 | val result = agent.start(List(reading, reading, reading), None) 151 | val session = result.right.get 152 | 153 | implicit val sessionF = agentSessionH(agent) 154 | canHandle(session) 155 | } 156 | 157 | "format advanced qAgent Session" in { 158 | val agent = AdvancedQAgent(3, ConvolutionBased.Settings(), SGDSettings()) 159 | val reading = (Seq(3d, 2d), ZonedDateTime.now) 160 | val result = agent.start((0 to 100).map(_ ⇒ reading), None) 161 | val session = result.right.get 162 | 163 | implicit val sessionF = agentSessionH(agent) 164 | canHandle(session) 165 | } 166 | 167 | } 168 | 169 | } 170 | -------------------------------------------------------------------------------- /persistence-mongodb/src/test/scala/glaux/persistence/mongodb/SessionRepoImplIntegration.scala: -------------------------------------------------------------------------------- 1 | package glaux.persistence.mongodb 2 | 3 | import java.time.{Clock, ZonedDateTime} 4 | 5 | import glaux.interfaces.api.domain.SessionId 6 | import glaux.linearalgebra.RowVector 7 | import glaux.neuralnetwork.trainers.SGD.SGDSettings 8 | import glaux.neuralnetwork.trainers.{SGD, VanillaSGD} 9 | import glaux.persistence.mongodb.GeneralHandlers._ 10 | import glaux.reinforcementlearning.DeepMindQLearner.ConvolutionBased 11 | import glaux.reinforcementlearning.QLearner.{Observation, TemporalState} 12 | import glaux.reinforcementlearning._ 13 | import org.specs2.mutable.Specification 14 | import org.specs2.concurrent.ExecutionEnv 15 | import org.specs2.specification.{AfterAll, AfterEach} 16 | 17 | import scala.concurrent.Await 18 | import scala.concurrent.duration._ 19 | 20 | class SessionRepoImplIntegration extends Specification with AfterEach with AfterAll { 21 | sequential 22 | import GlauxHandlers._ 23 | 24 | "SessionRepo with simpleAgent" should { 25 | 26 | val agent = SimpleQAgent(3, 3) 27 | val reading = (Seq(3d, 2d), ZonedDateTime.now) 28 | val result = agent.start(List(reading, reading, reading), None) 29 | val session = result.right.get 30 | implicit val h = agentSessionH(agent) 31 | val repo = SessionRepoImpl(agent) 32 | 33 | "Insert and retrieve" in { implicit ee: ExecutionEnv ⇒ 34 | 35 | val sessionId = SessionId("a Test", "111") 36 | 37 | repo.insert(sessionId, session) must be_==(()).await 38 | 39 | repo.get(sessionId) must beSome(session).await.eventually 40 | } 41 | 42 | "upsert add new record" in { implicit ee: ExecutionEnv ⇒ 43 | 44 | val agentId = SessionId("a Test", "112") 45 | repo.get(agentId) must beNone.await 46 | 47 | repo.upsert(agentId, session) must be_==(()).await 48 | 49 | repo.get(agentId) must beSome(session).await.eventually 50 | } 51 | 52 | "upsert update exisiting new record" in { implicit ee: ExecutionEnv ⇒ 53 | 54 | val agentId = SessionId("a Test", "113") 55 | 56 | val (_, newSession) = agent.requestAction(session) 57 | 58 | repo.insert(agentId, session) must be_==(()).await 59 | repo.get(agentId) must beSome(session).await.eventually 60 | 61 | repo.upsert(agentId, newSession) must be_==(()).await 62 | repo.get(agentId) must beSome(newSession).await.eventually 63 | } 64 | 65 | } 66 | 67 | "Session Repo with Advanced Agent" should { 68 | implicit val agent = AdvancedQAgent(3, ConvolutionBased.Settings(), SGDSettings()) 69 | val reading = (Seq(3d, 2d), ZonedDateTime.now) 70 | def mockReadings(n: Int) = (0 to n).map(i ⇒ reading.copy(_2 = reading._2.plusSeconds(i))) 71 | val result = agent.start(mockReadings(100), None) 72 | val init = result.right.get 73 | 74 | val (_, afterAction) = agent.requestAction(init) 75 | 76 | val last = (0 to 50).foldLeft(afterAction) { (session, i) ⇒ 77 | agent.report((Seq(1d, 2d), ZonedDateTime.now.plusSeconds(i + 101)), 1d, session) 78 | } 79 | 80 | val (_, finalResult) = agent.requestAction(last) 81 | 82 | implicit val h = agentSessionH(agent) 83 | val repo = SessionRepoImpl(agent) 84 | 85 | "Insert and retrieve" in { implicit ee: ExecutionEnv ⇒ 86 | val agentId = SessionId("a Test", "111") 87 | 88 | repo.insert(agentId, finalResult) must be_==(()).await 89 | 90 | repo.get(agentId) must beSome(finalResult).await.eventually 91 | } 92 | 93 | } 94 | 95 | def after: Any = Await.result(SessionRepoImpl.removeAll(), 30.seconds) 96 | 97 | def afterAll(): Unit = SessionRepoImpl.close() 98 | } 99 | -------------------------------------------------------------------------------- /project/Common.scala: -------------------------------------------------------------------------------- 1 | import sbt.Keys._ 2 | import sbt._ 3 | 4 | object Common { 5 | 6 | val settings = Seq( 7 | Helpers.gcTask, 8 | scalacOptions ++= Seq( 9 | "-deprecation", 10 | "-unchecked", 11 | "-Xlint" 12 | ) 13 | ) 14 | 15 | } 16 | -------------------------------------------------------------------------------- /project/Dependencies.scala: -------------------------------------------------------------------------------- 1 | import sbt.Keys._ 2 | import sbt._ 3 | 4 | object Dependencies { 5 | object Versions { 6 | val specs2 = "3.6.6" 7 | val nd4j = "0.0.3.5.5.5" 8 | val akka = "2.4.1" 9 | } 10 | 11 | 12 | val shapeless = Seq("com.chuusai" %% "shapeless" % "2.2.5") 13 | val cat = Seq("org.spire-math" %% "cats" % "0.3.0") 14 | 15 | val (test, integration) = { 16 | val specs = Seq( 17 | "org.specs2" %% "specs2-core" % Versions.specs2, 18 | "org.specs2" %% "specs2-mock" % Versions.specs2, 19 | "org.specs2" %% "specs2-scalacheck" % Versions.specs2 20 | ) 21 | 22 | (specs.map(_ % "test"), specs.map(_ % "integration")) 23 | } 24 | 25 | val nd4j = Seq ( 26 | //tobble between the following 2 lines to use GPU 27 | "org.nd4j" % "nd4j-jblas" % Versions.nd4j, 28 | // "org.nd4j" % "nd4j-jcublas-7.0" % Versions.nd4j, 29 | "org.nd4j" % "nd4j-api" % Versions.nd4j 30 | ) 31 | 32 | 33 | val akka = Seq ( 34 | "com.typesafe.akka" %% "akka-actor" % Versions.akka, 35 | "com.typesafe.akka" %% "akka-testkit" % Versions.akka % "test" 36 | ) 37 | 38 | val apacheCommonMath = Seq( 39 | "org.apache.commons" % "commons-math3" % "3.5" 40 | ) 41 | 42 | val mongodb = Seq ( 43 | "org.reactivemongo" %% "reactivemongo" % "0.11.7", 44 | "com.typesafe.play" %% "play-functional" % "2.4.2" 45 | ) 46 | 47 | val config = Seq( 48 | "com.typesafe" % "config" % "1.2.1" 49 | ) 50 | 51 | val commonSettings = Seq( 52 | scalaVersion in ThisBuild := "2.11.7", 53 | addCompilerPlugin("org.spire-math" %% "kind-projector" % "0.7.1"), 54 | resolvers ++= Seq( 55 | Resolver.sonatypeRepo("releases"), 56 | Resolver.sonatypeRepo("snapshots"), 57 | Resolver.bintrayRepo("scalaz", "releases"), 58 | Resolver.bintrayRepo("typesafe", "maven-releases") 59 | ) 60 | ) 61 | 62 | val coreModuleSettings = commonSettings ++ Seq( 63 | libraryDependencies ++= shapeless ++ cat ++ test 64 | ) 65 | 66 | 67 | } 68 | -------------------------------------------------------------------------------- /project/Format.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import com.typesafe.sbt.SbtScalariform 3 | import com.typesafe.sbt.SbtScalariform.ScalariformKeys 4 | 5 | object Format { 6 | lazy val settings = SbtScalariform.scalariformSettings ++ Seq( 7 | ScalariformKeys.preferences in Compile := formattingPreferences, 8 | ScalariformKeys.preferences in Test := formattingPreferences 9 | ) 10 | 11 | def formattingPreferences = { 12 | import scalariform.formatter.preferences._ 13 | FormattingPreferences() 14 | .setPreference(RewriteArrowSymbols, true) 15 | .setPreference(AlignParameters, true) 16 | .setPreference(AlignSingleLineCaseStatements, true) 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /project/Helpers.scala: -------------------------------------------------------------------------------- 1 | import sbt.TaskKey 2 | 3 | object Helpers { 4 | val gc = TaskKey[Unit]("gc", "runs garbage collector") 5 | val gcTask = gc := { 6 | println("requesting garbage collection") 7 | System gc() 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /project/Projects.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import Keys._ 3 | 4 | object Projects extends Build { 5 | 6 | lazy val glaux = project.in(file(".")) 7 | .configs(Testing.Integration) 8 | .settings(moduleName := "root") 9 | .aggregate(linearAlgebra, neuralNetwork, reinforcementLearning, statistics, akkaInterfaceService, interfaceAPI, persistenceMongoDB ) 10 | .settings(Common.settings:_*) 11 | .settings(Testing.witIntegrationSettings:_*) 12 | 13 | .settings(noPublishing: _*) 14 | 15 | lazy val linearAlgebra = project.in(file("linear-algebra")) 16 | .dependsOn(statistics) 17 | .aggregate(statistics) 18 | .settings(moduleName := "glaux-linear-algebra") 19 | .settings( 20 | libraryDependencies ++= Dependencies.nd4j 21 | ) 22 | .settings(coreModuleSettings:_*) 23 | 24 | lazy val neuralNetwork = project.in(file("neural-network")) 25 | .dependsOn( linearAlgebra ) 26 | .aggregate( linearAlgebra) 27 | .settings(moduleName := "glaux-neural-network") 28 | .settings(coreModuleSettings:_*) 29 | 30 | lazy val reinforcementLearning = project.in(file("reinforcement-learning")) 31 | .dependsOn( neuralNetwork ) 32 | .aggregate( neuralNetwork ) 33 | .settings(moduleName := "glaux-reinforcement-learning") 34 | .settings(coreModuleSettings:_*) 35 | 36 | lazy val statistics = project.in(file("statistics")) 37 | .settings(moduleName := "glaux-statistics") 38 | .settings( 39 | libraryDependencies ++= Dependencies.apacheCommonMath 40 | ) 41 | .settings(coreModuleSettings:_*) 42 | 43 | lazy val akkaInterfaceService = project.in(file("akka-interface-service")) 44 | .dependsOn(interfaceAPI) 45 | .aggregate(interfaceAPI) 46 | .settings(moduleName := "glaux-akka-interface") 47 | .settings( 48 | libraryDependencies ++= Dependencies.akka 49 | ) 50 | .settings(coreModuleSettings:_*) 51 | 52 | lazy val interfaceAPI = project.in(file("interface-api")) 53 | .dependsOn( reinforcementLearning) 54 | .aggregate( reinforcementLearning) 55 | .settings(moduleName := "glaux-akka-interface-api") 56 | .settings(coreModuleSettings:_*) 57 | 58 | lazy val persistenceMongoDB = project.in(file("persistence-mongodb")) 59 | .configs(Testing.Integration) 60 | .dependsOn( interfaceAPI ) 61 | .aggregate( interfaceAPI ) 62 | .settings(moduleName := "glaux-persistence-mongodb") 63 | .settings( 64 | libraryDependencies ++= Dependencies.mongodb 65 | ) 66 | .settings(coreModuleSettings:_*) 67 | .settings(Testing.witIntegrationSettings:_*) 68 | 69 | val coreModuleSettings = Common.settings ++ 70 | Publish.settings ++ 71 | Format.settings ++ 72 | Testing.settings ++ 73 | Dependencies.coreModuleSettings 74 | 75 | val noPublishing = Seq(publish := (), publishLocal := (), publishArtifact := false) 76 | } 77 | -------------------------------------------------------------------------------- /project/Publish.scala: -------------------------------------------------------------------------------- 1 | import sbt._, Keys._ 2 | import bintray.BintrayKeys._ 3 | 4 | 5 | object Publish { 6 | 7 | val bintraySettings = Seq( 8 | bintrayOrganization := Some("org.typeAI"), 9 | bintrayPackageLabels := Seq("glaux") 10 | ) 11 | 12 | val publishingSettings = Seq( 13 | organization in ThisBuild := "glaux", 14 | publishMavenStyle := true, 15 | licenses := Seq("Apache-2.0" -> url("https://www.apache.org/licenses/LICENSE-2.0.html")), 16 | homepage := Some(url("http://typeAI.github.io/glaux")), 17 | scmInfo := Some(ScmInfo(url("https://github.com/typeAI/glaux"), 18 | "git@github.com:typeAI/glaux.git")), 19 | pomIncludeRepository := { _ => false }, 20 | publishArtifact in Test := false 21 | ) 22 | 23 | val settings = bintraySettings ++ publishingSettings 24 | } 25 | 26 | -------------------------------------------------------------------------------- /project/Testing.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import sbt.Keys._ 3 | 4 | object Testing { 5 | lazy val Integration = config("integration").extend(Test) 6 | 7 | def isIntegrationTest(name: String): Boolean = name.endsWith("Integration") 8 | def isUnitTest(name: String): Boolean = !isIntegrationTest(name) 9 | 10 | lazy val settings = { 11 | Seq( 12 | scalacOptions in Test ++= Seq("-Yrangepos"), 13 | testOptions in Test := Seq(Tests.Filter(isUnitTest)) 14 | ) 15 | } 16 | 17 | lazy val witIntegrationSettings = settings ++ Seq( 18 | testOptions in Integration := Seq(Tests.Filter(isIntegrationTest)), 19 | libraryDependencies ++= Dependencies.integration 20 | ) ++ inConfig(Integration)(Defaults.testTasks) 21 | } 22 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.9 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | resolvers += "Sonatype OSS Releases" at "https://oss.sonatype.org/service/local/staging/deploy/maven2" 2 | 3 | resolvers += "Typesafe Repository" at "https://repo.typesafe.com/typesafe/releases/" 4 | 5 | addSbtPlugin("me.lessis" % "bintray-sbt" % "0.3.0") 6 | 7 | addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.1") 8 | 9 | addSbtPlugin("org.scalariform" % "sbt-scalariform" % "1.6.0") 10 | 11 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.3") 12 | 13 | addSbtPlugin("com.codacy" % "sbt-codacy-coverage" % "1.2.1") 14 | -------------------------------------------------------------------------------- /reinforcement-learning/src/main/scala/glaux/reinforcement/DeepMindQLearner.scala: -------------------------------------------------------------------------------- 1 | package glaux.reinforcementlearning 2 | 3 | import glaux.linearalgebra.Dimension.{ThreeD, Row} 4 | import glaux.linearalgebra.{Vol, RowVector} 5 | import glaux.neuralnetwork.trainers.VanillaSGD 6 | import glaux.neuralnetwork.trainers.SGD.SGDSettings 7 | import glaux.neuralnetwork.{HiddenLayer, Rectangle, InputLayer, Net} 8 | import Net.DefaultNet 9 | import glaux.neuralnetwork.layers._ 10 | import glaux.reinforcementlearning.QLearner.Transition 11 | 12 | import scala.util.Random 13 | 14 | /** 15 | * QLearner based on deepmind algorithm 16 | */ 17 | trait DeepMindQLearner extends QLearner { 18 | 19 | def gamma: Double 20 | def batchSize: Int 21 | def targetNetUpdateFreq: Int //avg # of iterations before updating the target net 22 | def minMemorySizeBeforeTraining: Int 23 | 24 | type Net = DefaultNet[NetInput] 25 | type Trainer = VanillaSGD[Net] 26 | 27 | case class DeepMindIteration( 28 | targetNet: Net, 29 | memory: Memory, //Seq because we need random access here 30 | trainingResult: TrainingResult, 31 | state: State, 32 | isTerminal: Boolean = false, 33 | targetNetHitCount: Int = 0 34 | ) extends IterationLike { 35 | lazy val net = trainingResult.net 36 | 37 | } 38 | 39 | assert(historyLength > 0) 40 | 41 | type Iteration = DeepMindIteration 42 | 43 | protected def buildNet(inputDimension: Input#Dimensionality, numOfActions: Int): Net 44 | 45 | override protected def doInit(initState: State, numOfActions: Action, inputDim: InputDimension): Iteration = { 46 | val initNet = buildNet(inputDim, numOfActions) 47 | 48 | DeepMindIteration(initNet, Nil, trainer.init(initNet), initState) 49 | } 50 | 51 | override protected def doIterate(lastIteration: Iteration, observation: Observation, currentState: State): Iteration = { 52 | val newMemory = if (lastIteration.isTerminal) lastIteration.memory 53 | else { 54 | val before = lastIteration.state 55 | lastIteration.memory :+ Transition(before, observation.lastAction, observation.reward, currentState) 56 | } 57 | 58 | val doTraining: Boolean = newMemory.size > minMemorySizeBeforeTraining 59 | val updateTarget: Boolean = doTraining && lastIteration.targetNetHitCount > targetNetUpdateFreq 60 | 61 | val targetNet = lastIteration.targetNet 62 | lazy val newResult = train(newMemory, lastIteration.trainingResult, targetNet) 63 | 64 | lastIteration.copy( 65 | targetNet = if (updateTarget) newResult.net else targetNet, 66 | memory = newMemory, 67 | trainingResult = if (doTraining) newResult else lastIteration.trainingResult, 68 | state = currentState, 69 | isTerminal = observation.isTerminal, 70 | targetNetHitCount = if (updateTarget) 0 else lastIteration.targetNetHitCount + 1 71 | ) 72 | } 73 | 74 | protected def validate(): Unit = { 75 | assert(minMemorySizeBeforeTraining > batchSize, "must have enough transitions in memory before training") 76 | } 77 | 78 | private def train(memory: Memory, lastResult: TrainingResult, targetNet: Net): TrainingResult = { 79 | def randomExamples: Memory = { 80 | (1 to batchSize).map { _ ⇒ 81 | memory(Random.nextInt(memory.size)) 82 | } 83 | } 84 | 85 | def toTrainingInput(transition: Transition[Input]): (NetInput, Trainer#ScalarOutputInfo) = { 86 | val regressionOnAction = if (transition.after.isTerminal) transition.reward else 87 | transition.reward + targetNet.predict(transition.after).seqView.max * gamma 88 | 89 | (transition.before, (regressionOnAction, transition.action)) 90 | } 91 | trainer.trainBatchWithScalaOutputInfo(lastResult)(randomExamples.map(toTrainingInput)) 92 | } 93 | 94 | } 95 | 96 | object DeepMindQLearner { 97 | case class Simplified( 98 | override protected val trainer: Simplified#Trainer = VanillaSGD[Simplified#Net](SGDSettings()), 99 | historyLength: Int = 50, 100 | gamma: Double = 0.95, 101 | batchSize: Int = 40, 102 | targetNetUpdateFreq: Int = 10, //avg # of iterations before updating the target net 103 | minMemorySizeBeforeTraining: Int = 100 104 | ) extends DeepMindQLearner { 105 | type NetInput = RowVector 106 | type Input = RowVector 107 | 108 | validate() 109 | 110 | implicit def inputToNet(state: State): NetOutput = { 111 | RowVector(state.fullHistory.flatMap(_.readings.seqView): _*) 112 | } 113 | 114 | protected def buildNet(inputDimension: Input#Dimensionality, numOfActions: Int): Net = { 115 | val inputSize = inputDimension.size * historyLength 116 | val netInputDimension = Row(inputSize) 117 | val inputLayer = InputLayer[RowVector](netInputDimension) 118 | val fc1 = FullyConnected(inputSize, inputSize) 119 | val relu = Relu[RowVector](netInputDimension) 120 | val fc2 = FullyConnected(inputSize, numOfActions) 121 | val lossLayer = Regression(numOfActions) 122 | DefaultNet(inputLayer, Seq(fc1, relu, fc2), lossLayer) 123 | } 124 | } 125 | 126 | case class ConvolutionBased( 127 | override protected val trainer: ConvolutionBased#Trainer = VanillaSGD[ConvolutionBased#Net](SGDSettings()), 128 | settings: ConvolutionBased.Settings 129 | ) extends DeepMindQLearner { 130 | type NetInput = Vol 131 | type Input = RowVector 132 | 133 | def gamma = settings.gamma 134 | def historyLength = settings.historyLength 135 | def batchSize = settings.batchSize 136 | def targetNetUpdateFreq = settings.targetNetUpdateFreq 137 | def minMemorySizeBeforeTraining = settings.minMemorySizeBeforeTraining 138 | 139 | validate() 140 | assert(historyLength > settings.filterSize * 2, "too short history makes convolution useless") 141 | 142 | implicit def inputToNet(state: State): NetInput = { 143 | Vol(netInputDimension(state.fullHistory.head.readings.dimension), state.fullHistory.flatMap(_.readings.seqView)) 144 | } 145 | 146 | protected def buildNet(inputDimension: Input#Dimensionality, numOfActions: Int): Net = { 147 | val netInputDim = netInputDimension(inputDimension) 148 | val inputLayer = InputLayer[Vol](netInputDim) 149 | type ConvLayerCombo = (Convolution, Relu[Vol]) 150 | type FullyConnectedLayerCombo = (FullyConnected[_], Relu[RowVector]) 151 | 152 | def flatten(seq: Seq[(HiddenLayer, HiddenLayer)]): Seq[HiddenLayer] = seq.flatMap(p ⇒ Seq(p._1, p._2)) 153 | 154 | val convolutions = (1 to settings.numOfConvolutions).foldLeft(Vector.empty[ConvLayerCombo]) { (convs, _) ⇒ 155 | val iDim: ThreeD = convs.lastOption.map(_._2.outDimension).getOrElse(netInputDim) 156 | val conv = Convolution( 157 | numOfFilters = settings.numOfFilters, 158 | filterSize = Rectangle(settings.filterSize, 1), 159 | inputDimension = iDim, 160 | padding = true 161 | ) 162 | val relu = Relu[Vol](conv.outDimension) 163 | convs :+ ((conv, relu)) 164 | } 165 | 166 | val midFc = FullyConnected[Vol](convolutions.last._2.outDimension, settings.numOfFullyConnectedNeurons) 167 | val midRelu = Relu[RowVector](midFc.outDimension) 168 | 169 | val fullyConnecteds = (1 until settings.numOfFullyConnected).foldLeft(Vector[FullyConnectedLayerCombo]((midFc, midRelu))) { (fcs, _) ⇒ 170 | val fc = FullyConnected(fcs.last._2.outDimension.size, settings.numOfFullyConnectedNeurons) 171 | val relu = Relu[RowVector](fc.outDimension) 172 | fcs :+ ((fc, relu)) 173 | } 174 | 175 | val lastFc = FullyConnected(fullyConnecteds.last._2.outDimension.totalSize, numOfActions) 176 | val lastRelu = Relu[RowVector](lastFc.outDimension) 177 | 178 | val lossLayer = Regression(numOfActions) 179 | DefaultNet(inputLayer, flatten(convolutions) ++ flatten(fullyConnecteds :+ ((lastFc, lastRelu))), lossLayer) 180 | } 181 | 182 | private def netInputDimension(inputDimension: Row): ThreeD = ThreeD(historyLength, 1, inputDimension.size) 183 | } 184 | 185 | object ConvolutionBased { 186 | case class Settings( 187 | historyLength: Int = 50, 188 | filterSize: Int = 5, 189 | gamma: Double = 0.95, 190 | batchSize: Int = 20, 191 | numOfFilters: Int = 10, 192 | numOfFullyConnectedNeurons: Int = 30, 193 | numOfConvolutions: Int = 2, 194 | numOfFullyConnected: Int = 2, 195 | targetNetUpdateFreq: Int = 10, //avg # of iterations before updating the target net 196 | minMemorySizeBeforeTraining: Int = 100 197 | ) 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /reinforcement-learning/src/main/scala/glaux/reinforcement/Policy.scala: -------------------------------------------------------------------------------- 1 | package glaux.reinforcementlearning 2 | 3 | import glaux.reinforcementlearning.Policy.{AnnealingContext, DecisionContext} 4 | import glaux.reinforcementlearning.QLearner.State 5 | import glaux.statistics.Probability 6 | 7 | import scala.util.Random 8 | 9 | trait Policy[StateT <: State[_]] { 10 | type Context <: DecisionContext 11 | type QFunction = (StateT, Action) ⇒ Q 12 | def numOfActions: Int 13 | 14 | def decide(state: StateT, qFunction: QFunction, context: Context): (Action, Context) 15 | 16 | def init: Context 17 | 18 | } 19 | 20 | object Policy { 21 | trait DecisionContext 22 | type NumberOfSteps = Int 23 | 24 | case class Annealing[StateT <: State[_]]( 25 | numOfActions: Action, 26 | minExploreProbability: Probability, 27 | lengthOfExploration: NumberOfSteps 28 | ) extends Policy[StateT] { 29 | type Context = AnnealingContext 30 | def decide(state: StateT, qFunction: QFunction, context: Context): (Action, Context) = { 31 | 32 | def actionWithMaxQ = (0 until numOfActions).map(qFunction(state, _)).zipWithIndex.maxBy(_._1)._2 33 | 34 | val explorationProbability = if (context.explorationProbability > minExploreProbability) { 35 | context.explorationProbability - ((context.explorationProbability - minExploreProbability) / context.stepsLeftForExploration) 36 | } else minExploreProbability 37 | val action = if (explorationProbability.nextBoolean()) 38 | Random.nextInt(numOfActions) 39 | else 40 | actionWithMaxQ 41 | 42 | (action, AnnealingContext(explorationProbability, Math.max(context.stepsLeftForExploration - 1, 0))) 43 | } 44 | 45 | def init: AnnealingContext = AnnealingContext(Probability(1), lengthOfExploration) 46 | } 47 | 48 | case class AnnealingContext( 49 | explorationProbability: Probability, 50 | stepsLeftForExploration: NumberOfSteps 51 | ) extends DecisionContext 52 | 53 | } 54 | -------------------------------------------------------------------------------- /reinforcement-learning/src/main/scala/glaux/reinforcement/QAgent.scala: -------------------------------------------------------------------------------- 1 | package glaux.reinforcementlearning 2 | 3 | import glaux.linearalgebra.{Vol, RowVector} 4 | import glaux.neuralnetwork.Net 5 | import glaux.neuralnetwork.Net.{DefaultNet, Updater} 6 | import glaux.neuralnetwork.trainers.VanillaSGD 7 | import glaux.neuralnetwork.trainers.SGD.SGDSettings 8 | import glaux.reinforcementlearning 9 | import glaux.reinforcementlearning.DeepMindQLearner.{ConvolutionBased, Simplified} 10 | import glaux.reinforcementlearning.Policy.DecisionContext 11 | import glaux.reinforcementlearning.QAgent.{Session ⇒ QSession} 12 | import glaux.reinforcementlearning.QLearner.{Observation, TemporalState} 13 | 14 | import scala.util.Random 15 | 16 | trait QAgent { 17 | type Learner <: QLearner 18 | val qLearner: Learner 19 | 20 | import qLearner.{Iteration, State} 21 | 22 | type Policy <: reinforcementlearning.Policy[State] 23 | 24 | val numOfActions: Int 25 | val policy: Policy 26 | type Session = QSession[Iteration, policy.Context] 27 | 28 | protected def readingsToInput(readings: Seq[Double]): qLearner.Input 29 | 30 | protected def toHistory(readings: Iterable[Reading]): qLearner.History = 31 | readings.toSeq.map { 32 | case (r, time) ⇒ TemporalState(readingsToInput(r), time) 33 | } 34 | 35 | def start(initReadings: Iterable[Reading], previous: Option[Session]): Either[String, Session] = { 36 | val initHistory = toHistory(initReadings) 37 | 38 | if (qLearner.canBuildStateFrom(initHistory)) 39 | Right(QSession( 40 | iteration = previous.map(_.iteration).getOrElse(qLearner.init(initHistory, numOfActions)), 41 | currentReward = 0, 42 | currentReadings = initReadings.toVector, 43 | decisionContext = policy.init 44 | )) 45 | else 46 | Left("Not enough initial history to start a session") 47 | } 48 | 49 | def report(reading: Reading, reward: Reward, session: Session): Session = { 50 | assert(!session.isClosed) 51 | session.copy( 52 | currentReadings = session.currentReadings :+ reading, 53 | currentReward = session.currentReward + reward 54 | ) 55 | } 56 | 57 | def requestAction(session: Session): (Action, Session) = { 58 | assert(!session.isClosed) 59 | val currentHistory = toHistory(session.currentReadings) 60 | session.status match { 61 | case session.Status.ReadyToForward ⇒ 62 | val newIteration = forward(session, false) 63 | val (action, decisionContext) = policy.decide(newIteration.state, newIteration.stateActionQ, session.decisionContext) 64 | (action, QSession(newIteration, 0, Vector.empty, decisionContext, Some(action))) 65 | 66 | case session.Status.PendingFirstAction ⇒ 67 | val currentState = qLearner.stateFromHistory(currentHistory, false) 68 | val (firstAction, decisionContext) = policy.decide(currentState, session.iteration.stateActionQ, session.decisionContext) 69 | (firstAction, session.copy( 70 | lastAction = Some(firstAction), 71 | currentReadings = Vector.empty, 72 | decisionContext = decisionContext 73 | )) 74 | 75 | case session.Status.PendingReadingAfterAction ⇒ 76 | (session.lastAction.get, session) //simply repeat the last action 77 | 78 | case _ ⇒ throw new NotImplementedError(s"request action not implemented for ${session.status}") 79 | } 80 | } 81 | 82 | def close(session: Session): Session = { 83 | assert(!session.isClosed) 84 | if (session.canForward) { 85 | session.copy(iteration = forward(session, true), isClosed = true) 86 | } else 87 | session.copy(isClosed = true) 88 | } 89 | 90 | private def forward(session: Session, terminal: Boolean): Iteration = { 91 | val observation = Observation(session.lastAction.get, session.currentReward, toHistory(session.currentReadings), terminal) 92 | qLearner.iterate(session.iteration, observation).asInstanceOf[Iteration] 93 | } 94 | } 95 | 96 | object QAgent { 97 | case class Session[IterationT <: QLearner#IterationLike, PolicyContextT <: Policy.DecisionContext]( 98 | iteration: IterationT, 99 | currentReward: Reward, 100 | currentReadings: Vector[Reading], 101 | decisionContext: PolicyContextT, 102 | lastAction: Option[Action] = None, 103 | isClosed: Boolean = false 104 | ) { 105 | def canForward: Boolean = !isClosed && lastAction.isDefined && !currentReadings.isEmpty 106 | 107 | object Status extends Enumeration { 108 | val ReadyToForward, PendingReadingAfterAction, PendingFirstAction, Closed = Value 109 | } 110 | 111 | def status: Status.Value = if (isClosed) Status.Closed 112 | else if (lastAction.isDefined) 113 | if (currentReadings.isEmpty) 114 | Status.PendingReadingAfterAction 115 | else 116 | Status.ReadyToForward 117 | else if (currentReadings.isEmpty) 118 | throw new Exception("session should not be in this state") 119 | else 120 | Status.PendingFirstAction 121 | } 122 | } 123 | 124 | trait DeepMindQAgent[LT <: DeepMindQLearner] extends QAgent { 125 | type Learner = LT 126 | import qLearner.State 127 | 128 | type Policy = Policy.Annealing[State] 129 | val policy: Policy = Policy.Annealing[State](numOfActions, 0.05, 10000) 130 | implicit val updater: Net.Updater[LT#Net] 131 | } 132 | 133 | case class SimpleQAgent(numOfActions: Int, historyLength: Int = 10) extends DeepMindQAgent[DeepMindQLearner.Simplified] { 134 | 135 | val trainer = VanillaSGD[Learner#Net](SGDSettings(learningRate = 0.05)) 136 | 137 | val qLearner = DeepMindQLearner.Simplified(historyLength = historyLength, batchSize = 20, trainer = trainer) 138 | 139 | protected def readingsToInput(readings: Seq[Double]): qLearner.Input = RowVector(readings: _*) 140 | implicit lazy val updater = implicitly[Updater[DefaultNet[RowVector]]] 141 | 142 | } 143 | 144 | case class AdvancedQAgent( 145 | numOfActions: Int, 146 | learnerSettings: ConvolutionBased.Settings, 147 | trainerSettings: SGDSettings 148 | ) extends DeepMindQAgent[DeepMindQLearner.ConvolutionBased] { 149 | 150 | val trainer = VanillaSGD[Learner#Net](trainerSettings) 151 | 152 | val qLearner = ConvolutionBased(trainer, learnerSettings) 153 | 154 | protected def readingsToInput(readings: Seq[Double]): qLearner.Input = RowVector(readings: _*) 155 | implicit lazy val updater = implicitly[Updater[DefaultNet[Vol]]] 156 | 157 | } 158 | -------------------------------------------------------------------------------- /reinforcement-learning/src/main/scala/glaux/reinforcement/QLearner.scala: -------------------------------------------------------------------------------- 1 | package glaux 2 | package reinforcementlearning 3 | 4 | import glaux.linearalgebra.Tensor 5 | import glaux.neuralnetwork.Loss 6 | import glaux.neuralnetwork.trainers.BatchTrainer 7 | import glaux.reinforcementlearning.QLearner.{History ⇒ QHistory, Observation ⇒ QObservation, State ⇒ QState, Transition} 8 | 9 | trait QLearner { 10 | type NetInput <: Tensor 11 | type Input <: Tensor 12 | type Net <: neuralnetwork.Net { type Input = NetInput } //Need to fix input to the type level 13 | 14 | type Trainer <: BatchTrainer[Net, _] 15 | type NetOutput = Net#Output 16 | 17 | protected val trainer: Trainer 18 | 19 | type TrainingResult = trainer.BatchResult 20 | 21 | type History = QHistory[Input] 22 | 23 | type InputDimension = Input#Dimensionality 24 | 25 | def historyLength: Int 26 | 27 | type Observation = QObservation[Input] 28 | type State = QState[Input] 29 | 30 | type Memory = Seq[Transition[Input]] 31 | 32 | trait IterationLike { 33 | def trainingResult: TrainingResult 34 | def net: Net 35 | def memory: Memory 36 | def isTerminal: Boolean 37 | def state: State 38 | 39 | lazy val actionQs: Map[Action, Q] = qMap(state) 40 | lazy val loss: Loss = trainingResult.lossInfo.cost 41 | 42 | def stateActionQ(s: State = state, action: Action): Q = { 43 | assert(!s.isTerminal) 44 | qMap(s).apply(action) 45 | } 46 | 47 | private def qMap(s: State): Map[Action, Q] = if (state.isTerminal) Map.empty else net.predict(s).seqView.zipWithIndex.map(_.swap).toMap 48 | 49 | } 50 | 51 | type Iteration <: IterationLike 52 | 53 | implicit protected def inputToNet(state: State): NetInput 54 | 55 | def iterate(lastIteration: Iteration, observation: Observation): Iteration = { 56 | assert( 57 | observation.recentHistory.forall(_.readings.dimension == lastIteration.state.inputDimension), 58 | s"input readings doesn't conform to preset reading dimension ${lastIteration.state.inputDimension}" 59 | ) 60 | 61 | val relevantHistory = if (lastIteration.isTerminal) observation.recentHistory else concat(lastIteration.state.fullHistory, observation.recentHistory) 62 | 63 | val currentState = stateFromHistory(relevantHistory, observation.isTerminal) 64 | 65 | doIterate(lastIteration, observation, currentState) 66 | 67 | } 68 | 69 | protected def doIterate(lastIteration: Iteration, observation: Observation, currentState: State): Iteration 70 | /** 71 | * 72 | * @param initHistory initial history to construct the fist state, MUST NOT BE Terminal 73 | * @param numOfActions 74 | * @return 75 | */ 76 | def init(initHistory: History, numOfActions: Int): Iteration = 77 | doInit(stateFromHistory(initHistory, false), numOfActions, inputDimensionOfHistory(initHistory).get) 78 | 79 | protected def doInit(initState: State, numOfActions: Action, inputDim: InputDimension): Iteration 80 | 81 | protected def concat(previous: History, newHistory: History): History = { 82 | val relevantPreviousHistory = previous.filter(_.time.isBefore(newHistory.head.time)) 83 | (relevantPreviousHistory ++ newHistory) 84 | } 85 | 86 | protected def inputDimensionOfHistory(history: History): Option[InputDimension] = 87 | history.headOption.flatMap { head ⇒ 88 | val inputDim = head.readings.dimension 89 | if (history.map(_.readings).exists(_.dimension != inputDim)) None else Some(inputDim) 90 | } 91 | 92 | private[reinforcementlearning] def stateFromHistory(history: History, isTerminal: Boolean): State = { 93 | assert(canBuildStateFrom(history), "incorrect history length or dimension to create a state") 94 | QState(history.takeRight(historyLength), isTerminal) 95 | } 96 | 97 | def canBuildStateFrom(history: History): Boolean = { 98 | assert(inputDimensionOfHistory(history).isDefined, "history has inconsistent dimension") 99 | history.size >= historyLength 100 | } 101 | 102 | } 103 | 104 | object QLearner { 105 | 106 | case class TemporalState[Input <: Tensor](readings: Input, time: Time) 107 | 108 | type History[Input <: Tensor] = Seq[TemporalState[Input]] 109 | 110 | case class Observation[Input <: Tensor]( 111 | lastAction: Action, 112 | reward: Reward, 113 | recentHistory: History[Input], 114 | isTerminal: Boolean 115 | ) { 116 | assert(!recentHistory.isEmpty, "Cannot create an observation without recent history") 117 | def startTime = recentHistory.head.time 118 | 119 | } 120 | 121 | case class State[Input <: Tensor](fullHistory: History[Input], isTerminal: Boolean) { 122 | def endTime = fullHistory.last.time 123 | lazy val inputDimension: Input#Dimensionality = fullHistory.head.readings.dimension 124 | } 125 | 126 | case class Transition[Input <: Tensor]( 127 | before: State[Input], 128 | action: Action, 129 | reward: Reward, 130 | after: State[Input] 131 | ) 132 | 133 | } 134 | 135 | -------------------------------------------------------------------------------- /reinforcement-learning/src/main/scala/glaux/reinforcement/package.scala: -------------------------------------------------------------------------------- 1 | package glaux 2 | 3 | import java.time.ZonedDateTime 4 | 5 | package object reinforcementlearning { 6 | 7 | type Action = Int 8 | 9 | type Reward = Double 10 | 11 | type Q = Double 12 | 13 | type Time = ZonedDateTime 14 | 15 | type Reading = (Seq[Double], Time) 16 | 17 | } 18 | -------------------------------------------------------------------------------- /reinforcement-learning/src/test/scala/glaux/reinforcementlearning/AnnealingPolicySpec.scala: -------------------------------------------------------------------------------- 1 | package glaux.reinforcementlearning 2 | 3 | import glaux.linearalgebra.RowVector 4 | import glaux.reinforcementlearning.Policy.{AnnealingContext, Annealing} 5 | import glaux.reinforcementlearning.QLearner.State 6 | import org.specs2.mutable.Specification 7 | import glaux.statistics.Probability 8 | 9 | class AnnealingPolicySpec extends Specification { 10 | val policy = Annealing[State[RowVector]](4, 0.5, 10000) 11 | import policy.QFunction 12 | 13 | val aState: State[RowVector] = State(Nil, false) 14 | val aQFunc: QFunction = (_, _) ⇒ 0 15 | 16 | "reduce exploreProbability until no steps left for exploration" >> { 17 | val stepsLeft = 10 18 | val initContext = AnnealingContext(policy.minExploreProbability + 0.3, stepsLeft) 19 | 20 | val lastContext = (1 until stepsLeft).foldLeft(initContext) { (ctx, _) ⇒ 21 | val (_, resultContext) = policy.decide(aState, aQFunc, ctx) 22 | resultContext.explorationProbability must be_>(policy.minExploreProbability) 23 | resultContext.explorationProbability must be_<(ctx.explorationProbability) 24 | resultContext.stepsLeftForExploration must be_<(ctx.stepsLeftForExploration) 25 | resultContext 26 | } 27 | 28 | lastContext.stepsLeftForExploration === 1 29 | val (_, finalContext) = policy.decide(aState, aQFunc, lastContext) 30 | finalContext.explorationProbability === policy.minExploreProbability 31 | 32 | } 33 | 34 | "keep at min exploration Rate after exploration period" >> { 35 | val lastCtx = AnnealingContext(policy.minExploreProbability, 0) 36 | val (_, result) = policy.decide(aState, aQFunc, lastCtx) 37 | result.explorationProbability === policy.minExploreProbability 38 | } 39 | 40 | "pick the action with Max Q when exploration rate is zero" >> { 41 | val policy = Annealing[State[RowVector]](4, 0d, 10000) 42 | val lastCtx = AnnealingContext(0d, 0) 43 | val pick3: QFunction = (_, a) ⇒ if (a == 3) 10 else 2 44 | 45 | val actions = (1 to 100).map(_ ⇒ policy.decide(aState, pick3, lastCtx)._1).distinct 46 | actions.length === 1 47 | actions.head === 3 48 | } 49 | 50 | "pick random action during exploratio time" >> { 51 | 52 | val initContext = AnnealingContext(1d, 100000) 53 | 54 | val actions = (1 to 100).scanLeft((0, initContext)) { (p, _) ⇒ 55 | policy.decide(aState, aQFunc, p._2) 56 | }.map(_._1).distinct 57 | 58 | actions.length === 4 //all four actions are used 59 | 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /reinforcement-learning/src/test/scala/glaux/reinforcementlearning/ConvolutionBasedSpec.scala: -------------------------------------------------------------------------------- 1 | package glaux.reinforcementlearning 2 | 3 | import java.time.{Clock, LocalTime, LocalDate, ZonedDateTime} 4 | 5 | import glaux.linearalgebra.Dimension.Row 6 | import glaux.linearalgebra.RowVector 7 | import glaux.reinforcementlearning.DeepMindQLearner.ConvolutionBased 8 | import glaux.reinforcementlearning.QLearner._ 9 | import org.specs2.mutable.Specification 10 | 11 | import scala.util.Random 12 | 13 | class ConvolutionBasedSpec extends Specification { 14 | val start = ZonedDateTime.of(2015, 2, 14, 14, 30, 0, 0, Clock.systemDefaultZone().getZone) 15 | val learner = ConvolutionBased(settings = ConvolutionBased.Settings(historyLength = 10, filterSize = 3, batchSize = 3, minMemorySizeBeforeTraining = 10)) 16 | val numOfAction = 3 17 | def mockHistory(from: ZonedDateTime, length: Int = 10, numOfReadings: Int = 5) = { 18 | def randomReading = RowVector((0 until numOfReadings).map(_ ⇒ Random.nextDouble()): _*) 19 | 20 | (0 until length).map { i ⇒ 21 | TemporalState(randomReading, from.plusMinutes(i)) 22 | } 23 | } 24 | 25 | def mockObservation(minuteFromStart: Int) = Observation( 26 | lastAction = Random.nextInt(numOfAction), 27 | reward = if (Random.nextBoolean) Random.nextInt(5) else 0, 28 | recentHistory = mockHistory(start.plusMinutes(minuteFromStart)), 29 | isTerminal = false 30 | ) 31 | 32 | "init" >> { 33 | val initIter = learner.init(mockHistory(start), numOfAction) 34 | initIter.net.outputDimension === Row(numOfAction) 35 | } 36 | 37 | "training" >> { 38 | val init = learner.init(mockHistory(start), numOfAction) 39 | 40 | "one iteration" >> { 41 | val iter = learner.iterate(init, Observation( 42 | lastAction = 0, 43 | reward = 1, 44 | recentHistory = mockHistory(start.plusMinutes(20)), 45 | isTerminal = false 46 | )) 47 | iter.memory.size === 1 48 | init.trainingResult.batchSize === 0 49 | } 50 | 51 | "start training after minMemorySizeBeforeTraining of meaningful iteration" >> { 52 | val result = (0 to (learner.minMemorySizeBeforeTraining + 1)).foldLeft(init) { (iter, i) ⇒ 53 | learner.iterate(iter, mockObservation(i * learner.historyLength + 1)) 54 | } 55 | result.trainingResult.batchSize === learner.batchSize 56 | } 57 | 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /reinforcement-learning/src/test/scala/glaux/reinforcementlearning/QAgentSpec.scala: -------------------------------------------------------------------------------- 1 | package glaux.reinforcementlearning 2 | 3 | import java.time.ZonedDateTime 4 | 5 | import glaux.linearalgebra.RowVector 6 | import glaux.neuralnetwork.trainers.VanillaSGD 7 | import glaux.neuralnetwork.trainers.SGD.SGDSettings 8 | import glaux.reinforcementlearning.DeepMindQLearner.Simplified 9 | import glaux.reinforcementlearning.Policy.DecisionContext 10 | import org.specs2.matcher.Scope 11 | import org.specs2.mutable.Specification 12 | 13 | class QAgentSpec extends Specification { 14 | trait QAgentScope extends Scope { 15 | case class TestAgent(numOfActions: Int, fixedReturnAction: Int) extends QAgent { 16 | self ⇒ 17 | type Learner = DeepMindQLearner.Simplified 18 | val trainer = VanillaSGD[Simplified#Net](SGDSettings(learningRate = 0.05)) 19 | val qLearner = DeepMindQLearner.Simplified(historyLength = 3, batchSize = 20, trainer = trainer) 20 | type Policy = glaux.reinforcementlearning.Policy[qLearner.State] 21 | 22 | val policy: Policy = new glaux.reinforcementlearning.Policy[qLearner.State] { 23 | type Context = DecisionContext 24 | def init: Context = new DecisionContext {} 25 | def numOfActions: Action = self.numOfActions 26 | def decide(state: qLearner.State, qFunction: QFunction, context: Context): (Action, Context) = (fixedReturnAction, init) 27 | } 28 | 29 | protected def readingsToInput(readings: Seq[Reward]): RowVector = RowVector(readings: _*) 30 | } 31 | 32 | lazy val agent = TestAgent(3, 2) 33 | } 34 | 35 | "Start" should { 36 | 37 | "does not start without enough history" in new QAgentScope { 38 | val result = agent.start(List((Seq(3d, 2d), ZonedDateTime.now)), None) 39 | result must beLeft[String] 40 | } 41 | 42 | "start with enough history" in new QAgentScope { 43 | val reading = (Seq(3d, 2d), ZonedDateTime.now) 44 | val result = agent.start(List(reading, reading, reading), None) 45 | result must beRight 46 | } 47 | } 48 | 49 | "Request Action Without Report" should { 50 | trait RequestActionScope extends QAgentScope { 51 | val testReading = (Seq(3d, 2d), ZonedDateTime.now) 52 | val result = agent.start(List(testReading, testReading, testReading), None) 53 | val session = result.right.get 54 | val (action, newSession) = agent.requestAction(session) 55 | } 56 | 57 | "return the action from policy" in new RequestActionScope { 58 | action === 2 59 | } 60 | 61 | "without empty recentHistory" in new RequestActionScope { 62 | newSession.currentReadings must beEmpty 63 | } 64 | 65 | "remember the action from policy" in new RequestActionScope { 66 | newSession.lastAction must beSome(action) 67 | } 68 | 69 | "not create memory for the first action request" in new RequestActionScope { 70 | newSession.iteration.memory must beEmpty 71 | } 72 | } 73 | 74 | "Request Action After Report" should { 75 | trait ReportScope extends QAgentScope { 76 | val testReading = (Seq(3d, 2d), ZonedDateTime.now) 77 | val start = agent.start(List(testReading, testReading, testReading), None) 78 | val startSession = start.right.get 79 | val (_, s) = agent.requestAction(startSession) 80 | val session = agent.report((Seq(9d, 5d), ZonedDateTime.now.plusMinutes(1)), 7, s) 81 | 82 | } 83 | 84 | "add reading and reward to the memory" in new ReportScope { 85 | val (_, result) = agent.requestAction(session) 86 | val memory = result.iteration.memory 87 | 88 | memory.size === 1 89 | memory.head.after.fullHistory.last.readings.seqView === Seq(9d, 5d) 90 | memory.head.reward === 7 91 | } 92 | 93 | "consecutive requests returns same action without changing session " in new ReportScope { 94 | val (a1, s1) = agent.requestAction(session) 95 | val (a2, s2) = agent.requestAction(s1) 96 | 97 | a1 === a2 98 | s1 === s2 99 | } 100 | } 101 | 102 | "close" should { 103 | trait CloseScope extends QAgentScope { 104 | val testReading = (Seq(3d, 2d), ZonedDateTime.now) 105 | val start = agent.start(List(testReading, testReading, testReading), None) 106 | val startSession = start.right.get 107 | val (_, session) = agent.requestAction(startSession) 108 | } 109 | 110 | "return a session that is closed" in new CloseScope { 111 | agent.close(session).isClosed must beTrue 112 | } 113 | 114 | "does not create extra memory if no further readings is provided" in new CloseScope { 115 | agent.close(session).iteration.memory must beEmpty 116 | } 117 | 118 | "creates extra memory if further readings is provided" in new CloseScope { 119 | 120 | val s = agent.report((Seq(9d, 5d), ZonedDateTime.now.plusMinutes(1)), 7, session) 121 | val closed = agent.close(s) 122 | 123 | closed.isClosed must beTrue 124 | val memory = closed.iteration.memory 125 | memory.size === 1 126 | memory.head.after.fullHistory.last.readings.seqView === Seq(9d, 5d) 127 | memory.head.reward === 7 128 | } 129 | } 130 | } 131 | 132 | -------------------------------------------------------------------------------- /reinforcement-learning/src/test/scala/glaux/reinforcementlearning/SimplifiedDeepMindQLearnerSpec.scala: -------------------------------------------------------------------------------- 1 | package glaux 2 | package reinforcementlearning 3 | 4 | import java.time.{Clock, LocalTime, LocalDate, ZonedDateTime} 5 | 6 | import glaux.linearalgebra.Dimension.Row 7 | import glaux.linearalgebra.RowVector 8 | import glaux.reinforcementlearning.QLearner._ 9 | 10 | import org.specs2.mutable.Specification 11 | 12 | class SimplifiedDeepMindQLearnerSpec extends Specification { 13 | val start = ZonedDateTime.of(LocalDate.of(2015, 2, 14), LocalTime.of(14, 30), Clock.systemDefaultZone().getZone) 14 | val learner = DeepMindQLearner.Simplified(historyLength = 2) 15 | 16 | val initHistory = Seq(TemporalState(RowVector(1, 0), start), TemporalState(RowVector(2, 1), start.plusMinutes(1))) 17 | 18 | val init = learner.init(initHistory, 2) 19 | 20 | "init" >> { 21 | 22 | "memory" >> { 23 | init.memory must beEmpty 24 | } 25 | 26 | "net" >> { 27 | init.net.inputDimension must_== (Row(4)) 28 | init.net.outputDimension must_== (Row(2)) 29 | } 30 | 31 | "trainResult" >> { 32 | init.trainingResult.batchSize === 0 33 | } 34 | 35 | "recentHistory" >> { 36 | init.state.fullHistory.size === 2 37 | } 38 | 39 | } 40 | 41 | "initial iterations" >> { 42 | 43 | val thirdIter = learner.iterate(init, Observation( 44 | lastAction = 1, 45 | reward = 1, 46 | recentHistory = Seq(TemporalState(RowVector(3, 2), start.plusMinutes(2))), 47 | isTerminal = true 48 | )) 49 | 50 | "third iter" >> { 51 | "has enough history for memory" >> { 52 | thirdIter.memory.size === 1 53 | } 54 | 55 | "build the trainsition" >> { 56 | val transition = thirdIter.memory.head 57 | transition.before.fullHistory.map(_.readings) must_== Seq(RowVector(1, 0), RowVector(2, 1)) 58 | transition.after.fullHistory.map(_.readings) must_== Seq(RowVector(2, 1), RowVector(3, 2)) 59 | transition.action === 1 60 | transition.reward === 1 61 | } 62 | 63 | "keep only relevant recentHistory" >> { 64 | thirdIter.state.fullHistory.size === 2 65 | } 66 | 67 | "latest state is terminal" >> { 68 | thirdIter.state.isTerminal must beTrue 69 | } 70 | 71 | } 72 | 73 | val fourth = learner.iterate(thirdIter, Observation( 74 | lastAction = 1, 75 | reward = 1, 76 | recentHistory = Seq( 77 | TemporalState(RowVector(4, 3), start.plusMinutes(3)), 78 | TemporalState(RowVector(4, 3), start.plusMinutes(4)) 79 | ), 80 | isTerminal = false 81 | )) 82 | 83 | "fourth iter" >> { 84 | "add to memory because last one is terminated" >> { 85 | fourth.memory.size === 1 86 | } 87 | } 88 | 89 | val fifth = learner.iterate(fourth, Observation( 90 | lastAction = 3, 91 | reward = 2, 92 | recentHistory = Seq( 93 | TemporalState(RowVector(5, 4), start.plusMinutes(5)), 94 | TemporalState(RowVector(5, 4), start.plusMinutes(6)) 95 | ), 96 | isTerminal = false 97 | )) 98 | 99 | "fifth iter" >> { 100 | "build new memory" >> { 101 | fifth.memory.size === 2 102 | } 103 | 104 | "build the transition" >> { 105 | val transition = fifth.memory.last 106 | transition.before.fullHistory.map(_.readings) must_== Seq(RowVector(4, 3), RowVector(4, 3)) 107 | transition.after.fullHistory.map(_.readings) must_== Seq(RowVector(5, 4), RowVector(5, 4)) 108 | transition.action === 3 109 | transition.reward === 2 110 | } 111 | 112 | } 113 | 114 | } 115 | 116 | } 117 | -------------------------------------------------------------------------------- /reinforcement-learning/src/test/scala/glaux/reinforcementlearning/integration/SimplifiedIntegration.scala: -------------------------------------------------------------------------------- 1 | package glaux.reinforcementlearning.integration 2 | 3 | import java.time.{Clock, LocalTime, LocalDate, ZonedDateTime} 4 | 5 | import glaux.linearalgebra.RowVector 6 | import glaux.neuralnetwork.trainers.VanillaSGD 7 | import glaux.neuralnetwork.trainers.SGD.SGDSettings 8 | import glaux.reinforcementlearning.DeepMindQLearner.Simplified 9 | import glaux.reinforcementlearning.QLearner._ 10 | import glaux.reinforcementlearning.{Time, Action, DeepMindQLearner} 11 | import org.specs2.mutable.Specification 12 | 13 | import scala.util.Random 14 | 15 | class SimplifiedIntegration extends Specification { 16 | 17 | val start = ZonedDateTime.of(LocalDate.of(2015, 2, 14), LocalTime.of(14, 30), Clock.systemDefaultZone().getZone) 18 | // val trainer = MomentumSGD[Simplified#Net](MomentumSGDOptions(SGDOptions(learningRate = 0.005), momentum = 0.9)) 19 | val trainer = VanillaSGD[Simplified#Net](SGDSettings(learningRate = 0.05)) 20 | val learner = DeepMindQLearner.Simplified(historyLength = 2, batchSize = 20, trainer = trainer) 21 | import learner.{State, History} 22 | 23 | def randomBinary = if (Random.nextBoolean) 1 else 0 24 | def randomReading = RowVector(randomBinary, randomBinary, randomBinary) 25 | 26 | def randomHistory(from: Time): History = Seq( 27 | TemporalState(randomReading, from), 28 | TemporalState(randomReading, from.plusMinutes(1)), 29 | TemporalState(randomReading, from.plusMinutes(2)) 30 | ) 31 | def randomTerminal: Boolean = Random.nextDouble > 0.97 32 | 33 | def newObservation(lastState: State, lastAction: Action): learner.Observation = { 34 | val time = lastState.endTime.plusMinutes(1) 35 | val reward = { 36 | if (lastState.isTerminal) 0 else { 37 | //reward when action matches the reading, that is, sum of three readings in the index exceed certain threshed 38 | if (lastState.fullHistory.takeRight(2).map(_.readings(lastAction)).sum > 1.5) 1.0 else 0 39 | } 40 | } 41 | Observation(lastAction, reward, randomHistory(time), randomTerminal) 42 | } 43 | 44 | val init = learner.init(randomHistory(start), 3) 45 | 46 | "can learn the right action" >> { 47 | //learning 48 | val lastIter = (1 to 500).foldLeft(init) { (lastIteration, _) ⇒ 49 | val obs = newObservation(lastIteration.state, Random.nextInt(3)) 50 | learner.iterate(lastIteration, obs) 51 | } 52 | 53 | val testSize = 100 54 | //testing 55 | val results = (1 to testSize).scanLeft(lastIter) { (lastIteration, _) ⇒ 56 | val result = learner.iterate(lastIteration, newObservation(lastIteration.state, Random.nextInt(3))) 57 | 58 | result 59 | }.filterNot(_.actionQs.isEmpty) 60 | 61 | val correct = results.filter { result ⇒ 62 | val cumReading = result.state.fullHistory.map(_.readings).takeRight(2).reduce(_ + _).seqView 63 | val correctActions = cumReading.zipWithIndex.filter(_._1 > 1.5).map(_._2) 64 | val predictedAction = result.actionQs.maxBy(_._2)._1 65 | // println(cumReading.map( v => (v * 10).toInt)) 66 | // println(result.actionQs.mapValues(v => (v * 100).toInt )) 67 | correctActions.contains(predictedAction) || correctActions.isEmpty 68 | } 69 | val correctionRate = correct.size.toDouble / results.size 70 | // println("correction rate " + correctionRate) 71 | correctionRate must be_>=(0.60) 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /statistics/src/main/scala/glaux/statistics/Distribution.scala: -------------------------------------------------------------------------------- 1 | package glaux.statistics 2 | 3 | trait Distribution { 4 | type Value 5 | 6 | def sample(size: Int): Iterable[Value] 7 | def sample: Value = sample(1).head 8 | } 9 | 10 | trait RealDistribution extends Distribution { 11 | type Value = Double 12 | } 13 | 14 | trait DistributionImplementations { 15 | def normal(mean: Double, std: Double): RealDistribution 16 | def uniform(min: Double = 0, max: Double = 1): RealDistribution 17 | } -------------------------------------------------------------------------------- /statistics/src/main/scala/glaux/statistics/impl/apache/ApacheImplementations.scala: -------------------------------------------------------------------------------- 1 | package glaux.statistics.impl.apache 2 | 3 | import glaux.statistics.{RealDistribution, DistributionImplementations} 4 | import org.apache.commons.math3.distribution.{RealDistribution ⇒ ApacheDistribution, UniformRealDistribution, NormalDistribution} 5 | 6 | object ApacheImplementations extends DistributionImplementations { 7 | implicit class ApacheBackedRealDist(ad: ApacheDistribution) extends RealDistribution { 8 | def sample(size: Int) = ad.sample(size) 9 | } 10 | 11 | def normal(mean: Double, std: Double): RealDistribution = new NormalDistribution(mean, std) 12 | 13 | def uniform(min: Double = 0, max: Double = 1): RealDistribution = new UniformRealDistribution(min, max) 14 | } 15 | -------------------------------------------------------------------------------- /statistics/src/main/scala/glaux/statistics/package.scala: -------------------------------------------------------------------------------- 1 | package glaux 2 | 3 | import glaux.statistics.impl.apache.ApacheImplementations 4 | 5 | import scala.util.Random 6 | 7 | package object statistics { 8 | val distributions: DistributionImplementations = ApacheImplementations 9 | 10 | /** 11 | * 12 | * @param value must be between 0 and 1, note this is not enforced due to performance concern. 13 | */ 14 | implicit class Probability(val value: Double) extends AnyVal with Ordered[Probability] { 15 | def +(that: Probability): Probability = value + that.value 16 | def -(that: Probability): Probability = value - that.value 17 | def *(that: Probability): Probability = value * that.value 18 | def /(factor: Double): Probability = value / factor 19 | def nextBoolean(): Boolean = Random.nextDouble() < value 20 | override def toString: String = "Probability of " + value 21 | def compare(that: Probability): Int = value.compare(that.value) 22 | 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /version.sbt: -------------------------------------------------------------------------------- 1 | version in ThisBuild := "0.0.1-SNAPSHOT" 2 | --------------------------------------------------------------------------------