├── .gitignore ├── README.md ├── common ├── .gitignore ├── build.sbt ├── project │ └── build.properties └── src │ └── main │ └── scala │ └── io │ └── ticofab │ └── akkaclusterkubernetes │ └── common │ ├── CustomLogSupport.scala │ ├── Job.scala │ └── JobCompleted.scala ├── k8s-deployment └── service.yaml ├── master ├── .gitignore ├── build.sbt ├── project │ ├── build.properties │ └── plugins.sbt └── src │ └── main │ ├── resources │ └── application.conf │ └── scala │ └── io │ └── ticofab │ └── akkaclusterkubernetes │ ├── AkkaClusterKubernetesMasterApp.scala │ ├── actor │ ├── JobSource.scala │ ├── Master.scala │ ├── Server.scala │ ├── Supervisor.scala │ └── scaling │ │ ├── ControllerMessages.scala │ │ ├── DummyScalingController.scala │ │ └── KubernetesController.scala │ └── config │ └── Config.scala ├── stats-visualizer ├── .gitignore ├── chart.js ├── visualizer.html └── wsscript.js └── worker ├── .gitignore ├── build.sbt ├── project ├── build.properties └── plugins.sbt └── src └── main ├── resources └── application.conf └── scala └── io └── ticofab └── akkaclusterkubernetes ├── AkkaClusterKubernetesWorkerApp.scala ├── Config.scala └── Worker.scala /.gitignore: -------------------------------------------------------------------------------- 1 | logs 2 | target 3 | /.idea 4 | /.idea_modules 5 | /.classpath 6 | /.project 7 | /.settings 8 | /RUNNING_PID 9 | out 10 | 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Akka Cluster Kubernetes 2 | 3 | This project is a working example of achieving Elasticity (in the sense of the [Reactive Manifesto](https://www.reactivemanifesto.org)) using Akka Cluster and Kubernetes. 4 | 5 | Elasticity is the ability of a system to scale its resources up and down according to the present need. We want to use just the right amount: nothing more, nothing less. 6 | 7 | We combine custom resource metrics with logic to automatically adjust the configuration of the underlying cloud infrastructure. 8 | 9 | An alternative example of cluster formation is in my other project [CloudMatch](https://github.com/ticofab/cloudmatch), which uses the `Akka Cluster Bootstrap - Kubernetes API` module for Akka. 10 | 11 | ## Usage 12 | 13 | First, package the app localy in a Docker container: 14 | 15 | ```sbt docker:publishLocal``` 16 | 17 | Upload the image to your Kubernetes project: 18 | 19 | ```gcloud docker -- push :latest``` 20 | 21 | From the `k8s-deployment` folder, start the whole thing with: 22 | 23 | ```kubectl apply -f service.yaml``` 24 | 25 | Now you can access nodes' logs. When done, shut down: 26 | 27 | ```kubectl delete deployment akka-master``` 28 | 29 | ## License 30 | 31 | Copyright 2018 Fabio Tiriticco, Adam Sandor 32 | 33 | Licensed under the Apache License, Version 2.0 (the "License"); 34 | you may not use this file except in compliance with the License. 35 | You may obtain a copy of the License at 36 | 37 | http://www.apache.org/licenses/LICENSE-2.0 38 | 39 | Unless required by applicable law or agreed to in writing, software 40 | distributed under the License is distributed on an "AS IS" BASIS, 41 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 42 | See the License for the specific language governing permissions and 43 | limitations under the License. 44 | 45 | -------------------------------------------------------------------------------- /common/.gitignore: -------------------------------------------------------------------------------- 1 | logs 2 | target 3 | /.idea 4 | /.idea_modules 5 | /.classpath 6 | /.project 7 | /.settings 8 | /RUNNING_PID 9 | out 10 | 11 | -------------------------------------------------------------------------------- /common/build.sbt: -------------------------------------------------------------------------------- 1 | name := "common" 2 | version := "0.0.1" 3 | scalaVersion := "2.12.6" 4 | organization := "ticofab.io" 5 | 6 | libraryDependencies ++= { 7 | val akkaVersion = "2.5.14" 8 | Seq( 9 | 10 | // akka stuff 11 | "com.typesafe.akka" %% "akka-actor" % akkaVersion, 12 | "com.typesafe.akka" %% "akka-stream" % akkaVersion, 13 | "com.typesafe.akka" %% "akka-cluster" % akkaVersion, 14 | 15 | // logging 16 | "org.wvlet.airframe" %% "airframe-log" % "0.51", 17 | 18 | // ficus for config 19 | // https://github.com/iheartradio/ficus 20 | "com.iheart" %% "ficus" % "1.4.3" 21 | ) 22 | } 23 | -------------------------------------------------------------------------------- /common/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 1.2.1 -------------------------------------------------------------------------------- /common/src/main/scala/io/ticofab/akkaclusterkubernetes/common/CustomLogSupport.scala: -------------------------------------------------------------------------------- 1 | package io.ticofab.akkaclusterkubernetes.common 2 | 3 | 4 | import java.time.format.DateTimeFormatter 5 | import java.time.{Instant, LocalDateTime, ZoneId} 6 | import java.util.Properties 7 | 8 | import wvlet.log.{LogFormatter, LogLevel, LogSupport, Logger} 9 | 10 | trait CustomLogSupport extends LogSupport { 11 | Logger.setDefaultFormatter(LogFormatter.BareFormatter) 12 | 13 | // say NO to java mutable object 14 | val logLevels = new Properties() 15 | logLevels.setProperty("sun.net.www.protocol.http", LogLevel.OFF.toString) 16 | logLevels.setProperty("com.google.api.client.http", LogLevel.OFF.toString) 17 | logLevels.setProperty("com.google.datastore.v1.client", LogLevel.OFF.toString) 18 | 19 | sealed trait Severity 20 | case object ERROR extends Severity 21 | case object INFO extends Severity 22 | 23 | Logger.setLogLevels(logLevels) 24 | 25 | private def mapToString(fields: Map[String, String]) = { 26 | val list = fields.map { case (key, value) => "\"" + key + "\" : \"" + value + "\"" }.toList 27 | "{" + list.mkString(",") + "}" 28 | } 29 | 30 | def logJson(s: String, sev: Severity = INFO) = mapToString(Map("msg" -> s, "severity" -> sev.toString)) 31 | 32 | def jsonErrorLog(timestamp: Long, msg: String) = { 33 | val readableDate = getReadableDate(timestamp) 34 | "{\"date\" : \"" + readableDate + "\", \"error\" : \"" + msg + "\"}" 35 | } 36 | 37 | private def getReadableDate(timestamp: Long) = LocalDateTime 38 | .ofInstant(Instant.ofEpochMilli(timestamp), ZoneId.systemDefault()) 39 | .format(DateTimeFormatter.ISO_DATE_TIME) 40 | } -------------------------------------------------------------------------------- /common/src/main/scala/io/ticofab/akkaclusterkubernetes/common/Job.scala: -------------------------------------------------------------------------------- 1 | package io.ticofab.akkaclusterkubernetes.common 2 | 3 | import java.time.LocalDateTime 4 | 5 | import akka.actor.ActorRef 6 | 7 | case class Job(number: Int, creationDate: LocalDateTime, sender: ActorRef) 8 | 9 | object Job { 10 | // how many jobs are completed in a second? 11 | val jobsRatePerSecond = 1 12 | } -------------------------------------------------------------------------------- /common/src/main/scala/io/ticofab/akkaclusterkubernetes/common/JobCompleted.scala: -------------------------------------------------------------------------------- 1 | package io.ticofab.akkaclusterkubernetes.common 2 | 3 | case class JobCompleted(number: Int, completer: String) 4 | -------------------------------------------------------------------------------- /k8s-deployment/service.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | name: akka-master 5 | spec: 6 | selector: 7 | matchLabels: 8 | cluster: cluster1 9 | role: master 10 | replicas: 1 11 | template: 12 | metadata: 13 | labels: 14 | cluster: cluster1 15 | role: master 16 | spec: 17 | terminationGracePeriodSeconds: 3 18 | containers: 19 | - name: akka-master 20 | image: eu.gcr.io/adam-akka/master:latest 21 | imagePullPolicy: Always # this command makes it re-use the latest image every time that a pod spins up 22 | env: 23 | - name: clusterid 24 | value: cluster1 25 | - name: WORKER_IMAGE 26 | value: eu.gcr.io/adam-akka/worker:latest 27 | - name: USE_KUBERNETES 28 | value: "true" 29 | - name: namespace 30 | valueFrom: 31 | fieldRef: 32 | fieldPath: metadata.namespace 33 | ports: 34 | - containerPort: 2551 35 | - containerPort: 8080 36 | --- 37 | apiVersion: v1 38 | kind: Service 39 | metadata: 40 | name: akka-master 41 | labels: 42 | cluster: cluster1 43 | spec: 44 | ports: 45 | - port: 2551 46 | name: akka 47 | type: ClusterIP 48 | selector: 49 | cluster: cluster1 50 | role: master 51 | --- 52 | apiVersion: v1 53 | kind: Service 54 | metadata: 55 | name: master-http 56 | labels: 57 | cluster: cluster1 58 | spec: 59 | ports: 60 | - port: 80 61 | targetPort: 8080 62 | type: LoadBalancer 63 | selector: 64 | cluster: cluster1 65 | role: master 66 | 67 | -------------------------------------------------------------------------------- /master/.gitignore: -------------------------------------------------------------------------------- 1 | logs 2 | target 3 | /.idea 4 | /.idea_modules 5 | /.classpath 6 | /.project 7 | /.settings 8 | /RUNNING_PID 9 | out 10 | 11 | -------------------------------------------------------------------------------- /master/build.sbt: -------------------------------------------------------------------------------- 1 | name := "master" 2 | version := "0.0.1" 3 | scalaVersion := "2.12.6" 4 | organization := "ticofab.io" 5 | 6 | lazy val common = RootProject(file("../common")) 7 | val main = Project(id = "master", base = file(".")).dependsOn(common) 8 | 9 | libraryDependencies ++= { 10 | val circeVersion = "0.9.3" 11 | 12 | Seq( 13 | // akka http for server tuff 14 | "com.typesafe.akka" %% "akka-http" % "10.1.4", 15 | 16 | // json 17 | "io.circe" %% "circe-core" % circeVersion, 18 | "io.circe" %% "circe-generic" % circeVersion, 19 | 20 | // kubernetes stuff 21 | "io.fabric8" % "kubernetes-client" % "3.1.1", 22 | "io.fabric8" % "kubernetes-api" % "3.0.8" 23 | ) 24 | } 25 | 26 | enablePlugins(JavaAppPackaging) 27 | enablePlugins(AshScriptPlugin) 28 | 29 | mainClass in Compile := Some("io.ticofab.akkaclusterkubernetes.AkkaClusterKubernetesMasterApp") 30 | packageName in Docker := "adam-akka/" + name.value 31 | version in Docker := "latest" 32 | dockerLabels := Map("maintainer" -> organization.value, "version" -> version.value) 33 | dockerBaseImage := "openjdk:8-jre" 34 | defaultLinuxInstallLocation in Docker := s"/opt/${name.value}" // to have consistent directory for files 35 | dockerRepository := Some("eu.gcr.io") 36 | -------------------------------------------------------------------------------- /master/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 1.2.1 -------------------------------------------------------------------------------- /master/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.2") 2 | -------------------------------------------------------------------------------- /master/src/main/resources/application.conf: -------------------------------------------------------------------------------- 1 | akka { 2 | 3 | log-dead-letters = 0 4 | 5 | // remote configuration 6 | remote { 7 | enabled-transports = ["akka.remote.netty.tcp"] 8 | netty.tcp { 9 | 10 | // this needs to be set to the DNS name visible in the cluster 11 | // the master's address will be akka-master (dns name created by Service) 12 | // we set the POD_IP env var for workers. they will be accessed by IP address not DNS name so we don't have to create services for them 13 | hostname = "akka-master" 14 | hostname = ${?POD_IP} // set this env var to "0.0.0.0" for local usage 15 | 16 | port = 2551 17 | port = ${?PORT} 18 | 19 | // this needs to be set for remoting within an Docker container 20 | bind-hostname = ${akka.remote.netty.tcp.hostname} 21 | bind-hostname = ${?HOSTNAME} // name of the pod 22 | bind-port = 2551 23 | bind-port = ${?PORT} 24 | } 25 | } 26 | 27 | // enable clustering 28 | actor { 29 | provider = "akka.cluster.ClusterActorRefProvider" 30 | warn-about-java-serializer-usage = false 31 | } 32 | 33 | // cluster configuration 34 | cluster { 35 | 36 | seed-nodes = ["akka.tcp://akka-cluster-kubernetes@"${akka.remote.netty.tcp.hostname}":2551"] 37 | //seed-nodes = ["akka.tcp://akka-cluster-kubernetes@akka-master:2551"] 38 | seed-nodes = ${?SEED_NODES} 39 | min-nr-of-members = 1 40 | 41 | } 42 | 43 | } 44 | 45 | kubernetes { 46 | // this parameter is to enable local testing. local instances need to be spun up manually. 47 | use-kubernetes = true 48 | use-kubernetes = ${?USE_KUBERNETES} 49 | max-replicas = 8 50 | } 51 | -------------------------------------------------------------------------------- /master/src/main/scala/io/ticofab/akkaclusterkubernetes/AkkaClusterKubernetesMasterApp.scala: -------------------------------------------------------------------------------- 1 | package io.ticofab.akkaclusterkubernetes 2 | 3 | import akka.actor.ActorSystem 4 | import akka.cluster.Cluster 5 | import io.ticofab.akkaclusterkubernetes.actor.Supervisor 6 | import io.ticofab.akkaclusterkubernetes.common.CustomLogSupport 7 | import io.ticofab.akkaclusterkubernetes.config.Config 8 | 9 | object AkkaClusterKubernetesMasterApp extends App with CustomLogSupport { 10 | 11 | implicit val as = ActorSystem("akka-cluster-kubernetes") 12 | info(logJson(s"Cluster config: ${Config.cluster}")) 13 | info(logJson(s"Remote config: ${Config.remote}")) 14 | info(logJson(s"Kubernetes config: ${Config.kubernetes}")) 15 | info(logJson(s"This node is a seed node")) 16 | 17 | // create the supervisor actor 18 | as.actorOf(Supervisor(), "supervisor") 19 | 20 | as.registerOnTermination(() => { 21 | info(logJson("Received system termination. Leaving cluster.")) 22 | val cluster = Cluster(as) 23 | cluster.registerOnMemberRemoved(() => as.terminate()) 24 | cluster.leave(cluster.selfAddress) 25 | }) 26 | 27 | } 28 | -------------------------------------------------------------------------------- /master/src/main/scala/io/ticofab/akkaclusterkubernetes/actor/JobSource.scala: -------------------------------------------------------------------------------- 1 | package io.ticofab.akkaclusterkubernetes.actor 2 | 3 | import java.time.LocalDateTime 4 | 5 | import akka.actor.{Actor, ActorRef, Props} 6 | import io.ticofab.akkaclusterkubernetes.common.{CustomLogSupport, Job} 7 | 8 | import scala.concurrent.ExecutionContext.Implicits.global 9 | import scala.concurrent.duration._ 10 | 11 | /** 12 | * Actor to provide a source of input messages for the recipient 13 | * 14 | * @param target The recipient of our messages 15 | */ 16 | class JobSource(target: ActorRef) extends Actor with CustomLogSupport { 17 | 18 | info(logJson(s"job source starting, target is ${target.path.name}")) 19 | 20 | implicit val as = context.system 21 | var counter = 0 22 | val sendingFunction: Runnable = () => { 23 | counter += 1 24 | val now = LocalDateTime.now 25 | target ! Job(counter, now, self) 26 | } 27 | 28 | // initial rate of two jobs per second 29 | var cancellableSchedule = as.scheduler.schedule(0.second, 500.milliseconds, sendingFunction) 30 | 31 | override def receive = { 32 | case interval: FiniteDuration => 33 | cancellableSchedule.cancel() 34 | cancellableSchedule = as.scheduler.schedule(0.second, interval, sendingFunction) 35 | } 36 | } 37 | 38 | object JobSource { 39 | def apply(target: ActorRef): Props = Props(new JobSource(target)) 40 | } 41 | -------------------------------------------------------------------------------- /master/src/main/scala/io/ticofab/akkaclusterkubernetes/actor/Master.scala: -------------------------------------------------------------------------------- 1 | package io.ticofab.akkaclusterkubernetes.actor 2 | 3 | import java.time.{LocalDateTime, ZoneOffset} 4 | 5 | import akka.actor.{Actor, ActorRef, Props} 6 | import akka.cluster.Cluster 7 | import akka.cluster.ClusterEvent.{MemberEvent, MemberExited, MemberUp, UnreachableMember} 8 | import akka.cluster.routing.{ClusterRouterGroup, ClusterRouterGroupSettings} 9 | import akka.routing.RoundRobinGroup 10 | import akka.stream.ActorMaterializer 11 | import io.ticofab.akkaclusterkubernetes.actor.Master.EvaluateRate 12 | import io.ticofab.akkaclusterkubernetes.actor.Server.{RegisterStatsListener, Stats} 13 | import io.ticofab.akkaclusterkubernetes.actor.scaling.{AddNode, RemoveNode} 14 | import io.ticofab.akkaclusterkubernetes.common.{CustomLogSupport, Job, JobCompleted} 15 | 16 | import scala.collection.mutable.ListBuffer 17 | import scala.concurrent.duration._ 18 | 19 | class Master(scalingController: ActorRef) extends Actor with CustomLogSupport { 20 | 21 | implicit val ec = context.dispatcher 22 | implicit val am = ActorMaterializer()(context) 23 | 24 | Cluster(context.system).subscribe(self, classOf[MemberEvent], classOf[UnreachableMember]) 25 | 26 | // the router 27 | val workersPool = context.actorOf( 28 | ClusterRouterGroup( 29 | RoundRobinGroup(Nil), 30 | ClusterRouterGroupSettings( 31 | totalInstances = 100, 32 | routeesPaths = List("/user/worker"), 33 | allowLocalRoutees = false)).props(), 34 | name = "workerRouter") 35 | 36 | // the stats listeners 37 | var listeners: Set[ActorRef] = Set() 38 | 39 | // the queue of jobs to run 40 | val waitingJobs = ListBuffer.empty[Job] 41 | val jobsArrivedInWindow = ListBuffer.empty[Job] 42 | val jobsCompletedInWindows = ListBuffer.empty[JobCompleted] 43 | 44 | def submitNextJob(): Unit = 45 | waitingJobs.headOption.foreach( 46 | job => { 47 | workersPool ! job 48 | waitingJobs -= job 49 | }) 50 | 51 | val evaluationWindow = 5.seconds 52 | val actionInterval = 20 53 | val singleWorkerPower = (Job.jobsRatePerSecond * evaluationWindow.toSeconds).toInt 54 | context.system.scheduler.schedule(evaluationWindow, evaluationWindow, self, EvaluateRate) 55 | 56 | // initial value of last action taken, simulating it happened in the past 57 | var lastActionTaken: LocalDateTime = LocalDateTime.now minusSeconds 60 58 | var workers: Int = 0 59 | 60 | override def receive: Receive = { 61 | 62 | case MemberUp(m) => 63 | // wait a little and trigger a new job 64 | info(logJson(s"a member joined: ${m.address.toString}")) 65 | workers += 1 66 | context.system.scheduler.scheduleOnce(1.second, () => submitNextJob()) 67 | 68 | // a member left the cluster 69 | case MemberExited(_) => workers -= 1 70 | 71 | case UnreachableMember(_) => workers -= 1 72 | 73 | case RegisterStatsListener(listener) => listeners = listeners + listener 74 | 75 | case job: Job => 76 | // enqueue the new job 77 | waitingJobs += job 78 | jobsArrivedInWindow += job 79 | 80 | case job@JobCompleted(number, name) => 81 | // a job has been completed: trigger the next one 82 | info(logJson(s"job $number completed by $name.")) 83 | jobsCompletedInWindows += job 84 | if (workers > 0) submitNextJob() 85 | 86 | case EvaluateRate => 87 | val now = LocalDateTime.now 88 | val arrivedCompletedDelta = jobsArrivedInWindow.size - jobsCompletedInWindows.size 89 | 90 | val workerPoolPower = singleWorkerPower * workers 91 | val difference = jobsArrivedInWindow.size - workerPoolPower 92 | val possibleToTakeAction = now isAfter (lastActionTaken plusSeconds actionInterval) 93 | 94 | info(logJson("evaluating rate:")) 95 | info(logJson(s" queue size: ${waitingJobs.size}")) 96 | info(logJson(s" workers amount: $workers")) 97 | info(logJson(s" burndown rate: $difference")) 98 | info(logJson(s" time: ${now.toString}")) 99 | info(logJson(s" jobs arrived in window: ${jobsArrivedInWindow.size}")) 100 | info(logJson(s" arrivedCompletedDelta: $arrivedCompletedDelta")) 101 | info(logJson(s" jobs completed in window: ${jobsCompletedInWindows.size}")) 102 | info(logJson(s" workers power: $workerPoolPower")) 103 | info(logJson(s" possible to take action: $possibleToTakeAction")) 104 | 105 | val stats = Stats( 106 | now.toEpochSecond(ZoneOffset.UTC), 107 | waitingJobs.size, 108 | workers, 109 | difference, 110 | jobsArrivedInWindow.size, 111 | arrivedCompletedDelta, 112 | jobsCompletedInWindows.size, 113 | workerPoolPower) 114 | 115 | listeners.foreach(_ ! stats) 116 | 117 | // uncommenting the next line output logs in a CSV-friendly format 118 | // log.info(" csv {}", now.toString + "," + waitingJobs.size / 10 + "," + jobsArrivedInWindow.size + 119 | // "," + arrivedCompletedDelta + "," + workers + "," + workerPoolPower) 120 | 121 | if (possibleToTakeAction) { 122 | 123 | if (difference > 0) { 124 | 125 | // we are receiving more jobs than we can handle. 126 | 127 | info(logJson(" we need more power, add node")) 128 | scalingController ! AddNode 129 | lastActionTaken = now 130 | 131 | } else if (difference == 0) { 132 | 133 | // we are burning as much as it comes. 134 | 135 | // but do we have older jobs to burn? 136 | if (waitingJobs.size > singleWorkerPower) { 137 | info(logJson(" we're burning as much as we receive, but we have a long queue. add worker.")) 138 | scalingController ! AddNode 139 | lastActionTaken = now 140 | } else { 141 | // it seems we're at a perfect balance! 142 | info(logJson(" we're burning as much as we receive, and it seems we don't need more power. Excellent!.")) 143 | } 144 | 145 | } else if (difference < 0) { 146 | 147 | // we are burning down jobs! 148 | 149 | if (waitingJobs.size <= singleWorkerPower) { 150 | 151 | // we are close to the optimal. 152 | 153 | // did we strike a balance or do we have too much power? 154 | if (Math.abs(difference) <= singleWorkerPower) { 155 | info(logJson(" we have a little more processing power than we need. stay like this.")) 156 | } else { 157 | info(logJson(" we have too much power for what we need. remove worker")) 158 | scalingController ! RemoveNode 159 | lastActionTaken = now 160 | } 161 | 162 | } else { 163 | 164 | // we are burning down stuff and need to keep burning. 165 | 166 | // are we burning fast enough? 167 | if (waitingJobs.size > singleWorkerPower * 4) { 168 | 169 | info(logJson(" we are burning the old queue but not fast enough. add worker.")) 170 | scalingController ! AddNode 171 | lastActionTaken = now 172 | 173 | } else { 174 | info(logJson(" we have more power than we need but still burning down old jobs. stay like this.")) 175 | } 176 | 177 | } 178 | 179 | } 180 | 181 | } 182 | 183 | jobsArrivedInWindow.clear() 184 | jobsCompletedInWindows.clear() 185 | 186 | } 187 | } 188 | 189 | object Master { 190 | def apply(scalingController: ActorRef): Props = Props(new Master(scalingController)) 191 | 192 | case object EvaluateRate 193 | 194 | } 195 | -------------------------------------------------------------------------------- /master/src/main/scala/io/ticofab/akkaclusterkubernetes/actor/Server.scala: -------------------------------------------------------------------------------- 1 | package io.ticofab.akkaclusterkubernetes.actor 2 | 3 | import akka.actor.{Actor, ActorRef, Props} 4 | import akka.http.scaladsl.Http 5 | import akka.http.scaladsl.model.ws.{BinaryMessage, Message, TextMessage} 6 | import akka.http.scaladsl.server.Directives.{complete, get, path, _} 7 | import akka.stream.scaladsl.{Flow, GraphDSL, Keep, Sink, Source} 8 | import akka.stream.{ActorMaterializer, FlowShape, OverflowStrategy} 9 | import io.circe.generic.auto._ 10 | import io.circe.syntax._ 11 | import io.ticofab.akkaclusterkubernetes.actor.Server.{RegisterStatsListener, Stats} 12 | import io.ticofab.akkaclusterkubernetes.common.CustomLogSupport 13 | 14 | import scala.concurrent.ExecutionContext.Implicits.global 15 | import scala.concurrent.duration._ 16 | 17 | class Server(jobSource: ActorRef, master: ActorRef) extends Actor with CustomLogSupport { 18 | override def receive = Actor.emptyBehavior 19 | 20 | // http server to control the rate per second of inputs 21 | implicit val as = context.system 22 | implicit val am = ActorMaterializer() 23 | val routes = path(IntNumber) { 24 | // curl http://0.0.0.0:8080/4 --> rate will be 4 jobs per second 25 | // curl http://0.0.0.0:8080/1 --> rate will be 1 job per second 26 | // curl http://0.0.0.0:8080/2 --> rate will be 2 jobs per second 27 | jobsPerSecond => 28 | val interval = (1000.0 / jobsPerSecond.toDouble).milliseconds 29 | jobSource ! interval 30 | complete(s"rate set to 1 message every ${interval.toCoarsest}.\n") 31 | } ~ path("stats") { 32 | 33 | info(logJson("Stats client connected")) 34 | val (statsSource, publisher) = Source 35 | .actorRef[Stats](100, OverflowStrategy.dropNew) 36 | .map(stats => TextMessage(stats.asJson.noSpaces)) 37 | .toMat(Sink.asPublisher(fanout = false))(Keep.both) 38 | .run() 39 | 40 | master ! RegisterStatsListener(statsSource) 41 | 42 | val handlingFlow = Flow.fromGraph(GraphDSL.create() { implicit b => 43 | val msgSink = b.add(Sink.foreach[Message] { 44 | case tm: TextMessage => tm.textStream.runWith(Sink.ignore) 45 | case bm: BinaryMessage => bm.dataStream.runWith(Sink.ignore) 46 | }) 47 | 48 | val pubSrc = b.add(Source.fromPublisher(publisher)) 49 | 50 | FlowShape(msgSink.in, pubSrc.out) 51 | }) 52 | 53 | handleWebSocketMessages(handlingFlow) 54 | } ~ get { 55 | // curl http://0.0.0.0:8080 --> simple health check 56 | complete("Akka Cluster Kubernetes is alive!\n") 57 | } 58 | 59 | Http().bindAndHandle(routes, "0.0.0.0", 8080) 60 | } 61 | 62 | object Server { 63 | def apply(jobSource: ActorRef, master: ActorRef): Props = Props(new Server(jobSource, master)) 64 | 65 | case class RegisterStatsListener(listener: ActorRef) 66 | 67 | case class Stats(time: Long, 68 | queueSize: Int, 69 | workersAmount: Int, 70 | burndownRate: Int, 71 | jobsArrivedInWindow: Int, 72 | arrivedCompletedDelta: Int, 73 | jobsCompletedInwindow: Int, 74 | workersPower: Int) 75 | 76 | } 77 | -------------------------------------------------------------------------------- /master/src/main/scala/io/ticofab/akkaclusterkubernetes/actor/Supervisor.scala: -------------------------------------------------------------------------------- 1 | package io.ticofab.akkaclusterkubernetes.actor 2 | 3 | import akka.actor.SupervisorStrategy.Restart 4 | import akka.actor.{Actor, OneForOneStrategy, Props} 5 | import io.ticofab.akkaclusterkubernetes.actor.scaling.{DummyScalingController, KubernetesController} 6 | import io.ticofab.akkaclusterkubernetes.common.CustomLogSupport 7 | import io.ticofab.akkaclusterkubernetes.config.Config 8 | 9 | class Supervisor extends Actor with CustomLogSupport { 10 | 11 | info(logJson("Supervisor starting")) 12 | 13 | override def supervisorStrategy = OneForOneStrategy() { 14 | case t: Throwable => 15 | error("supervisor, caught exception, restarting failing child", t) 16 | Restart 17 | } 18 | 19 | // create scaling controller 20 | val scalingController = { 21 | val useK8S = Config.kubernetes.`use-kubernetes` 22 | val props = if (useK8S) KubernetesController() else Props(new DummyScalingController) 23 | context.actorOf(props) 24 | } 25 | 26 | // the master 27 | val master = context.actorOf(Master(scalingController), "master") 28 | 29 | // the tunable source of jobs 30 | val jobSource = context.actorOf(JobSource(master), "jobSource") 31 | 32 | // the server 33 | context.actorOf(Server(jobSource, master)) 34 | 35 | override def receive = Actor.emptyBehavior 36 | } 37 | 38 | object Supervisor { 39 | def apply(): Props = Props(new Supervisor) 40 | } 41 | -------------------------------------------------------------------------------- /master/src/main/scala/io/ticofab/akkaclusterkubernetes/actor/scaling/ControllerMessages.scala: -------------------------------------------------------------------------------- 1 | package io.ticofab.akkaclusterkubernetes.actor.scaling 2 | 3 | sealed trait ControllerMessage 4 | 5 | case object AddNode extends ControllerMessage 6 | 7 | case object RemoveNode extends ControllerMessage 8 | 9 | -------------------------------------------------------------------------------- /master/src/main/scala/io/ticofab/akkaclusterkubernetes/actor/scaling/DummyScalingController.scala: -------------------------------------------------------------------------------- 1 | package io.ticofab.akkaclusterkubernetes.actor.scaling 2 | 3 | import akka.actor.Actor 4 | import io.ticofab.akkaclusterkubernetes.common.CustomLogSupport 5 | 6 | class DummyScalingController extends Actor with CustomLogSupport { 7 | info(logJson("creating dummy controller")) 8 | 9 | override def receive: Receive = { 10 | case AddNode => info(logJson(s"dummy controller, received AddNode message")) 11 | case RemoveNode => info(logJson(s"dummy controller, received RemoveNode message")) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /master/src/main/scala/io/ticofab/akkaclusterkubernetes/actor/scaling/KubernetesController.scala: -------------------------------------------------------------------------------- 1 | package io.ticofab.akkaclusterkubernetes.actor.scaling 2 | 3 | import akka.actor.{Actor, Props} 4 | import io.fabric8.kubernetes.api.model._ 5 | import io.fabric8.kubernetes.api.model.extensions.DeploymentSpecBuilder 6 | import io.fabric8.kubernetes.client.{DefaultKubernetesClient, NamespacedKubernetesClient} 7 | import io.ticofab.akkaclusterkubernetes.common.CustomLogSupport 8 | import io.ticofab.akkaclusterkubernetes.config.Config 9 | 10 | import scala.collection.JavaConverters 11 | 12 | /** 13 | * This guy knows the K8S ways, while the rest of the app is K8S-agnostic. 14 | */ 15 | class KubernetesController extends Actor with CustomLogSupport { 16 | 17 | val client: NamespacedKubernetesClient = new DefaultKubernetesClient().inNamespace(System.getenv("namespace")) 18 | val workerDeploymentName = s"akka-worker" 19 | 20 | override def postStop(): Unit = { 21 | super.postStop() 22 | info(logJson("Stopping controller - deleting all workers")) 23 | client.extensions().deployments().withName(workerDeploymentName).delete() 24 | } 25 | 26 | override def receive = { 27 | 28 | case AddNode => 29 | 30 | val workers = client.extensions.deployments.withName(workerDeploymentName) 31 | 32 | // check if the deployment is there 33 | if (workers.get != null) { 34 | 35 | // scale up the replicas 36 | val currentReplicas = workers.get.getSpec.getReplicas 37 | 38 | if ((currentReplicas + 1) < Config.kubernetes.`max-replicas`) { 39 | val newReplicasAmount = currentReplicas + 1 40 | info(logJson(s"We currently have $currentReplicas nodes, scaling up Deployment $workerDeploymentName to $newReplicasAmount replicas")) 41 | workers.scale(newReplicasAmount) 42 | } else { 43 | info(logJson(s"Can't scale up. Reached maximum number of $currentReplicas replicas.")) 44 | } 45 | 46 | } else { 47 | 48 | // create new deployment 49 | info(logJson(s"Creating new Deployment $workerDeploymentName")) 50 | 51 | val workerSpec = getWorkerSpec 52 | val nameMetadata = new ObjectMetaBuilder().withName(workerDeploymentName).build 53 | client.extensions().deployments() 54 | .createNew() 55 | .withMetadata(nameMetadata) 56 | .withSpec(workerSpec) 57 | .done 58 | } 59 | 60 | case RemoveNode => 61 | 62 | val workers = client.extensions.deployments.withName(workerDeploymentName) 63 | 64 | if (workers.get != null) { 65 | 66 | // scale the replicas down 67 | val replicas = workers.get.getSpec.getReplicas - 1 68 | 69 | if (replicas >= 1) { 70 | info(logJson(s"Scaling down Deployment $workerDeploymentName to $replicas replicas")) 71 | workers.scale(replicas) 72 | } else { 73 | info(logJson("Only one replica remains in the Deployment - not scaling down")) 74 | } 75 | } else { 76 | 77 | // nothing to scale down 78 | info(logJson("Deployment doesn't exist, not scaling down")) 79 | 80 | } 81 | 82 | } 83 | 84 | def getWorkerSpec = { 85 | 86 | val role = "worker" 87 | val envVars = JavaConverters.seqAsJavaList( 88 | List[EnvVar]( 89 | new EnvVarBuilder().withName("POD_IP").withNewValueFrom().withFieldRef( 90 | new ObjectFieldSelectorBuilder().withFieldPath("status.podIP").build()).endValueFrom().build())) 91 | 92 | val labels = JavaConverters.mapAsJavaMap(Map("app" -> s"akka-$role", "role" -> role, "cluster" -> "cluster1")) 93 | 94 | val containerPort = new ContainerPortBuilder().withContainerPort(2551).build() 95 | 96 | val container = new ContainerBuilder() 97 | .withName(s"$role") 98 | .withImage(System.getenv("WORKER_IMAGE")) 99 | .withImagePullPolicy("Always") 100 | .withEnv(envVars) 101 | .withPorts(JavaConverters.seqAsJavaList[ContainerPort](List(containerPort))) 102 | .build 103 | 104 | val spec = new PodSpecBuilder() 105 | .withTerminationGracePeriodSeconds(10L) 106 | .withContainers(container) 107 | .build 108 | 109 | val labelMetadata = new ObjectMetaBuilder().withLabels(labels).build 110 | 111 | val labelSelector = new LabelSelectorBuilder().withMatchLabels(labels).build 112 | 113 | val podTemplate = new PodTemplateSpecBuilder() 114 | .withMetadata(labelMetadata) 115 | .withSpec(spec) 116 | .build 117 | 118 | new DeploymentSpecBuilder() 119 | .withSelector(labelSelector) 120 | .withReplicas(1) 121 | .withTemplate(podTemplate) 122 | .build 123 | } 124 | 125 | } 126 | 127 | object KubernetesController { 128 | def apply(): Props = Props(new KubernetesController) 129 | } 130 | -------------------------------------------------------------------------------- /master/src/main/scala/io/ticofab/akkaclusterkubernetes/config/Config.scala: -------------------------------------------------------------------------------- 1 | package io.ticofab.akkaclusterkubernetes.config 2 | 3 | import com.typesafe.config.ConfigFactory 4 | import net.ceedubs.ficus.Ficus._ 5 | import net.ceedubs.ficus.readers.ArbitraryTypeReader._ 6 | 7 | object Config { 8 | 9 | case class Remote(hostname: String, 10 | port: Int, 11 | `bind-hostname`: String, 12 | `bind-port`: Int) 13 | 14 | case class Cluster(`seed-nodes`: List[String]) 15 | 16 | case class Kubernetes(`use-kubernetes`: Boolean, 17 | `max-replicas`: Int) 18 | 19 | val config = ConfigFactory.load() 20 | val remote = config.as[Remote]("akka.remote.netty.tcp") 21 | val cluster = config.as[Cluster]("akka.cluster") 22 | val kubernetes = config.as[Kubernetes]("kubernetes") 23 | } 24 | -------------------------------------------------------------------------------- /stats-visualizer/.gitignore: -------------------------------------------------------------------------------- 1 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 2 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 3 | 4 | .idea/ 5 | 6 | # User-specific stuff 7 | .idea/**/workspace.xml 8 | .idea/**/tasks.xml 9 | .idea/**/usage.statistics.xml 10 | .idea/**/dictionaries 11 | .idea/**/shelf 12 | 13 | # Generated files 14 | .idea/**/contentModel.xml 15 | 16 | # Sensitive or high-churn files 17 | .idea/**/dataSources/ 18 | .idea/**/dataSources.ids 19 | .idea/**/dataSources.local.xml 20 | .idea/**/sqlDataSources.xml 21 | .idea/**/dynamic.xml 22 | .idea/**/uiDesigner.xml 23 | .idea/**/dbnavigator.xml 24 | 25 | # Gradle 26 | .idea/**/gradle.xml 27 | .idea/**/libraries 28 | 29 | # Gradle and Maven with auto-import 30 | # When using Gradle or Maven with auto-import, you should exclude module files, 31 | # since they will be recreated, and may cause churn. Uncomment if using 32 | # auto-import. 33 | # .idea/modules.xml 34 | # .idea/*.iml 35 | # .idea/modules 36 | 37 | # CMake 38 | cmake-build-*/ 39 | 40 | # Mongo Explorer plugin 41 | .idea/**/mongoSettings.xml 42 | 43 | # File-based project format 44 | *.iws 45 | 46 | # IntelliJ 47 | out/ 48 | 49 | # mpeltonen/sbt-idea plugin 50 | .idea_modules/ 51 | 52 | # JIRA plugin 53 | atlassian-ide-plugin.xml 54 | 55 | # Cursive Clojure plugin 56 | .idea/replstate.xml 57 | 58 | # Crashlytics plugin (for Android Studio and IntelliJ) 59 | com_crashlytics_export_strings.xml 60 | crashlytics.properties 61 | crashlytics-build.properties 62 | fabric.properties 63 | 64 | # Editor-based Rest Client 65 | .idea/httpRequests -------------------------------------------------------------------------------- /stats-visualizer/chart.js: -------------------------------------------------------------------------------- 1 | var stats = [ 2 | ['Time', 'Queue size', 'Workers', 'Burndown rate', 'New jobs per interval'] 3 | ]; 4 | 5 | google.charts.load('current', {'packages': ['corechart']}); 6 | 7 | function drawChart(input) { 8 | const data = google.visualization.arrayToDataTable(input); 9 | 10 | const options = { 11 | legend: {position: 'bottom'}, 12 | vAxis: { 13 | ticks: [-20, 0, 20, 40], 14 | viewWindow: { 15 | min: -20, 16 | max: 40 17 | } 18 | }, 19 | hAxis: { 20 | textPosition: 'none' 21 | } 22 | }; 23 | 24 | const chart = new google.visualization.LineChart(document.getElementById('curve_chart')); 25 | 26 | chart.draw(data, options); 27 | } 28 | 29 | function randInt() { 30 | return Math.floor(Math.random() * 1000) + 1; 31 | } 32 | 33 | function randomDrawChart() { 34 | const input = [ 35 | ['Year', 'Sales', 'Expenses'], 36 | ['2004', randInt(), randInt()], 37 | ['2005', 1170, 460], 38 | ['2006', randInt(), randInt()], 39 | ['2007', 1030, 540] 40 | ]; 41 | 42 | drawChart(input); 43 | } -------------------------------------------------------------------------------- /stats-visualizer/visualizer.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 |
12 | 13 | 14 |
15 | 16 | 17 | -------------------------------------------------------------------------------- /stats-visualizer/wsscript.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Open a new WebSocket connection using the given parameters 3 | */ 4 | function openWSConnection(protocol, hostname, port, endpoint) { 5 | var webSocketURL = protocol + "://" + hostname + ":" + port + endpoint; 6 | console.log("openWSConnection::Connecting to: " + webSocketURL); 7 | try { 8 | webSocket = new WebSocket(webSocketURL); 9 | webSocket.onopen = function (openEvent) { 10 | console.log("WebSocket OPEN: " + JSON.stringify(openEvent, null, 4)); 11 | }; 12 | webSocket.onclose = function (closeEvent) { 13 | console.log("WebSocket CLOSE: " + JSON.stringify(closeEvent, null, 4)); 14 | }; 15 | webSocket.onerror = function (errorEvent) { 16 | console.log("WebSocket ERROR: " + JSON.stringify(errorEvent, null, 4)); 17 | }; 18 | webSocket.onmessage = function (messageEvent) { 19 | const wsMsg = JSON.parse(messageEvent.data); 20 | const values = [ 21 | new Date(Number(wsMsg.time) * 1000), 22 | wsMsg.queueSize / 2, 23 | wsMsg.workersAmount, 24 | wsMsg.burndownRate, 25 | wsMsg.jobsArrivedInWindow 26 | ]; 27 | stats.push(values); 28 | drawChart(stats); 29 | console.log("New graph values: " + values); 30 | }; 31 | } catch (exception) { 32 | console.error(exception); 33 | } 34 | } 35 | 36 | -------------------------------------------------------------------------------- /worker/.gitignore: -------------------------------------------------------------------------------- 1 | logs 2 | target 3 | /.idea 4 | /.idea_modules 5 | /.classpath 6 | /.project 7 | /.settings 8 | /RUNNING_PID 9 | out 10 | 11 | -------------------------------------------------------------------------------- /worker/build.sbt: -------------------------------------------------------------------------------- 1 | 2 | name := "worker" 3 | version := "0.0.1" 4 | scalaVersion := "2.12.6" 5 | organization := "ticofab.io" 6 | 7 | lazy val common = RootProject(file("../common")) 8 | val main = Project(id = "worker", base = file(".")).dependsOn(common) 9 | 10 | enablePlugins(JavaAppPackaging) 11 | enablePlugins(AshScriptPlugin) 12 | 13 | mainClass in Compile := Some("io.ticofab.akkaclusterkubernetes.AkkaClusterKubernetesWorkerApp") 14 | packageName in Docker := "adam-akka/" + name.value 15 | version in Docker := "latest" 16 | dockerLabels := Map("maintainer" -> organization.value, "version" -> version.value) 17 | dockerBaseImage := "openjdk:8-jre" 18 | defaultLinuxInstallLocation in Docker := s"/opt/${name.value}" // to have consistent directory for files 19 | dockerRepository := Some("eu.gcr.io") 20 | -------------------------------------------------------------------------------- /worker/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 1.2.1 -------------------------------------------------------------------------------- /worker/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.2") 2 | -------------------------------------------------------------------------------- /worker/src/main/resources/application.conf: -------------------------------------------------------------------------------- 1 | akka { 2 | 3 | log-dead-letters = 0 4 | 5 | // remote configuration 6 | remote { 7 | enabled-transports = ["akka.remote.netty.tcp"] 8 | netty.tcp { 9 | 10 | // this needs to be set to the DNS name visible in the cluster 11 | // the master's address will be akka-master (dns name created by Service) 12 | // we set the POD_IP env var for workers. they will be accessed by IP address not DNS name so we don't have to create services for them 13 | hostname = "0.0.0.0" 14 | hostname = ${?POD_IP} 15 | 16 | port = 2552 17 | port = ${?PORT} 18 | 19 | // this needs to be set when we run in a docker container 20 | bind-hostname = ${akka.remote.netty.tcp.hostname} 21 | bind-hostname = ${?HOSTNAME} 22 | bind-port = 2552 23 | bind-port = ${?PORT} 24 | } 25 | } 26 | 27 | // enable clustering 28 | actor { 29 | provider = "akka.cluster.ClusterActorRefProvider" 30 | warn-about-java-serializer-usage = false 31 | } 32 | 33 | // cluster configuration 34 | cluster { 35 | 36 | seed-nodes = ["akka.tcp://akka-cluster-kubernetes@akka-master:2551"] 37 | seed-nodes = ${?SEED_NODES} 38 | min-nr-of-members = 1 39 | 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /worker/src/main/scala/io/ticofab/akkaclusterkubernetes/AkkaClusterKubernetesWorkerApp.scala: -------------------------------------------------------------------------------- 1 | package io.ticofab.akkaclusterkubernetes 2 | 3 | import akka.actor.ActorSystem 4 | import akka.cluster.Cluster 5 | import io.ticofab.akkaclusterkubernetes.common.CustomLogSupport 6 | 7 | object AkkaClusterKubernetesWorkerApp extends App with CustomLogSupport { 8 | 9 | implicit val as = ActorSystem("akka-cluster-kubernetes") 10 | info(logJson(s"Cluster config: ${Config.cluster}")) 11 | info(logJson(s"This node is a worker")) 12 | as.actorOf(Worker(), "worker") 13 | 14 | as.registerOnTermination(() => { 15 | info(logJson(s"Received system termination. Leaving cluster.")) 16 | val cluster = Cluster(as) 17 | cluster.registerOnMemberRemoved(() => as.terminate()) 18 | cluster.leave(cluster.selfAddress) 19 | }) 20 | 21 | } 22 | -------------------------------------------------------------------------------- /worker/src/main/scala/io/ticofab/akkaclusterkubernetes/Config.scala: -------------------------------------------------------------------------------- 1 | package io.ticofab.akkaclusterkubernetes 2 | 3 | import com.typesafe.config.ConfigFactory 4 | import net.ceedubs.ficus.Ficus._ 5 | import net.ceedubs.ficus.readers.ArbitraryTypeReader._ 6 | 7 | object Config { 8 | case class Cluster(`seed-nodes`: List[String]) 9 | val config = ConfigFactory.load() 10 | val cluster = config.as[Cluster]("akka.cluster") 11 | } 12 | -------------------------------------------------------------------------------- /worker/src/main/scala/io/ticofab/akkaclusterkubernetes/Worker.scala: -------------------------------------------------------------------------------- 1 | package io.ticofab.akkaclusterkubernetes 2 | 3 | import akka.actor.{Actor, Props} 4 | import io.ticofab.akkaclusterkubernetes.common.{CustomLogSupport, Job, JobCompleted} 5 | 6 | class Worker extends Actor with CustomLogSupport { 7 | 8 | val jobsMillis = 1000 / Job.jobsRatePerSecond 9 | val namePort = self.path.name + self.path.address.port 10 | info(logJson(s"creating worker $namePort")) 11 | info(logJson(s"each job will take $jobsMillis millis.")) 12 | 13 | override def receive = { 14 | 15 | case job: Job => 16 | info(logJson(s"worker ${self.path.name}, received job ${job.number}")) 17 | 18 | // Simulate a CPU-intensive workload that takes ~2000 milliseconds 19 | val start = System.currentTimeMillis() 20 | while ((System.currentTimeMillis() - start) < jobsMillis) {} 21 | 22 | sender ! JobCompleted(job.number, namePort) 23 | } 24 | 25 | } 26 | 27 | object Worker { 28 | def apply(): Props = Props(new Worker) 29 | } 30 | --------------------------------------------------------------------------------