├── frontend ├── public │ ├── stylesheets │ │ └── main.css │ └── images │ │ └── favicon.png ├── conf │ ├── prod.conf │ ├── application.conf │ ├── logback.xml │ ├── routes │ └── base.conf └── app │ ├── library │ ├── GenericExtensions.scala │ └── ConfigValueExtensions.scala │ ├── common │ └── AppSettings.scala │ ├── services │ ├── ConnectionSetService.scala │ ├── NodeSetService.scala │ ├── ClusterMetadataService.scala │ └── ClusterClientService.scala │ ├── templates │ └── Page.scala │ ├── controllers │ └── StreamController.scala │ └── actors │ └── ConnectionStore.scala ├── project ├── build.properties ├── plugins.sbt └── Dependencies.scala ├── backend └── src │ ├── test │ ├── resources │ │ └── test.conf │ └── scala │ │ └── tmt │ │ └── common │ │ └── Utils.scala │ └── main │ ├── resources │ ├── dev.conf │ ├── prod.conf │ └── base.conf │ └── scala │ └── tmt │ ├── integration │ ├── camera │ │ ├── Listener.scala │ │ └── Simulator.scala │ └── bridge │ │ ├── SourceSimulator.scala │ │ └── StreamListener.scala │ ├── app │ ├── ActorConfigs.scala │ ├── Types.scala │ ├── CustomDirectives.scala │ ├── NodeInfoPublisher.scala │ ├── Main.scala │ ├── ConfigLoader.scala │ ├── AppSettings.scala │ └── Assembly.scala │ ├── library │ ├── Sources.scala │ ├── RequestHandlerExtensions.scala │ ├── Connector.scala │ ├── InetSocketAddressExtensions.scala │ ├── ResponseExtensions.scala │ ├── IpInfo.scala │ ├── FlowExtensions.scala │ ├── SourceExtensions.scala │ └── Cors.scala │ ├── io │ ├── Producer.scala │ ├── ScienceImageReadService.scala │ ├── ScienceImageWriteService.scala │ ├── WavefrontWriteService.scala │ └── WavefrontReadService.scala │ ├── server │ ├── ServerFactory.scala │ ├── Publisher.scala │ ├── Server.scala │ ├── RouteInstances.scala │ ├── RouteFactory.scala │ └── MediaRoute.scala │ ├── transformations │ ├── MetricsTransformations.scala │ ├── ImageTransformations.scala │ └── ImageProcessor.scala │ ├── clients │ ├── OneToOneTransfer.scala │ ├── OneToManyTransfer.scala │ └── clients.scala │ ├── actors │ ├── SourceActorLink.scala │ ├── Subscription.scala │ └── Ticker.scala │ └── marshalling │ ├── BinaryMarshallers.scala │ └── BFormat.scala ├── activator-launch-1.3.2.jar ├── diagrams ├── wavefront-browser-pipeline ├── wavefront-metadata-metric-pipeline ├── dependencies ├── science-images ├── wavefront-rotator-metadata-metric-pipeline ├── complex-pipeline ├── throttle ├── subscribe ├── unsubscribe ├── connections └── nodes ├── common └── src │ └── main │ ├── resources │ ├── prod-bindings.conf │ ├── dev-bindings.conf │ ├── akka-base.conf │ └── akka-http-core-reference.conf │ └── scala │ └── tmt │ ├── common │ ├── Keys.scala │ └── Messages.scala │ └── library │ └── ConfigObjectExtensions.scala ├── client └── src │ └── main │ └── scala │ └── tmt │ ├── views │ ├── View.scala │ ├── FrequencyView.scala │ ├── ImageView.scala │ ├── ScienceImageView.scala │ ├── Body.scala │ ├── ThrottleView.scala │ └── SubscriptionView.scala │ ├── common │ ├── Constants.scala │ └── Stream.scala │ ├── app │ ├── ClientMain.scala │ ├── ClientAssembly.scala │ ├── ViewData.scala │ └── DataStore.scala │ ├── framework │ ├── JQueryMaterialize.scala │ ├── FormRx.scala │ ├── ScienceImageRx.scala │ ├── WebsocketRx.scala │ ├── Helpers.scala │ └── Framework.scala │ ├── images │ ├── Rendering.scala │ └── ImageRendering.scala │ └── metrics │ └── FrequencyRendering.scala ├── shared └── src │ └── main │ └── scala │ └── tmt │ └── shared │ ├── Topics.scala │ └── models │ ├── Image.scala │ ├── ConnectionSet.scala │ ├── metrics.scala │ ├── NodeSet.scala │ └── Role.scala ├── .gitignore ├── README.md ├── activator.bat └── activator /frontend/public/stylesheets/main.css: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.9 2 | -------------------------------------------------------------------------------- /backend/src/test/resources/test.conf: -------------------------------------------------------------------------------- 1 | include "base.conf" 2 | 3 | max-transfer-files = 10000 4 | -------------------------------------------------------------------------------- /frontend/conf/prod.conf: -------------------------------------------------------------------------------- 1 | include "base.conf" 2 | include "prod-bindings.conf" 3 | 4 | env = prod 5 | -------------------------------------------------------------------------------- /activator-launch-1.3.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tmtsoftware/bulkdata/HEAD/activator-launch-1.3.2.jar -------------------------------------------------------------------------------- /diagrams/wavefront-browser-pipeline: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | Wavefront1 -> [ Images ] Browser1 4 | 5 | @enduml -------------------------------------------------------------------------------- /frontend/conf/application.conf: -------------------------------------------------------------------------------- 1 | include "base.conf" 2 | include "dev-bindings.conf" 3 | 4 | env = dev 5 | -------------------------------------------------------------------------------- /frontend/public/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tmtsoftware/bulkdata/HEAD/frontend/public/images/favicon.png -------------------------------------------------------------------------------- /common/src/main/resources/prod-bindings.conf: -------------------------------------------------------------------------------- 1 | include "dev-bindings.conf" 2 | 3 | seed1.hostname = "172.31.0.94" 4 | seed2.hostname = "172.31.11.97" 5 | -------------------------------------------------------------------------------- /diagrams/wavefront-metadata-metric-pipeline: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | Wavefront1 -> [ Images] Metadata1 4 | Metadata1 -> [ Metadata] Metric1 5 | 6 | @enduml -------------------------------------------------------------------------------- /diagrams/dependencies: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | "Shared" --> [ ] "Common" 4 | "Shared" --> "Client" 5 | 6 | "Common" --> "Backend" 7 | "Common" --> "Frontend" 8 | 9 | @enduml -------------------------------------------------------------------------------- /diagrams/science-images: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | Camera1 --> [ ] Client1 4 | Camera1 --> [ ] Client2 5 | Camera1 --> [ ] Client3 6 | Camera1 --> [ Science Images ] ClientN 7 | 8 | @enduml -------------------------------------------------------------------------------- /backend/src/main/resources/dev.conf: -------------------------------------------------------------------------------- 1 | include "base.conf" 2 | include "dev-bindings.conf" 3 | 4 | env = dev 5 | 6 | image-processing { 7 | thread-pool-size = 8 8 | parallelism = 4 9 | } 10 | -------------------------------------------------------------------------------- /backend/src/main/resources/prod.conf: -------------------------------------------------------------------------------- 1 | include "base.conf" 2 | include "prod-bindings.conf" 3 | 4 | env = prod 5 | 6 | image-processing { 7 | thread-pool-size = 20 8 | parallelism = 16 9 | } 10 | -------------------------------------------------------------------------------- /diagrams/wavefront-rotator-metadata-metric-pipeline: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | Wavefront1 -> [ Images] "Rotator1 (16 cores)" 4 | "Rotator1 (16 cores)" -> [ Images] Metadata1 5 | Metadata1 -> [Metadata] Metric1 6 | 7 | @enduml -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/integration/camera/Listener.scala: -------------------------------------------------------------------------------- 1 | package tmt.integration.camera 2 | 3 | trait Listener[T] { 4 | def onEvent(event: T): Unit 5 | def onError(ex: Throwable): Unit 6 | def onComplete(): Unit 7 | } 8 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/views/View.scala: -------------------------------------------------------------------------------- 1 | package tmt.views 2 | 3 | import scalatags.JsDom.all.Modifier 4 | 5 | trait View { 6 | def viewTitle: Modifier 7 | def viewContent: Modifier 8 | def viewAction: Modifier 9 | } 10 | -------------------------------------------------------------------------------- /common/src/main/scala/tmt/common/Keys.scala: -------------------------------------------------------------------------------- 1 | package tmt.common 2 | 3 | import akka.cluster.ddata.LWWMapKey 4 | import tmt.shared.Topics 5 | import tmt.shared.models.NodeS 6 | 7 | object Keys { 8 | val Nodes = LWWMapKey[NodeS](Topics.Nodes) 9 | } 10 | -------------------------------------------------------------------------------- /common/src/main/resources/dev-bindings.conf: -------------------------------------------------------------------------------- 1 | seed1 { 2 | hostname = "127.0.0.1" 3 | port = 2551 4 | address = ${seed1.hostname}":"${seed1.port} 5 | } 6 | seed2 = { 7 | hostname = "127.0.0.1" 8 | port = 2552 9 | address = ${seed2.hostname}":"${seed2.port} 10 | } 11 | -------------------------------------------------------------------------------- /shared/src/main/scala/tmt/shared/Topics.scala: -------------------------------------------------------------------------------- 1 | package tmt.shared 2 | 3 | object Topics { 4 | val Throttle = "throttle" 5 | val Subscription = "subscription" 6 | val Connections = "connections" 7 | val Nodes = "nodes" 8 | val ScienceImages = "science-images" 9 | } 10 | -------------------------------------------------------------------------------- /backend/src/test/scala/tmt/common/Utils.scala: -------------------------------------------------------------------------------- 1 | package tmt.common 2 | 3 | import scala.concurrent.duration.{Duration, DurationInt} 4 | import scala.concurrent.{Await, Future} 5 | 6 | object Utils { 7 | def await[T](f: Future[T], duration: Duration = 30.seconds) = Await.result(f, duration) 8 | } 9 | -------------------------------------------------------------------------------- /frontend/app/library/GenericExtensions.scala: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import play.api.libs.json.{Json, Writes} 4 | 5 | object GenericExtensions { 6 | 7 | implicit class RichGeneric[T](val value: T) extends AnyVal { 8 | def toJson(implicit writes: Writes[T]) = Json.toJson(value) 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /shared/src/main/scala/tmt/shared/models/Image.scala: -------------------------------------------------------------------------------- 1 | package tmt.shared.models 2 | 3 | import boopickle.Default._ 4 | 5 | case class Image(name: String, bytes: Array[Byte], createdAt: Long) { 6 | def size = bytes.length 7 | } 8 | 9 | object Image { 10 | implicit val pickler = generatePickler[Image] 11 | } 12 | -------------------------------------------------------------------------------- /frontend/app/common/AppSettings.scala: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import javax.inject.{Inject, Singleton} 4 | 5 | import play.api.Configuration 6 | 7 | @Singleton 8 | class AppSettings @Inject()(configuration: Configuration) { 9 | private val config = configuration.underlying 10 | val env = config.getString("env") 11 | } 12 | -------------------------------------------------------------------------------- /common/src/main/scala/tmt/common/Messages.scala: -------------------------------------------------------------------------------- 1 | package tmt.common 2 | 3 | import tmt.shared.models.Connection 4 | 5 | import scala.concurrent.duration.FiniteDuration 6 | 7 | object Messages { 8 | case class UpdateDelay(serverName: String, value: FiniteDuration) 9 | case class Subscribe(connection: Connection) 10 | case class Unsubscribe(connection: Connection) 11 | } 12 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/common/Constants.scala: -------------------------------------------------------------------------------- 1 | package tmt.common 2 | 3 | import org.scalajs.dom.URL 4 | 5 | import scala.concurrent.duration.DurationInt 6 | import scala.scalajs.js 7 | 8 | object Constants { 9 | val CanvasWidth = 192*3 10 | val CanvasHeight = 108*3 11 | val URL = js.Dynamic.global.window.URL.asInstanceOf[URL] 12 | val RefreshRate = 5.seconds 13 | } 14 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/app/ActorConfigs.scala: -------------------------------------------------------------------------------- 1 | package tmt.app 2 | 3 | import akka.actor.ActorSystem 4 | import akka.stream.Materializer 5 | 6 | import scala.concurrent.ExecutionContext 7 | 8 | class ActorConfigs(_system: ActorSystem, _mat: Materializer, _ec: ExecutionContext) { 9 | implicit val system = _system 10 | implicit val mat = _mat 11 | implicit val ec = _ec 12 | } 13 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/app/ClientMain.scala: -------------------------------------------------------------------------------- 1 | package tmt.app 2 | 3 | import org.scalajs.dom.document 4 | 5 | import scala.scalajs.js.JSApp 6 | import scala.scalajs.js.annotation.JSExport 7 | 8 | object ClientMain extends JSApp { 9 | @JSExport 10 | override def main() = { 11 | val body = new ClientAssembly().body.layout.render 12 | document.body.appendChild(body) 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /frontend/app/library/ConfigValueExtensions.scala: -------------------------------------------------------------------------------- 1 | package library 2 | 3 | import com.typesafe.config.{ConfigRenderOptions, ConfigValue} 4 | import play.api.libs.json.{Json, Reads} 5 | 6 | object ConfigValueExtensions { 7 | 8 | implicit class RichConfigValue(val configValue: ConfigValue) extends AnyVal { 9 | def as[T: Reads] = Json.parse(configValue.render(ConfigRenderOptions.concise())).as[T] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/library/Sources.scala: -------------------------------------------------------------------------------- 1 | package tmt.library 2 | 3 | import akka.stream.scaladsl.Source 4 | 5 | import scala.concurrent.duration.FiniteDuration 6 | 7 | object Sources { 8 | def ticks(duration: FiniteDuration) = Source(duration, duration, ()) 9 | def interval(duration: FiniteDuration) = ticks(duration).scan(0)((acc, elm) => acc + 1).drop(1) 10 | def numbers = Source(() => Iterator.from(1)) 11 | } 12 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/library/RequestHandlerExtensions.scala: -------------------------------------------------------------------------------- 1 | package tmt.library 2 | 3 | import akka.http.scaladsl.model.HttpRequest 4 | import akka.stream.scaladsl.Flow 5 | import tmt.app.Types 6 | 7 | object RequestHandlerExtensions { 8 | 9 | implicit class RichRequestHandler(val requestHandler: Types.RequestHandler) extends AnyVal { 10 | def toFlow = Flow[HttpRequest].mapAsync(1)(requestHandler) 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/io/Producer.scala: -------------------------------------------------------------------------------- 1 | package tmt.io 2 | 3 | import java.io.File 4 | 5 | import tmt.app.AppSettings 6 | 7 | class Producer(appSettings: AppSettings) { 8 | def numbers() = Iterator.from(1) 9 | def list(dir: String) = new File(dir).listFiles().sortBy(_.getName) 10 | 11 | def files(dir: String): Iterator[File] = { 12 | val files = list(dir) 13 | Iterator.continually(files).flatten 14 | }.take(appSettings.maxTransferFiles) 15 | } 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | logs 2 | project/project 3 | project/target 4 | target 5 | tmp 6 | .history 7 | dist 8 | .idea 9 | *.iml 10 | /out 11 | .idea_modules 12 | .classpath 13 | .project 14 | /RUNNING_PID 15 | .settings 16 | .DS_Store 17 | 18 | # hidden cross project folders 19 | shared/.js 20 | shared/.jvm 21 | 22 | # temp files 23 | .~* 24 | *~ 25 | *.orig 26 | 27 | # eclipse 28 | .scala_dependencies 29 | .buildpath 30 | .cache 31 | .target 32 | bin/ 33 | journal 34 | snapshots 35 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/integration/bridge/SourceSimulator.scala: -------------------------------------------------------------------------------- 1 | package tmt.integration.bridge 2 | 3 | import akka.stream.Materializer 4 | import tmt.integration.camera.Simulator 5 | 6 | object SourceSimulator { 7 | def apply[T](producer: () => Iterator[T])(implicit materializer: Materializer) = { 8 | val simulator = new Simulator(producer()) 9 | val listener = new StreamListener[T] 10 | simulator.subscribe(listener) 11 | listener.source 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.4.3") 2 | 3 | addSbtPlugin("com.vmunier" % "sbt-play-scalajs" % "0.2.8") 4 | 5 | addSbtPlugin("com.typesafe.sbt" % "sbt-gzip" % "1.0.0") 6 | 7 | addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.1.8") 8 | 9 | addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.5") 10 | 11 | addSbtPlugin("com.orrsella" % "sbt-stats" % "1.0.5") 12 | 13 | addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.7.5") 14 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/library/Connector.scala: -------------------------------------------------------------------------------- 1 | package tmt.library 2 | 3 | import akka.stream.scaladsl.{Keep, Sink, Source} 4 | import akka.stream.{Materializer, OverflowStrategy} 5 | import org.reactivestreams.Publisher 6 | 7 | object Connector { 8 | def coupling[T](sink: Sink[T, Publisher[T]])(implicit mat: Materializer) = { 9 | val (actorRef, publisher) = Source.actorRef[T](2, OverflowStrategy.dropHead).toMat(sink)(Keep.both).run() 10 | (actorRef, Source(publisher)) 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/library/InetSocketAddressExtensions.scala: -------------------------------------------------------------------------------- 1 | package tmt.library 2 | 3 | import java.net.InetSocketAddress 4 | 5 | import akka.http.scaladsl.model.Uri 6 | 7 | object InetSocketAddressExtensions { 8 | 9 | implicit class RichInetSocketAddress(address: InetSocketAddress) { 10 | def absoluteUri(relativeUri: String) = Uri(s"http://${address.getHostName}:${address.getPort}$relativeUri") 11 | def update(uri: Uri) = uri.withAuthority(address.getHostName, address.getPort) 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/server/ServerFactory.scala: -------------------------------------------------------------------------------- 1 | package tmt.server 2 | 3 | import tmt.app.{ActorConfigs, AppSettings} 4 | import tmt.shared.models.Role 5 | 6 | class ServerFactory( 7 | routeInstances: RouteInstances, 8 | actorConfigs: ActorConfigs, 9 | appSettings: AppSettings 10 | ) { 11 | 12 | import actorConfigs._ 13 | 14 | def make() = { 15 | val role = Role.withName(appSettings.binding.role) 16 | new Server(appSettings.binding.httpAddress, routeInstances.find(role), actorConfigs) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/framework/JQueryMaterialize.scala: -------------------------------------------------------------------------------- 1 | package tmt.framework 2 | 3 | import org.scalajs.jquery.JQuery 4 | 5 | import scala.language.implicitConversions 6 | import scala.scalajs.js 7 | 8 | @js.native 9 | trait JQueryMaterialize extends JQuery { 10 | def material_select(): this.type = js.native 11 | def material_select(b: String): this.type = js.native 12 | } 13 | 14 | object JQueryMaterialize { 15 | implicit def jq2Materialize(jq: JQuery): JQueryMaterialize = 16 | jq.asInstanceOf[JQueryMaterialize] 17 | } 18 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/app/Types.scala: -------------------------------------------------------------------------------- 1 | package tmt.app 2 | 3 | import akka.http.scaladsl.marshalling.Marshaller 4 | import akka.http.scaladsl.model.HttpEntity.Chunked 5 | import akka.http.scaladsl.model.{HttpResponse, HttpRequest} 6 | import akka.stream.scaladsl.{Source, Flow} 7 | 8 | import scala.concurrent.Future 9 | 10 | object Types { 11 | type RequestHandler = HttpRequest => Future[HttpResponse] 12 | type ConnectionFlow = Flow[HttpRequest, HttpResponse, Any] 13 | type Stream[T] = Marshaller[Source[T, Any], Chunked] 14 | } 15 | -------------------------------------------------------------------------------- /frontend/app/services/ConnectionSetService.scala: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import javax.inject.{Inject, Singleton} 4 | 5 | import scala.async.Async._ 6 | import scala.concurrent.ExecutionContext 7 | 8 | @Singleton 9 | class ConnectionSetService @Inject()( 10 | clusterClientService: ClusterClientService, 11 | clusterMetadataService: ClusterMetadataService)(implicit ec: ExecutionContext) { 12 | 13 | def connectionSet = async { 14 | await(clusterClientService.allConnections).pruneBy(clusterMetadataService.onlineRoles) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/transformations/MetricsTransformations.scala: -------------------------------------------------------------------------------- 1 | package tmt.transformations 2 | 3 | import tmt.actors.SubscriptionService 4 | import tmt.shared.models.{ImageMetric, ImageMetadata, PerSecMetric} 5 | 6 | import scala.concurrent.duration.DurationInt 7 | 8 | class MetricsTransformations(imageInfoSubscriber: SubscriptionService[ImageMetadata]) { 9 | lazy val imageMetrics = imageInfoSubscriber.source.map(ImageMetric.from) 10 | lazy val perSecMetrics = imageMetrics.groupedWithin(10000, 1.second).map(PerSecMetric.from) 11 | } 12 | -------------------------------------------------------------------------------- /diagrams/complex-pipeline: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | Wavefront1 --> [ Image] Rotator1 4 | Rotator1 --> [ Image] Metadata1 5 | Metadata1 --> [ Metadata] Metric1 6 | Metric1 --> [ Metric] Browser1 7 | Wavefront1 --> [ Image] Browser4 8 | Rotator1 --> Browser4 9 | 10 | Wavefront1 --> [ Image]Metadata2 11 | Metadata2 --> [ Metadata] Metric2 12 | Metric2 --> [ Metric] Browser2 13 | 14 | Wavefront1 --> [ Image] Metadata3 15 | Wavefront2 --> [ Image] Metadata3 16 | Metadata3 --> [ Metadata] Metric3 17 | Metric3 --> [ Metric] Browser3 18 | 19 | 20 | 21 | @enduml -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/library/ResponseExtensions.scala: -------------------------------------------------------------------------------- 1 | package tmt.library 2 | 3 | import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpResponse} 4 | import akka.stream.Materializer 5 | import tmt.library.SourceExtensions.RichSource 6 | 7 | object ResponseExtensions { 8 | 9 | implicit class RichResponse(val response: HttpResponse) extends AnyVal { 10 | 11 | def multicastEntity(implicit mat: Materializer) = HttpEntity.Chunked.fromData( 12 | ContentTypes.`application/octet-stream`, 13 | response.entity.dataBytes.multicast 14 | ) 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/server/Publisher.scala: -------------------------------------------------------------------------------- 1 | package tmt.server 2 | 3 | import akka.cluster.pubsub.DistributedPubSub 4 | import akka.cluster.pubsub.DistributedPubSubMediator.Publish 5 | import akka.stream.scaladsl.Source 6 | import tmt.app.ActorConfigs 7 | 8 | class Publisher(actorConfigs: ActorConfigs) { 9 | import actorConfigs._ 10 | 11 | private val mediator = DistributedPubSub(system).mediator 12 | 13 | def publish(topic: String, xs: Source[Any, Any]) = xs.runForeach { x => 14 | println(s"publishing: $topic: $x") 15 | mediator ! Publish(topic, x) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/framework/FormRx.scala: -------------------------------------------------------------------------------- 1 | package tmt.framework 2 | 3 | import rx._ 4 | import tmt.app.ViewData 5 | 6 | abstract class FormRx(name: String, viewData: ViewData) { 7 | 8 | def getUrl: Option[String] 9 | 10 | val server: Var[String] = Var("") 11 | val selectedServer = Var(name) 12 | val currentUrl = Var("") 13 | 14 | def action() = getUrl.foreach { url => 15 | selectedServer() = server() 16 | currentUrl() = url 17 | } 18 | 19 | Obs(viewData.diffs) { 20 | if (viewData.diffs() contains server()) { 21 | action() 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/app/ClientAssembly.scala: -------------------------------------------------------------------------------- 1 | package tmt.app 2 | 3 | import tmt.views._ 4 | 5 | class ClientAssembly { 6 | 7 | implicit val scheduler = monifu.concurrent.Implicits.globalScheduler 8 | 9 | val dataStore = new DataStore 10 | 11 | val body = new Body( 12 | new ThrottleView(dataStore.data), 13 | new SubscriptionView(dataStore.data), 14 | new ScienceImageView(dataStore.data), 15 | Seq( 16 | new FrequencyView(dataStore.data) 17 | ), 18 | Seq( 19 | new ImageView(dataStore.data), 20 | new ImageView(dataStore.data) 21 | ) 22 | ) 23 | } 24 | -------------------------------------------------------------------------------- /common/src/main/scala/tmt/library/ConfigObjectExtensions.scala: -------------------------------------------------------------------------------- 1 | package tmt.library 2 | 3 | import com.typesafe.config.{Config, ConfigObject, ConfigValueFactory} 4 | 5 | object ConfigObjectExtensions { 6 | 7 | implicit class RichConfigObject(val config: ConfigObject) extends AnyVal { 8 | def withPair(key: String, value: AnyRef) = config.withValue(key, ConfigValueFactory.fromAnyRef(value)) 9 | } 10 | 11 | implicit class RichConfig(val config: Config) extends AnyVal { 12 | def withPair(key: String, value: AnyRef) = config.withValue(key, ConfigValueFactory.fromAnyRef(value)) 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /backend/src/main/resources/base.conf: -------------------------------------------------------------------------------- 1 | include "akka-base.conf" 2 | 3 | image-read-throttle = 3ms 4 | 5 | akka { 6 | 7 | cluster.roles = [${binding.role}, ${binding.name}] 8 | 9 | http { 10 | host-connection-pool.max-connections = 16 11 | parsing.max-content-length = 1g 12 | } 13 | 14 | } 15 | 16 | data-location { 17 | frames { 18 | input = "/usr/local/data/tmt/frames/input" 19 | output = "/usr/local/data/tmt/frames/output" 20 | } 21 | science-images { 22 | input = "/usr/local/data/tmt/science-images/input" 23 | output = "/usr/local/data/tmt/science-images/output" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/clients/OneToOneTransfer.scala: -------------------------------------------------------------------------------- 1 | package tmt.clients 2 | 3 | import java.net.InetSocketAddress 4 | 5 | class OneToOneTransfer(producingClient: ProducingClient, consumingClient: ConsumingClient) { 6 | val flow = producingClient.flow.via(consumingClient.flow) 7 | } 8 | 9 | class OneToOneTransferFactory(producingClientFactory: ProducingClientFactory, consumingClientFactory: ConsumingClientFactory) { 10 | def make(source: InetSocketAddress, destination: InetSocketAddress) = new OneToOneTransfer( 11 | producingClientFactory.make(source), 12 | consumingClientFactory.make(source) 13 | ) 14 | } 15 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/io/ScienceImageReadService.scala: -------------------------------------------------------------------------------- 1 | package tmt.io 2 | 3 | import java.io.File 4 | 5 | import akka.stream.io.SynchronousFileSource 6 | import akka.stream.scaladsl.Source 7 | import tmt.app.AppSettings 8 | 9 | class ScienceImageReadService(settings: AppSettings) { 10 | def send(name: String) = { 11 | val file = new File(s"${settings.scienceImagesInputDir}/$name") 12 | println(s"reading from $file") 13 | SynchronousFileSource(file) 14 | } 15 | 16 | def scienceImages = new File(settings.scienceImagesInputDir).list().toList 17 | 18 | def listScienceImages = Source(() => scienceImages.iterator) 19 | } 20 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/library/IpInfo.scala: -------------------------------------------------------------------------------- 1 | package tmt.library 2 | 3 | import java.io.{InputStreamReader, BufferedReader} 4 | import java.net._ 5 | 6 | object IpInfo { 7 | 8 | private val url = new URL("http://checkip.amazonaws.com") 9 | 10 | def privateIp(env: String) = env match { 11 | case "prod" => InetAddress.getLocalHost.getHostAddress 12 | case _ => "127.0.0.1" 13 | } 14 | 15 | def externalIp(env: String) = env match { 16 | case "prod" => 17 | val reader = new InputStreamReader(url.openStream()) 18 | new BufferedReader(reader).readLine() 19 | case _ => privateIp(env) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/images/Rendering.scala: -------------------------------------------------------------------------------- 1 | package tmt.images 2 | 3 | import org.scalajs.dom._ 4 | import org.scalajs.dom.html.{Canvas, Image} 5 | import tmt.common.{Constants, Stream} 6 | 7 | class Rendering(url: String) { 8 | 9 | val img = document.createElement("img").asInstanceOf[Image] 10 | 11 | def loaded = { 12 | val rendering = Stream.event(img.onload_=).map(_ => this) 13 | img.src = url 14 | rendering 15 | } 16 | 17 | def render(ctx: CanvasRenderingContext2D) = { 18 | ctx.drawImage(img, 0, 0, Constants.CanvasWidth, Constants.CanvasHeight) 19 | Constants.URL.revokeObjectURL(img.src) 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/integration/bridge/StreamListener.scala: -------------------------------------------------------------------------------- 1 | package tmt.integration.bridge 2 | 3 | import akka.actor.Status.{Failure, Success} 4 | import akka.stream.scaladsl.Sink 5 | import akka.stream.{Materializer, OverflowStrategy} 6 | import tmt.integration.camera.Listener 7 | import tmt.library.Connector 8 | 9 | class StreamListener[T](implicit mat: Materializer) extends Listener[T] { 10 | val (actorRef, source) = Connector.coupling[T](Sink.publisher) 11 | override def onEvent(event: T) = actorRef ! event 12 | override def onError(ex: Throwable) = actorRef ! Failure(ex) 13 | override def onComplete() = actorRef ! Success("done") 14 | } 15 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/io/ScienceImageWriteService.scala: -------------------------------------------------------------------------------- 1 | package tmt.io 2 | 3 | import java.io.File 4 | 5 | import akka.stream.io.SynchronousFileSink 6 | import akka.stream.scaladsl.Source 7 | import akka.util.ByteString 8 | import tmt.app.{AppSettings, ActorConfigs} 9 | 10 | class ScienceImageWriteService(actorConfigs: ActorConfigs, settings: AppSettings) { 11 | 12 | import actorConfigs._ 13 | 14 | def copy(name: String, byteArrays: Source[ByteString, Any]) = { 15 | val file = new File(s"${settings.scienceImagesOutputDir}/$name") 16 | println(s"writing to $file") 17 | byteArrays.runWith(SynchronousFileSink(file)).map(_ => ()) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/views/FrequencyView.scala: -------------------------------------------------------------------------------- 1 | package tmt.views 2 | 3 | import monifu.concurrent.Scheduler 4 | import rx.core.Rx 5 | import tmt.app.ViewData 6 | import tmt.framework.Framework._ 7 | import tmt.framework.Helpers._ 8 | import tmt.metrics.FrequencyRendering 9 | 10 | import scalatags.JsDom.all._ 11 | 12 | class FrequencyView(viewData: ViewData)(implicit scheduler: Scheduler) extends View { 13 | val frequencyRendering = new FrequencyRendering(viewData) 14 | 15 | def viewTitle = frequencyRendering.title 16 | 17 | def viewContent = div(frequencyRendering.frequency) 18 | 19 | def viewAction = makeForm(viewData.metricServers, frequencyRendering) 20 | } 21 | -------------------------------------------------------------------------------- /frontend/conf/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %coloredLevel - %logger - %message%n%xException 6 | 7 | 8 | 9 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /diagrams/throttle: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | participant Client 4 | 5 | box "Frontend" #LightBlue 6 | participant StreamController 7 | participant ClusterClientService 8 | end box 9 | 10 | participant DistributedPubSub 11 | 12 | box "Backend" #LightBlue 13 | participant "Ticker(s)" 14 | end box 15 | 16 | "Ticker(s)" -> DistributedPubSub: Subscribe("throttle", self) 17 | Client -> StreamController: wavefront1/throttle/10 18 | StreamController -> ClusterClientService: throttle(wavefront1, 10) 19 | ClusterClientService --> DistributedPubSub: Publish("throttle", UpdateDelay(wavefront1, 10)) 20 | StreamController -> Client: Accepted 21 | DistributedPubSub --> "Ticker(s)": UpdateDelay(wavefront1, 10) 22 | 23 | 24 | @enduml 25 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/app/CustomDirectives.scala: -------------------------------------------------------------------------------- 1 | package tmt.app 2 | 3 | import akka.http.scaladsl.model.ws.{Message, UpgradeToWebsocket} 4 | import akka.http.scaladsl.server._ 5 | import akka.http.scaladsl.server.directives.HeaderDirectives._ 6 | import akka.http.scaladsl.server.directives.RouteDirectives._ 7 | import akka.stream.scaladsl.{Sink, Source} 8 | 9 | trait CustomDirectives { 10 | 11 | def handleWebsocketMessages(inSink: Sink[Message, Any], outSource: Source[Message, Any]): Route = 12 | optionalHeaderValueByType[UpgradeToWebsocket]() { 13 | case Some(upgrade) ⇒ complete(upgrade.handleMessagesWithSinkSource(inSink, outSource)) 14 | case None ⇒ reject(ExpectedWebsocketRequestRejection) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/app/NodeInfoPublisher.scala: -------------------------------------------------------------------------------- 1 | package tmt.app 2 | 3 | import akka.cluster.Cluster 4 | import akka.cluster.ddata.Replicator.{Update, WriteLocal} 5 | import akka.cluster.ddata.{DistributedData, LWWMap} 6 | import tmt.common.Keys 7 | import tmt.shared.models.NodeS 8 | 9 | class NodeInfoPublisher(actorConfigs: ActorConfigs, appSettings: AppSettings) { 10 | 11 | import actorConfigs._ 12 | 13 | val replicator = DistributedData(system).replicator 14 | implicit val cluster = Cluster(system) 15 | 16 | def publish(httpPort: Int) = replicator ! Update(Keys.Nodes, LWWMap.empty[NodeS], WriteLocal) { map => 17 | map + (appSettings.binding.name -> NodeS.fromNode(appSettings.binding.roleMapping(httpPort))) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/framework/ScienceImageRx.scala: -------------------------------------------------------------------------------- 1 | package tmt.framework 2 | 3 | import monifu.concurrent.Scheduler 4 | import org.scalajs.dom.ext.Ajax 5 | import prickle.Unpickle 6 | import rx._ 7 | import tmt.app.ViewData 8 | 9 | import scala.async.Async._ 10 | 11 | class ScienceImageRx(viewData: ViewData)(implicit scheduler: Scheduler) extends FormRx("Science Images", viewData) { 12 | 13 | def getUrl = viewData.nodeSet().getScienceImageUrl(server()) 14 | 15 | val imageNames = Var(Seq.empty[String]) 16 | 17 | Obs(currentUrl, skipInitial = true) { 18 | async { 19 | val responseText = await(Ajax.get(currentUrl())).responseText 20 | imageNames() = Unpickle[Seq[String]].fromString(responseText).get 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/views/ImageView.scala: -------------------------------------------------------------------------------- 1 | package tmt.views 2 | 3 | import monifu.concurrent.Scheduler 4 | import rx.core.Rx 5 | import tmt.app.ViewData 6 | import tmt.framework.Helpers._ 7 | import tmt.framework.Framework._ 8 | import tmt.images.ImageRendering 9 | 10 | import scalatags.JsDom.all._ 11 | 12 | class ImageView(viewData: ViewData)(implicit scheduler: Scheduler) extends View { 13 | 14 | import tmt.common.Constants._ 15 | 16 | val cvs = canvas(widthA := CanvasWidth, heightA := CanvasHeight).render 17 | 18 | val imageRendering = new ImageRendering(cvs, viewData) 19 | 20 | def viewTitle = imageRendering.title 21 | 22 | def viewContent = cvs 23 | 24 | def viewAction = makeForm(viewData.imageServers, imageRendering) 25 | } 26 | -------------------------------------------------------------------------------- /frontend/conf/routes: -------------------------------------------------------------------------------- 1 | GET / controllers.StreamController.streams() 2 | GET /nodes controllers.StreamController.nodes() 3 | GET /connections controllers.StreamController.connections() 4 | 5 | POST /:server/throttle/:delay controllers.StreamController.throttle(server, delay: Long) 6 | POST /:server/subscribe/:topic controllers.StreamController.subscribe(server, topic) 7 | POST /:server/unsubscribe/:topic controllers.StreamController.unsubscribe(server, topic) 8 | 9 | # Map static resources from the /public folder to the /assets URL path 10 | GET /assets/*file controllers.Assets.at(path="/public", file) 11 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/library/FlowExtensions.scala: -------------------------------------------------------------------------------- 1 | package tmt.library 2 | 3 | import akka.stream.scaladsl._ 4 | 5 | import scala.concurrent.duration.FiniteDuration 6 | 7 | object FlowExtensions { 8 | 9 | implicit class RichFlow[In, Out, Mat](val flow: Flow[In, Out, Mat]) extends AnyVal { 10 | 11 | def zip[Out2, Mat2](other: Source[Out2, Mat2]) = Flow(flow) { implicit b => flw => 12 | import FlowGraph.Implicits._ 13 | 14 | val zipper = b.add(Zip[Out, Out2]()) 15 | 16 | flw.outlet ~> zipper.in0 17 | other ~> zipper.in1 18 | 19 | (flw.inlet, zipper.out) 20 | } 21 | 22 | def zipWithIndex = zip(Source(() => Iterator.from(0))) 23 | 24 | def throttle(duration: FiniteDuration) = flow.zip(Sources.ticks(duration)).map(_._1) 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/actors/SourceActorLink.scala: -------------------------------------------------------------------------------- 1 | package tmt.actors 2 | 3 | import akka.actor.ActorRef 4 | import akka.cluster.pubsub.DistributedPubSub 5 | import akka.cluster.pubsub.DistributedPubSubMediator.Subscribe 6 | import akka.stream.scaladsl.Sink 7 | import tmt.app.ActorConfigs 8 | import tmt.library.Connector 9 | 10 | abstract class SourceActorLink[T](actorConfigs: ActorConfigs, topics: String*) { 11 | def wrap(sourceLinkedRef: ActorRef): ActorRef 12 | 13 | import actorConfigs._ 14 | private val (sourceLinkedRef, _source) = Connector.coupling[T](Sink.publisher) 15 | private val mediator = DistributedPubSub(system).mediator 16 | val wrappedActor = wrap(sourceLinkedRef) 17 | topics.foreach(topic => mediator ! Subscribe(topic, wrappedActor)) 18 | 19 | def source = _source 20 | } 21 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/app/Main.scala: -------------------------------------------------------------------------------- 1 | package tmt.app 2 | 3 | import scala.concurrent.Await 4 | import scala.concurrent.duration._ 5 | 6 | object Main extends App { 7 | args match { 8 | case Array(role, serverName, env, seedName) => new Main(role, serverName, env, Some(seedName)) 9 | case Array(role, serverName, env) => new Main(role, serverName, env, None) 10 | } 11 | } 12 | 13 | class Main(role: String, serverName: String, env: String, seedName: Option[String]) { 14 | val assembly = new Assembly(role, serverName, env, seedName) 15 | val server = assembly.serverFactory.make() 16 | val binding = Await.result(server.run(), 1.second) 17 | assembly.nodeInfoPublisher.publish(binding.localAddress.getPort) 18 | 19 | def stop() = { 20 | Await.result(binding.unbind(), 1.second) 21 | assembly.system.terminate() 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/marshalling/BinaryMarshallers.scala: -------------------------------------------------------------------------------- 1 | package tmt.marshalling 2 | 3 | import akka.http.scaladsl.marshalling.Marshaller 4 | import akka.http.scaladsl.model.HttpEntity.Chunked 5 | import akka.http.scaladsl.model.{ContentTypes, HttpEntity} 6 | import akka.http.scaladsl.unmarshalling.Unmarshaller 7 | import akka.stream.scaladsl.Source 8 | 9 | trait BinaryMarshallers { 10 | implicit def byteStringMarshaller[T: BFormat]: Marshaller[Source[T, Any], Chunked] = Marshaller.opaque { source: Source[T, Any] => 11 | val byteStrings = source.map(BFormat[T].write) 12 | HttpEntity.Chunked.fromData(ContentTypes.`application/octet-stream`, byteStrings) 13 | } 14 | 15 | implicit def byteStringUnmarshaller[T: BFormat]: Unmarshaller[HttpEntity, Source[T, Any]] = Unmarshaller.strict { entity: HttpEntity => 16 | entity.dataBytes.map(BFormat[T].read) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/views/ScienceImageView.scala: -------------------------------------------------------------------------------- 1 | package tmt.views 2 | 3 | import monifu.concurrent.Scheduler 4 | import rx._ 5 | import tmt.app.ViewData 6 | import tmt.framework.Framework._ 7 | import tmt.framework.Helpers._ 8 | import tmt.framework.ScienceImageRx 9 | 10 | import scalatags.JsDom.all._ 11 | 12 | class ScienceImageView(viewData: ViewData)(implicit scheduler: Scheduler) extends View { 13 | 14 | val scienceImageRx = new ScienceImageRx(viewData) 15 | 16 | def viewTitle = Rx(h5(scienceImageRx.selectedServer().capitalize)) 17 | 18 | def viewContent = Rx { 19 | div(cls := "collection")( 20 | scienceImageRx.imageNames().map { name => 21 | a(cls := "collection-item")(href := s"${scienceImageRx.currentUrl()}/$name")(name) 22 | } 23 | ) 24 | } 25 | 26 | def viewAction = makeForm(viewData.scienceImageServers, scienceImageRx) 27 | } 28 | -------------------------------------------------------------------------------- /shared/src/main/scala/tmt/shared/models/ConnectionSet.scala: -------------------------------------------------------------------------------- 1 | package tmt.shared.models 2 | 3 | case class ConnectionSet(connections: Set[Connection]) { 4 | lazy val index = connections.groupBy(_.server).mapValues(_.map(_.topic)) 5 | 6 | def getTopics(serverName: String) = index.getOrElse(serverName, Set.empty) 7 | 8 | def add(connection: Connection) = ConnectionSet(connections + connection) 9 | def remove(connection: Connection) = ConnectionSet(connections - connection) 10 | 11 | def pruneBy(onlineRoles: Set[String]) = ConnectionSet( 12 | connections.filter(_.isOnline(onlineRoles)) 13 | ) 14 | } 15 | 16 | object ConnectionSet { 17 | def empty = ConnectionSet(Set.empty) 18 | } 19 | 20 | case class Connection(server: String, topic: String) { 21 | def isOnline(onlineRoles: Set[String]) = 22 | onlineRoles.contains(server) && onlineRoles.contains(topic) 23 | } 24 | -------------------------------------------------------------------------------- /diagrams/subscribe: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | participant Client 4 | 5 | box "Frontend" #LightBlue 6 | participant StreamController 7 | participant ClusterClientService 8 | participant ConnectionStore 9 | end box 10 | 11 | participant DistributedPubSub 12 | 13 | box "Backend" #LightBlue 14 | participant "Subscription(s)" 15 | end box 16 | 17 | "Subscription(s)" -> DistributedPubSub: Subscribe("subscription", self) 18 | Client -> StreamController: server1/subscribe/topic1 19 | StreamController -> ClusterClientService: subscribe(server1, topic1) 20 | ClusterClientService --> DistributedPubSub: Publish(\n"subscription", \nMessages.Subscribe(server1, topic1)\n) 21 | StreamController -> Client: Accepted 22 | DistributedPubSub --> "Subscription(s)": Messages.Subscribe(server1, topic1) 23 | DistributedPubSub --> ConnectionStore: Messages.Subscribe(server1, topic1) 24 | 25 | @enduml 26 | -------------------------------------------------------------------------------- /diagrams/unsubscribe: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | participant Client 4 | 5 | box "Frontend" #LightBlue 6 | participant StreamController 7 | participant ClusterClientService 8 | participant ConnectionStore 9 | end box 10 | 11 | participant DistributedPubSub 12 | 13 | box "Backend" #LightBlue 14 | participant "Subscription(s)" 15 | end box 16 | 17 | "Subscription(s)" -> DistributedPubSub: Subscribe("subscription", self) 18 | Client -> StreamController: server1/unsubscribe/topic1 19 | StreamController -> ClusterClientService: unsubscribe(server1, topic1) 20 | ClusterClientService --> DistributedPubSub: Publish(\n"subscription", \nMessages.Unsubscribe(server1, topic1)\n) 21 | StreamController -> Client: Accepted 22 | DistributedPubSub --> "Subscription(s)": Messages.Unsubscribe(server1, topic1) 23 | DistributedPubSub --> ConnectionStore: Messages.Unsubscribe(server1, topic1) 24 | 25 | @enduml 26 | -------------------------------------------------------------------------------- /diagrams/connections: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | participant Client 4 | 5 | box "Frontend" #LightBlue 6 | participant StreamController 7 | participant ClusterClientService 8 | participant ConnectionStore 9 | participant ConnectionSetService 10 | participant ClusterMetadataService 11 | end box 12 | 13 | Client -> StreamController: /connections 14 | StreamController -> ConnectionSetService: connectionSet() 15 | ConnectionSetService -> ClusterClientService: allConnections() 16 | ClusterClientService -> ConnectionStore: GetConnections 17 | ConnectionStore -> ClusterClientService: Connections 18 | ClusterClientService -> ConnectionSetService: Connections 19 | ConnectionSetService -> ClusterMetadataService: onlineRoles() 20 | ClusterMetadataService ->ConnectionSetService: OnlineRoles 21 | ConnectionSetService -> StreamController: OnlineConnections 22 | StreamController -> Client: OnlineConnections 23 | @enduml -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/integration/camera/Simulator.scala: -------------------------------------------------------------------------------- 1 | package tmt.integration.camera 2 | 3 | import java.util.concurrent.Executors 4 | 5 | import scala.concurrent.duration._ 6 | import scala.util.control.NonFatal 7 | 8 | class Simulator[T](producer: Iterator[T]) { 9 | private val delay = 1.milli 10 | 11 | private val singleThreadScheduler = Executors.newScheduledThreadPool(1) 12 | 13 | def subscribe(listener: Listener[T]) = singleThreadScheduler.scheduleAtFixedRate( 14 | runnable(listener), 15 | delay.length, 16 | delay.length, 17 | delay.unit 18 | ) 19 | 20 | private def runnable(listener: Listener[T]): Runnable = new Runnable { 21 | def run() = try { 22 | if (producer.hasNext) 23 | listener.onEvent(producer.next()) 24 | else 25 | listener.onComplete() 26 | } catch { 27 | case NonFatal(ex) => listener.onError(ex) 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/views/Body.scala: -------------------------------------------------------------------------------- 1 | package tmt.views 2 | 3 | import scalatags.JsDom.all._ 4 | 5 | class Body( 6 | throttleView: ThrottleView, 7 | subscriptionView: SubscriptionView, 8 | scienceImageView: ScienceImageView, 9 | frequencyViews: Seq[FrequencyView], 10 | streamViews: Seq[ImageView] 11 | ) { 12 | 13 | def layout = div(cls := "container")( 14 | div(cls := "row")( 15 | div(cls := "col l4")( 16 | makeCard(throttleView), 17 | makeCard(subscriptionView), 18 | makeCard(scienceImageView) 19 | ), 20 | div(cls := "col l8")( 21 | (frequencyViews ++ streamViews).map(makeCard) 22 | ) 23 | ) 24 | ) 25 | 26 | def makeCard(view: View) = div(cls := "card")( 27 | div(cls := "card-content")( 28 | view.viewTitle, 29 | view.viewContent 30 | ), 31 | div(cls := "card-action")( 32 | view.viewAction 33 | ) 34 | ) 35 | } 36 | 37 | 38 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/app/ViewData.scala: -------------------------------------------------------------------------------- 1 | package tmt.app 2 | 3 | import rx._ 4 | import tmt.shared.models._ 5 | 6 | class ViewData(val nodeSet: Rx[NodeSet], val connectionSet: Rx[ConnectionSet]) { 7 | val producers = Rx(nodeSet().producers) 8 | def consumersOf(topicName: Rx[String]) = Rx(nodeSet().compatibleConsumers(topicName())) 9 | 10 | val imageServers = Rx(nodeSet().getNames(ItemType.Image)) 11 | val metricServers = Rx(nodeSet().getNames(ItemType.Metric)) 12 | 13 | val allServers = Rx(imageServers() ++ metricServers()) 14 | 15 | val diffs = { 16 | val result = Var(Seq.empty[String]) 17 | var current = allServers() 18 | Obs(allServers) { 19 | result() = allServers().diff(current) 20 | current = allServers() 21 | } 22 | result 23 | } 24 | 25 | val wavefrontServers = Rx(nodeSet().getNames(Role.Wavefront)) 26 | val scienceImageServers = Rx(nodeSet().getNames(Role.ScienceImageSource)) 27 | } 28 | -------------------------------------------------------------------------------- /frontend/app/services/NodeSetService.scala: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import javax.inject.{Inject, Singleton} 4 | 5 | import common.AppSettings 6 | import tmt.shared.models.{Connection, ItemType} 7 | import scala.async.Async._ 8 | import scala.concurrent.ExecutionContext 9 | 10 | @Singleton 11 | class NodeSetService @Inject()( 12 | appSettings: AppSettings, 13 | clusterMetadataService: ClusterMetadataService)(implicit ec: ExecutionContext) { 14 | 15 | def onlineNodeSet = async { 16 | await(clusterMetadataService.nodeSet).pruneBy(clusterMetadataService.onlineRoles) 17 | } 18 | 19 | def validate(connection: Connection) = async { 20 | val Connection(serverName, topic) = connection 21 | val nodeSet = await(onlineNodeSet) 22 | val inputType = nodeSet.getRole(serverName).input 23 | val outputType = nodeSet.getRole(topic).output 24 | (serverName != topic) && (inputType != ItemType.Empty) && (inputType == outputType) 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/marshalling/BFormat.scala: -------------------------------------------------------------------------------- 1 | package tmt.marshalling 2 | 3 | import akka.util.ByteString 4 | import boopickle.Default.Pickle 5 | import boopickle.Default.Unpickle 6 | import boopickle.Pickler 7 | 8 | trait BFormat[T] { 9 | def read(byteString: ByteString): T 10 | def write(o: T): ByteString 11 | } 12 | 13 | object BFormat { 14 | def apply[T: BFormat] = implicitly[BFormat[T]] 15 | 16 | def make[T](fromBinary: ByteString => T, toBinary: T => ByteString): BFormat[T] = new BFormat[T] { 17 | def read(byteString: ByteString) = fromBinary(byteString) 18 | def write(o: T) = toBinary(o) 19 | } 20 | 21 | implicit val stringFormat = make[String](_.utf8String, ByteString.apply) 22 | implicit val byeStringFormat = make[ByteString](identity, identity) 23 | 24 | implicit def objectFormat[T: Pickler]: BFormat[T] = BFormat.make[T]( 25 | x => Unpickle[T].fromBytes(x.toByteBuffer), 26 | x => ByteString(Pickle.intoBytes(x)) 27 | ) 28 | } 29 | -------------------------------------------------------------------------------- /shared/src/main/scala/tmt/shared/models/metrics.scala: -------------------------------------------------------------------------------- 1 | package tmt.shared.models 2 | 3 | import boopickle.Default._ 4 | 5 | case class ImageMetadata(name: String, size: Int, createdAt: Long) { 6 | def latency = System.currentTimeMillis() - createdAt 7 | } 8 | 9 | case class ImageMetric(size: Int, latency: Long) 10 | 11 | object ImageMetric { 12 | def from(imageInfo: ImageMetadata) = ImageMetric(imageInfo.size, imageInfo.latency) 13 | } 14 | 15 | object ImageMetadata { 16 | implicit val pickler = generatePickler[ImageMetadata] 17 | } 18 | 19 | case class PerSecMetric(size: Int, count: Int, latency: Double) 20 | 21 | object PerSecMetric { 22 | def from(imageMetrics: Seq[ImageMetric]) = { 23 | val totalLatency = imageMetrics.map(_.latency).sum 24 | PerSecMetric( 25 | imageMetrics.map(_.size).sum, 26 | imageMetrics.length, 27 | totalLatency.toDouble / imageMetrics.length 28 | ) 29 | } 30 | 31 | implicit val pickler = generatePickler[PerSecMetric] 32 | } 33 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/common/Stream.scala: -------------------------------------------------------------------------------- 1 | package tmt.common 2 | 3 | import monifu.reactive.Observable 4 | import org.scalajs.dom.{ErrorEvent, Event, MessageEvent, WebSocket} 5 | 6 | import scala.scalajs.js 7 | 8 | object Stream { 9 | def socket(socket: WebSocket) = Observable.create[MessageEvent] { subscriber => 10 | socket.onopen = { e: Event => 11 | println("***********open") 12 | } 13 | socket.onmessage = { e: MessageEvent => 14 | subscriber.onNext(e) 15 | } 16 | socket.onclose = { e: Event => 17 | println("**************closed") 18 | subscriber.onComplete() 19 | } 20 | socket.onerror = { e: ErrorEvent => 21 | println("**************error") 22 | subscriber.onError(throw new RuntimeException(e.message)) 23 | } 24 | } 25 | 26 | def event[T <: Event](listener: js.Function1[T, _] => Unit) = Observable.create[T] { subscriber => 27 | listener { e: T => 28 | subscriber.onNext(e) 29 | subscriber.onComplete() 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /diagrams/nodes: -------------------------------------------------------------------------------- 1 | @startuml 2 | 3 | participant Client 4 | 5 | box "Frontend" #LightBlue 6 | participant StreamController 7 | participant NodeSetService 8 | participant ClusterMetadataService 9 | end box 10 | 11 | participant DistributedDataReplicator 12 | 13 | box "Backend" #LightBlue 14 | participant NodeInfoPublisher 15 | end box 16 | 17 | NodeInfoPublisher -> DistributedDataReplicator: Update(\n"nodes", \n"node-name"-> "node-details"\n) 18 | Client -> StreamController: /nodes 19 | StreamController -> NodeSetService: onlineNodeSet() 20 | NodeSetService -> ClusterMetadataService: nodeSet() 21 | ClusterMetadataService -> DistributedDataReplicator: Get("nodes") 22 | DistributedDataReplicator-> ClusterMetadataService: NodeSet 23 | ClusterMetadataService -> NodeSetService: NodeSet 24 | NodeSetService -> ClusterMetadataService: onlineRoles() 25 | ClusterMetadataService -> NodeSetService: OnlineRoles 26 | NodeSetService -> StreamController: OnlineNodeSet 27 | StreamController -> Client: OnlineNodeSet 28 | 29 | @enduml -------------------------------------------------------------------------------- /client/src/main/scala/tmt/app/DataStore.scala: -------------------------------------------------------------------------------- 1 | package tmt.app 2 | 3 | import monifu.concurrent.Scheduler 4 | import org.scalajs.dom.ext.Ajax 5 | import prickle._ 6 | import rx._ 7 | import tmt.common.Constants 8 | import tmt.shared.Topics 9 | import tmt.shared.models._ 10 | 11 | import scala.async.Async._ 12 | import scala.concurrent.duration._ 13 | 14 | class DataStore(implicit scheduler: Scheduler) { 15 | 16 | scheduleRefresh(Constants.RefreshRate) 17 | 18 | private val nodeSet = Var(NodeSet.empty) 19 | private val connectionSet = Var(ConnectionSet.empty) 20 | 21 | val data = new ViewData(nodeSet, connectionSet) 22 | 23 | def scheduleRefresh(delay: FiniteDuration) = { 24 | scheduler.scheduleWithFixedDelay(0.seconds, delay) { 25 | async { 26 | val nSet = Unpickle[NodeSet].fromString(await(Ajax.get(Topics.Nodes)).responseText).get 27 | val cSet = Unpickle[ConnectionSet].fromString(await(Ajax.get(Topics.Connections)).responseText).get 28 | nodeSet() = nSet 29 | connectionSet() = cSet 30 | } 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/server/Server.scala: -------------------------------------------------------------------------------- 1 | package tmt.server 2 | 3 | import java.net.InetSocketAddress 4 | 5 | import akka.http.scaladsl.Http 6 | import akka.stream.scaladsl.Sink 7 | import tmt.app.{ActorConfigs, Types} 8 | 9 | import scala.util.{Failure, Success} 10 | 11 | class Server(address: InetSocketAddress, connectionFlow: Types.ConnectionFlow, actorConfigs: ActorConfigs) { 12 | import actorConfigs._ 13 | 14 | private val runnableGraph = { 15 | val connections = Http().bind(address.getHostName, address.getPort) 16 | 17 | connections.to(Sink.foreach { connection => 18 | println(s"Accepted new connection from: ${connection.remoteAddress}") 19 | connection.handleWith(connectionFlow) 20 | }) 21 | } 22 | 23 | def run() = { 24 | val binding = runnableGraph.run() 25 | 26 | binding.onComplete { 27 | case Success(b) => println(s"Server started, listening on: ${b.localAddress}") 28 | case Failure(e) => println(s"Server could not bind to $address due to: ${e.getMessage}"); system.terminate() 29 | } 30 | 31 | binding 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/transformations/ImageTransformations.scala: -------------------------------------------------------------------------------- 1 | package tmt.transformations 2 | 3 | import akka.stream.scaladsl.Source 4 | import tmt.actors.SubscriptionService 5 | import tmt.app.{AppSettings, ActorConfigs} 6 | import tmt.io.WavefrontWriteService 7 | import tmt.shared.models.{Image, ImageMetadata} 8 | 9 | class ImageTransformations( 10 | wavefrontWriteService: WavefrontWriteService, 11 | actorConfigs: ActorConfigs, 12 | appSettings: AppSettings, 13 | imageSubscriber: SubscriptionService[Image], 14 | imageRotationUtility: ImageProcessor) { 15 | 16 | import actorConfigs._ 17 | 18 | lazy val images: Source[Image, Unit] = imageSubscriber.source 19 | 20 | lazy val filteredImages = images.filter(_.name.contains("9")) 21 | 22 | lazy val copiedImages = images.mapAsync(1) { image => 23 | wavefrontWriteService.copyImage(image).map(_ => image) 24 | } 25 | 26 | lazy val imageMetadata = images.map(image => ImageMetadata(image.name, image.size, image.createdAt)) 27 | 28 | lazy val rotatedImages = images.mapAsync( 29 | appSettings.imageProcessingParallelism 30 | )(imageRotationUtility.rotate) 31 | } 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | TMT-Bulkdata 3 | ============ 4 | 5 | Report 6 | ------ 7 | https://docs.google.com/document/d/1L7bLMSYixV2xf42YjDp9IU9eJmzLQfUhNoQyOyNhpCs/edit 8 | 9 | Slides 10 | ------ 11 | https://docs.google.com/presentation/d/1c6NhHBFh-SdMWpIuarHAbSZ7Ia6eOt9op6SG42TdAaU/edit#slide=id.p 12 | 13 | How to run 14 | ---------- 15 | ``` 16 | ./activator run 17 | 18 | ./activator "backend/runMain tmt.app.Main wavefront wavefront1 dev seed1" 19 | 20 | ./activator "backend/runMain tmt.app.Main metadata metadata1 dev" 21 | 22 | ./activator "backend/runMain tmt.app.Main metric metric1 dev" 23 | 24 | ./activator "backend/runMain tmt.app.Main rotator rotator1 dev" 25 | 26 | ./activator "backend/runMain tmt.app.Main science-image-source camera1 dev" 27 | ``` 28 | 29 | Data Setup 30 | ---------- 31 | 32 | - Download [video](http://distribution.bbb3d.renderfarming.net/video/mp4/bbb_sunflower_1080p_60fps_normal.mp4) 33 | - Install [ffmpeg](https://www.ffmpeg.org/) 34 | - `mkdir input` 35 | - `cd input` 36 | - Split the video at 20 fps: `ffmpeg -i ../bbb_sunflower_1080p_60fps_normal.mp4 -r 20 image-%05d.jpg` 37 | - `cd ..` 38 | - `mv input /usr/local/data/tmt/frames` 39 | -------------------------------------------------------------------------------- /common/src/main/resources/akka-base.conf: -------------------------------------------------------------------------------- 1 | include "akka-http-core-reference.conf" 2 | 3 | akka.scheduler.tick-duration = 1ms 4 | 5 | akka { 6 | // loglevel = "DEBUG" 7 | 8 | actor.provider = "akka.cluster.ClusterActorRefProvider" 9 | 10 | remote { 11 | log-remote-lifecycle-events = off 12 | netty.tcp = ${binding} 13 | netty.tcp.maximum-frame-size = 256000b 14 | } 15 | 16 | cluster { 17 | seed-nodes = [ 18 | "akka.tcp://ClusterSystem@"${seed1.address} 19 | "akka.tcp://ClusterSystem@"${seed2.address} 20 | ] 21 | 22 | // auto-down-unreachable-after = 60s 23 | 24 | # Disable legacy metrics in akka-cluster. 25 | metrics.enabled = off 26 | 27 | # Sigar native library extract location during tests. 28 | # Note: use per-jvm-instance folder when running multiple jvm on one host. 29 | metrics.native-library-extract-folder = ${user.dir}/target/native 30 | } 31 | 32 | extensions = [ 33 | "akka.cluster.client.ClusterClientReceptionist" 34 | "akka.cluster.pubsub.DistributedPubSub" 35 | "akka.cluster.metrics.ClusterMetricsExtension" 36 | "akka.cluster.ddata.DistributedData" 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/app/ConfigLoader.scala: -------------------------------------------------------------------------------- 1 | package tmt.app 2 | 3 | import com.typesafe.config._ 4 | import tmt.library.ConfigObjectExtensions.RichConfig 5 | import tmt.library.IpInfo 6 | 7 | class ConfigLoader { 8 | 9 | def load(role: String, serverName: String, env: String, seedName: Option[String]) = { 10 | val config = parse(env) 11 | 12 | val (privateIp, port) = seedName match { 13 | case Some(seed) => (config.getString(s"$seed.hostname"), config.getInt(s"$seed.port")) 14 | case None => (IpInfo.privateIp(env), 0) 15 | } 16 | 17 | val bindingConfig = ConfigFactory.empty() 18 | .withPair("role", role) 19 | .withPair("name", serverName) 20 | .withPair("hostname", privateIp) 21 | .withPair("port", Integer.valueOf(port)) 22 | .withPair("externalIp", IpInfo.externalIp(env)) 23 | 24 | val binding = ConfigFactory.empty().withValue("binding", bindingConfig.root()) 25 | 26 | config.withFallback(binding).resolve() 27 | } 28 | 29 | def parse(name: String) = ConfigFactory.load( 30 | name, 31 | ConfigParseOptions.defaults(), 32 | ConfigResolveOptions.defaults().setAllowUnresolved(true) 33 | ) 34 | } 35 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/metrics/FrequencyRendering.scala: -------------------------------------------------------------------------------- 1 | package tmt.metrics 2 | 3 | import boopickle.Default 4 | import monifu.concurrent 5 | import monifu.reactive.Observable 6 | import org.scalajs.dom._ 7 | import rx._ 8 | import rx.ops._ 9 | import tmt.app.ViewData 10 | import tmt.common.Stream 11 | import tmt.framework.WebsocketRx 12 | import tmt.shared.models.PerSecMetric 13 | 14 | import scala.scalajs.js.typedarray.{ArrayBuffer, TypedArrayBuffer} 15 | 16 | class FrequencyRendering(viewData: ViewData)(implicit scheduler: concurrent.Scheduler) extends WebsocketRx("Metric", viewData) { 17 | 18 | def cleanup() = { 19 | frequency() = "" 20 | } 21 | 22 | val frequency = Var("") 23 | 24 | val stream = webSocket.map { 25 | case None => Observable.empty 26 | case Some(socket) => Stream.socket(socket).map(makeItem) 27 | } 28 | 29 | Obs(stream) { 30 | import prickle._ 31 | stream().foreach(metric => frequency() = Pickle.intoString(metric)) 32 | } 33 | 34 | def makeItem(messageEvent: MessageEvent) = { 35 | val arrayBuffer = messageEvent.data.asInstanceOf[ArrayBuffer] 36 | val byteBuffer = TypedArrayBuffer.wrap(arrayBuffer) 37 | Default.Unpickle[PerSecMetric].fromBytes(byteBuffer) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/views/ThrottleView.scala: -------------------------------------------------------------------------------- 1 | package tmt.views 2 | 3 | import org.scalajs.dom.ext.Ajax 4 | import rx._ 5 | import tmt.app.ViewData 6 | import tmt.framework.Framework._ 7 | import tmt.framework.Helpers._ 8 | 9 | import scalatags.JsDom.all._ 10 | 11 | class ThrottleView(dataStore: ViewData) extends View { 12 | val server = Var("") 13 | val rate = Var("") 14 | 15 | val noData = Rx(server().isEmpty || dataStore.wavefrontServers().isEmpty) 16 | 17 | val disabledStyle = Rx(if(noData()) "disabled" else "") 18 | 19 | def viewTitle = h5("Throttle") 20 | 21 | def viewContent = div( 22 | label("Wavefront to throttle"), 23 | makeSelection(dataStore.wavefrontServers, server), 24 | 25 | label("New rate"), 26 | p(cls := "range-field")( 27 | input(`type` := "range", min := 3, max := 100)( 28 | onchange := { setValue(rate) } 29 | ) 30 | ) 31 | ) 32 | 33 | def viewAction = { 34 | button(cls := Rx(s"waves-effect waves-light btn ${disabledStyle()}"))( 35 | `type` := "submit", 36 | onclick := { () => if(!noData()) throttle(server(), rate()) } 37 | )("Change") 38 | } 39 | 40 | def throttle(server: String, rate: String) = Ajax.post(s"$server/throttle/$rate") 41 | 42 | } 43 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/clients/OneToManyTransfer.scala: -------------------------------------------------------------------------------- 1 | package tmt.clients 2 | 3 | import java.net.InetSocketAddress 4 | 5 | import akka.http.scaladsl.model.{HttpResponse, MessageEntity, Uri} 6 | import akka.stream.scaladsl.{Broadcast, Flow, FlowGraph, Merge} 7 | 8 | class OneToManyTransfer(producingClient: ProducingClient, consumingClients: Seq[ConsumingClient]) { 9 | 10 | val flow = Flow() { implicit b => 11 | import FlowGraph.Implicits._ 12 | 13 | val pipe = b.add(producingClient.flow) 14 | val broadcast = b.add(Broadcast[(MessageEntity, Uri)](consumingClients.size)) 15 | val merge = b.add(Merge[HttpResponse](consumingClients.size)) 16 | 17 | pipe.outlet ~> broadcast.in 18 | 19 | consumingClients.zipWithIndex.foreach { case (consumingClient, i) => 20 | broadcast.out(i) ~> consumingClient.flow ~> merge.in(i) 21 | } 22 | 23 | (pipe.inlet, merge.out) 24 | } 25 | 26 | } 27 | 28 | class OneToManyTransferFactory(producingClientFactory: ProducingClientFactory, consumingClientFactory: ConsumingClientFactory) { 29 | def make(source: InetSocketAddress, destinations: Seq[InetSocketAddress]) = new OneToManyTransfer( 30 | producingClientFactory.make(source), 31 | destinations.map(consumingClientFactory.make) 32 | ) 33 | } 34 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/io/WavefrontWriteService.scala: -------------------------------------------------------------------------------- 1 | package tmt.io 2 | 3 | import java.io.File 4 | import java.nio.file.Files 5 | 6 | import akka.stream.io.SynchronousFileSink 7 | import akka.stream.scaladsl.{Sink, Source} 8 | import akka.util.ByteString 9 | import tmt.app.{ActorConfigs, AppSettings} 10 | import tmt.library.SourceExtensions.RichSource 11 | import tmt.shared.models.Image 12 | 13 | import scala.concurrent.Future 14 | 15 | class WavefrontWriteService(actorConfigs: ActorConfigs, settings: AppSettings) { 16 | 17 | import actorConfigs._ 18 | 19 | private val parallelism = 1 20 | 21 | def copyBytes(byteArrays: Source[ByteString, Any]) = byteArrays.zipWithIndex 22 | .map { case (data, index) => data -> f"out-image-$index%05d.jpg" } 23 | .mapAsync(parallelism) { case (data, file) => copyFile(file, data.toArray) } 24 | .runWith(Sink.ignore) 25 | 26 | def copyImages(images: Source[Image, Any]) = images.mapAsync(parallelism)(copyImage).runWith(Sink.ignore) 27 | 28 | def copyImage(image: Image) = copyFile(image.name, image.bytes).map(_ => ()) 29 | 30 | private def copyFile(name: String, data: Array[Byte]) = Future { 31 | val file = new File(s"${settings.framesOutputDir}/$name") 32 | println(s"writing to $file") 33 | Files.write(file.toPath, data) 34 | }(settings.fileIoDispatcher) 35 | } 36 | -------------------------------------------------------------------------------- /frontend/conf/base.conf: -------------------------------------------------------------------------------- 1 | include "akka-base.conf" 2 | 3 | application.secret="U3spB5^7L;EK_:<=OKxwpOJGM:0JsAyTu2pWhKsqbiGMO@tT_8voDNs:x7q=x?e<" 4 | 5 | play.akka.actor-system = "ClusterSystem" 6 | 7 | akka.persistence.journal.plugin = "akka.persistence.journal.leveldb" 8 | akka.persistence.snapshot-store.plugin = "akka.persistence.snapshot-store.local" 9 | 10 | binding = ${seed2} 11 | 12 | #application.langs="en" 13 | #application.global=Global 14 | 15 | # Router 16 | # ~~~~~ 17 | # Define the Router object to use for this application. 18 | # This router will be looked up first when the application is starting up, 19 | # so make sure this is the entry point. 20 | # Furthermore, it's assumed your route file is named properly. 21 | # So for an application router like `my.application.Router`, 22 | # you may need to define a router file `conf/my.application.routes`. 23 | # Default to Routes in the root package (and conf/routes) 24 | # application.router=my.application.Routes 25 | 26 | # Database configuration 27 | # ~~~~~ 28 | # You can declare as many datasources as you want. 29 | # By convention, the default datasource is named `default` 30 | # 31 | # db.default.driver=org.h2.Driver 32 | # db.default.url="jdbc:h2:mem:play" 33 | # db.default.user=sa 34 | # db.default.password="" 35 | 36 | # Evolutions 37 | # ~~~~~ 38 | # You can disable evolutions if needed 39 | # evolutionplugin=disabled 40 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/framework/WebsocketRx.scala: -------------------------------------------------------------------------------- 1 | package tmt.framework 2 | 3 | import org.scalajs.dom.raw.WebSocket 4 | import rx._ 5 | import rx.core.Rx 6 | import tmt.app.ViewData 7 | import Framework._ 8 | 9 | import scalatags.JsDom.all._ 10 | 11 | abstract class WebsocketRx(name: String, viewData: ViewData) extends FormRx(name, viewData) { 12 | 13 | def cleanup(): Unit 14 | 15 | def disconnect() = { 16 | webSocket().foreach(_.close()) 17 | selectedServer() = name 18 | cleanup() 19 | } 20 | 21 | def getUrl = viewData.nodeSet().getWsUrl(server()) 22 | 23 | val webSocket = Var(Option.empty[WebSocket]) 24 | 25 | Obs(currentUrl, skipInitial = true) { 26 | webSocket().foreach(_.close()) 27 | val newSocket = new WebSocket(currentUrl()) 28 | newSocket.binaryType = "arraybuffer" 29 | webSocket() = Some(newSocket) 30 | } 31 | 32 | val disabledStyle = Rx(if(selectedServer() == name) "disabled" else "") 33 | 34 | val disconnectButton = a(cls := Rx(s"btn-floating waves-effect waves-light red ${disabledStyle()}"))( 35 | display := "inline-block", 36 | float := "right", 37 | onclick := { () => disconnect() }, 38 | i(cls := "material-icons")("remove") 39 | ) 40 | 41 | val title = Rx { 42 | div( 43 | h5(display := "inline-block")(selectedServer().capitalize), 44 | disconnectButton 45 | ) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /frontend/app/services/ClusterMetadataService.scala: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import javax.inject.{Inject, Singleton} 4 | 5 | import akka.actor.ActorSystem 6 | import akka.cluster.ddata.DistributedData 7 | import akka.cluster.ddata.Replicator.{Get, GetSuccess, NotFound, ReadLocal} 8 | import akka.cluster.{Cluster, MemberStatus} 9 | import akka.pattern.ask 10 | import akka.util.Timeout 11 | import tmt.common.Keys 12 | import tmt.shared.models.NodeSet 13 | 14 | import scala.concurrent.ExecutionContext 15 | import scala.concurrent.duration.DurationInt 16 | 17 | @Singleton 18 | class ClusterMetadataService@Inject()(implicit system: ActorSystem, ec: ExecutionContext) { 19 | val cluster = Cluster(system) 20 | val replicator = DistributedData(system).replicator 21 | implicit val timeout = Timeout(1.second) 22 | 23 | def onlineMembers = { 24 | val upMembers = cluster.state.members.filter(_.status == MemberStatus.Up) 25 | val unreachableMembers = cluster.state.unreachable 26 | upMembers diff unreachableMembers 27 | } 28 | 29 | def onlineRoles = onlineMembers.flatMap(_.roles) 30 | 31 | def nodeSet = (replicator ? Get(Keys.Nodes, ReadLocal)).map { 32 | case g@GetSuccess(Keys.Nodes, _) => 33 | val roleMappings = g.get(Keys.Nodes).entries.values.toList 34 | NodeSet(roleMappings.map(_.node)) 35 | case NotFound(Keys.Nodes, _) => NodeSet.empty 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/io/WavefrontReadService.scala: -------------------------------------------------------------------------------- 1 | package tmt.io 2 | 3 | import java.io.File 4 | import java.nio.file.Files 5 | 6 | import akka.http.scaladsl.model.ws.BinaryMessage 7 | import akka.stream.io.SynchronousFileSource 8 | import akka.stream.scaladsl.Source 9 | import akka.util.ByteString 10 | import tmt.actors.TickerService 11 | import tmt.app.{ActorConfigs, AppSettings} 12 | import tmt.library.SourceExtensions.RichSource 13 | import tmt.shared.models.Image 14 | 15 | import scala.concurrent.Future 16 | 17 | class WavefrontReadService(actorConfigs: ActorConfigs, settings: AppSettings, producer: Producer, tickerService: TickerService) { 18 | 19 | val ticks = tickerService.source 20 | 21 | private val parallelism = 1 22 | 23 | private def files = Source(() => producer.files(settings.framesInputDir)) 24 | 25 | def sendBytes = files.mapAsync(parallelism)(readFile).map(ByteString.apply) 26 | def sendImages = files.mapAsync(parallelism)(readImage) 27 | .throttleBy(ticks) 28 | .map(img => img.copy(createdAt = System.currentTimeMillis())) 29 | def sendMessages = files.map(SynchronousFileSource(_)).map(BinaryMessage.apply) 30 | 31 | private def readImage(file: File) = readFile(file).map(data => Image(file.getName, data, 0))(actorConfigs.ec) 32 | private def readFile(file: File) = Future { 33 | Files.readAllBytes(file.toPath) 34 | }(settings.fileIoDispatcher) 35 | } 36 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/images/ImageRendering.scala: -------------------------------------------------------------------------------- 1 | package tmt.images 2 | 3 | import monifu.concurrent.Scheduler 4 | import org.scalajs.dom._ 5 | import org.scalajs.dom.html.Canvas 6 | import org.scalajs.dom.raw.Blob 7 | import rx._ 8 | import tmt.app.ViewData 9 | import tmt.common.{Constants, Stream} 10 | import tmt.framework.WebsocketRx 11 | 12 | import scala.concurrent.duration._ 13 | import scala.scalajs.js 14 | import scala.scalajs.js.typedarray.ArrayBuffer 15 | 16 | class ImageRendering(cvs: Canvas, viewData: ViewData)(implicit scheduler: Scheduler) extends WebsocketRx("Wavefront", viewData) { 17 | 18 | 19 | def cleanup() = { 20 | ctx.clearRect(0, 0, Constants.CanvasWidth, Constants.CanvasHeight) 21 | } 22 | 23 | Rx(webSocket().foreach(drain)) 24 | 25 | val ctx = cvs.getContext("2d").asInstanceOf[CanvasRenderingContext2D] 26 | 27 | def drain(socket: WebSocket) = Stream.socket(socket) 28 | .map(makeUrl) 29 | .map(new Rendering(_)) 30 | .flatMap(_.loaded) 31 | .map(_.render(ctx)) 32 | .buffer(1.second).map(_.size).foreach(println) 33 | 34 | def makeUrl(messageEvent: MessageEvent) = { 35 | val arrayBuffer = messageEvent.data.asInstanceOf[ArrayBuffer] 36 | val properties = js.Dynamic.literal("type" -> "image/jpeg").asInstanceOf[BlobPropertyBag] 37 | val blob = new Blob(js.Array(arrayBuffer), properties) 38 | Constants.URL.createObjectURL(blob) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/transformations/ImageProcessor.scala: -------------------------------------------------------------------------------- 1 | package tmt.transformations 2 | 3 | import java.awt.image.BufferedImage 4 | import java.io.{ByteArrayInputStream, ByteArrayOutputStream} 5 | import java.util.concurrent.Executors 6 | import javax.imageio.ImageIO 7 | import javax.inject.Singleton 8 | 9 | import akka.http.scaladsl.model.DateTime 10 | import org.imgscalr.Scalr 11 | import org.imgscalr.Scalr.Rotation 12 | import tmt.app.AppSettings 13 | import tmt.shared.models.Image 14 | 15 | import scala.concurrent.ExecutionContext 16 | import async.Async._ 17 | 18 | @Singleton 19 | class ImageProcessor(appSettings: AppSettings) { 20 | 21 | private val imageProcessingEc = ExecutionContext.fromExecutorService( 22 | Executors.newFixedThreadPool(appSettings.imageProcessingThreadPoolSize) 23 | ) 24 | 25 | def rotate(image: Image) = async { 26 | val bufferedImage = ImageIO.read(new ByteArrayInputStream(image.bytes)) 27 | val rotatedBufferedImage = Scalr.rotate(bufferedImage, Rotation.FLIP_VERT) 28 | makeNewImage(image.name, rotatedBufferedImage) 29 | }(imageProcessingEc) 30 | 31 | private def makeNewImage(name: String, rotatedBufferedImage: BufferedImage): Image = { 32 | val baos = new ByteArrayOutputStream() 33 | ImageIO.write(rotatedBufferedImage, "jpeg", baos) 34 | baos.flush() 35 | val rotatedImage = Image(name, baos.toByteArray, DateTime.now.clicks) 36 | baos.close() 37 | rotatedImage 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /frontend/app/templates/Page.scala: -------------------------------------------------------------------------------- 1 | package templates 2 | 3 | import play.twirl.api.Html 4 | 5 | import scalatags.Text.all._ 6 | import scalatags.Text.{tags2 => t} 7 | import controllers.routes 8 | 9 | class Page(titleValue: String) { 10 | val docType = "" 11 | 12 | def page = html( 13 | head( 14 | meta(charset := "utf-8"), 15 | meta("http-equiv".attr :="X-UA-Compatible", content := "IE=edge"), 16 | link(rel := "stylesheet", href := "http://fonts.googleapis.com/icon?family=Material+Icons"), 17 | meta(name :="viewport", content := "width=device-width, initial-scale=1"), 18 | 19 | t.title(titleValue), 20 | 21 | link(rel := "stylesheet", `type` := "text/css", media := "screen,projection", href := routes.Assets.at("lib/materialize/dist/css/materialize.min.css").url), 22 | link(rel := "stylesheet", `type` := "text/css", href := routes.Assets.at("stylesheets/main.css").url), 23 | link(rel := "shortcut icon", `type` := "image/png", href := routes.Assets.at("images/favicon.png").url) 24 | ), 25 | body( 26 | raw(playscalajs.html.jsdeps("client").toString()), 27 | script(`type` := "text/javascript", src := routes.Assets.at("lib/materialize/dist/js/materialize.min.js").url), 28 | raw(playscalajs.html.selectScript("client").toString()), 29 | raw(playscalajs.html.launcher("client").toString()) 30 | ) 31 | ) 32 | 33 | def render = Html(docType + "\n" + page.render) 34 | } 35 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/framework/Helpers.scala: -------------------------------------------------------------------------------- 1 | package tmt.framework 2 | 3 | import org.scalajs.dom.html._ 4 | import rx._ 5 | import rx.core.Rx 6 | import tmt.framework.Framework._ 7 | 8 | import scala.scalajs.js 9 | import scalatags.JsDom.all._ 10 | object Helpers { 11 | 12 | def makeForm(options: Rx[Seq[String]], formRx: FormRx) = { 13 | val noData = Rx(formRx.server().isEmpty || options().isEmpty) 14 | val disabledStyle = Rx(if(noData()) "disabled" else "") 15 | 16 | div(cls := "row")( 17 | div(cls := "col l8")(makeSelection(options, formRx.server)), 18 | 19 | button(cls := Rx(s"waves-effect waves-light btn col l4 ${disabledStyle()}"))( 20 | onclick := { () => if(!noData()) formRx.action() } 21 | )("Set") 22 | ) 23 | } 24 | 25 | def makeSelection(options: Rx[Seq[String]], selection: Var[String]) = Rx { 26 | select(cls := "browser-default")(onchange := setValue(selection))( 27 | optionHint("select"), 28 | makeOptions(options(), selection()) 29 | ) 30 | } 31 | 32 | def setValue(selection: Var[String]): js.ThisFunction = { e: Select => 33 | selection() = e.value 34 | } 35 | 36 | lazy val optionHint = option(selected := true, disabled, value := "") 37 | 38 | private def makeOptions(values: Seq[String], selectedValue: String) = values.map { 39 | case v@`selectedValue` => option(value := v, selected := true)(v) 40 | case v => option(value := v)(v) 41 | } 42 | } 43 | 44 | 45 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/actors/Subscription.scala: -------------------------------------------------------------------------------- 1 | package tmt.actors 2 | 3 | import akka.actor.{Actor, ActorRef, Props} 4 | import akka.cluster.pubsub.DistributedPubSub 5 | import akka.cluster.pubsub.DistributedPubSubMediator.{Subscribe, Unsubscribe} 6 | import tmt.app.{ActorConfigs, AppSettings} 7 | import tmt.common.Messages 8 | import tmt.shared.Topics 9 | import tmt.shared.models.{Connection, ConnectionSet} 10 | 11 | class SubscriptionService[T]( 12 | appSettings: AppSettings, actorConfigs: ActorConfigs 13 | ) extends SourceActorLink[T](actorConfigs, Topics.Subscription, Topics.Connections) { 14 | 15 | import actorConfigs._ 16 | def wrap(sourceLinkedRef: ActorRef) = 17 | system.actorOf(Subscription.props(appSettings.binding.name, sourceLinkedRef)) 18 | } 19 | 20 | class Subscription(serverName: String, sourceLinkedRef: ActorRef) extends Actor { 21 | val mediator = DistributedPubSub(context.system).mediator 22 | 23 | def receive = { 24 | case Messages.Subscribe(Connection(`serverName`, topic)) => mediator ! Subscribe(topic, sourceLinkedRef) 25 | case Messages.Unsubscribe(Connection(`serverName`, topic)) => mediator ! Unsubscribe(topic, sourceLinkedRef) 26 | case cs: ConnectionSet => cs.getTopics(serverName).foreach(topic => mediator ! Subscribe(topic, sourceLinkedRef)) 27 | } 28 | } 29 | 30 | object Subscription { 31 | def props(serverName: String, sourceLinkedRef: ActorRef) = Props(new Subscription(serverName, sourceLinkedRef)) 32 | } 33 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/server/RouteInstances.scala: -------------------------------------------------------------------------------- 1 | package tmt.server 2 | 3 | import tmt.app.{ActorConfigs, AppSettings} 4 | import tmt.io.WavefrontReadService 5 | import tmt.library.SourceExtensions.RichSource 6 | import tmt.marshalling.BinaryMarshallers 7 | import tmt.shared.models.Role 8 | import tmt.transformations.{ImageTransformations, MetricsTransformations} 9 | 10 | class RouteInstances( 11 | routeFactory: RouteFactory, 12 | imageTransformations: ImageTransformations, 13 | metricsTransformations: MetricsTransformations, 14 | actorConfigs: ActorConfigs, 15 | wavefrontReadService: WavefrontReadService, 16 | appSettings: AppSettings 17 | ) extends BinaryMarshallers { 18 | 19 | import actorConfigs._ 20 | 21 | val serverName = appSettings.binding.name 22 | 23 | def find(role: Role) = role match { 24 | case Role.ScienceImageSource => routeFactory.scienceImages 25 | 26 | case Role.Wavefront => routeFactory.wavefront(serverName, wavefrontReadService.sendImages.hotMulticast) 27 | case Role.Rotator => routeFactory.wavefront(serverName, imageTransformations.rotatedImages) 28 | 29 | case Role.Copier => routeFactory.generic(serverName, imageTransformations.copiedImages) 30 | case Role.Filter => routeFactory.generic(serverName, imageTransformations.filteredImages) 31 | 32 | case Role.Metadata => routeFactory.generic(serverName, imageTransformations.imageMetadata) 33 | case Role.Metric => routeFactory.generic(serverName, metricsTransformations.perSecMetrics) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/actors/Ticker.scala: -------------------------------------------------------------------------------- 1 | package tmt.actors 2 | 3 | import akka.actor.{Actor, ActorRef, Props} 4 | import tmt.app.{ActorConfigs, AppSettings} 5 | import tmt.common.Messages 6 | import tmt.shared.Topics 7 | 8 | import scala.concurrent.duration.FiniteDuration 9 | 10 | class TickerService( 11 | appSettings: AppSettings, actorConfigs: ActorConfigs 12 | ) extends SourceActorLink[Ticker.Tick](actorConfigs, Topics.Throttle) { 13 | 14 | import actorConfigs._ 15 | def wrap(sourceLinkedRef: ActorRef) = 16 | system.actorOf(Ticker.props(appSettings.binding.name, appSettings.imageReadThrottle, sourceLinkedRef)) 17 | } 18 | 19 | class Ticker(serverName: String, initialDelay: FiniteDuration, sourceLinkedRef: ActorRef) extends Actor { 20 | import context.dispatcher 21 | 22 | val scheduler = context.system.scheduler 23 | 24 | def startNewSchedule(delay: FiniteDuration) = scheduler.schedule(delay, delay, sourceLinkedRef, Ticker.Tick) 25 | 26 | var currentSchedule = startNewSchedule(initialDelay) 27 | 28 | def receive = { 29 | case Messages.UpdateDelay(`serverName`, newDelay) => 30 | println("**** new delay is: ", newDelay) 31 | currentSchedule.cancel() 32 | currentSchedule = startNewSchedule(newDelay) 33 | } 34 | } 35 | 36 | object Ticker { 37 | trait Tick 38 | case object Tick extends Tick 39 | 40 | def props(serverName: String, initialDelay: FiniteDuration, sourceLinkedRef: ActorRef) = Props( 41 | new Ticker(serverName, initialDelay, sourceLinkedRef) 42 | ) 43 | } 44 | -------------------------------------------------------------------------------- /frontend/app/services/ClusterClientService.scala: -------------------------------------------------------------------------------- 1 | package services 2 | 3 | import javax.inject.{Inject, Singleton} 4 | 5 | import actors.ConnectionStore 6 | import akka.actor.ActorSystem 7 | import akka.cluster.pubsub.DistributedPubSub 8 | import akka.cluster.pubsub.DistributedPubSubMediator.Publish 9 | import akka.pattern.ask 10 | import akka.util.Timeout 11 | import tmt.common.Messages 12 | import tmt.shared.Topics 13 | import tmt.shared.models.{Connection, ConnectionSet} 14 | import async.Async._ 15 | import scala.concurrent.ExecutionContext 16 | 17 | import scala.concurrent.duration.{DurationInt, FiniteDuration} 18 | 19 | @Singleton 20 | class ClusterClientService @Inject()(roleIndexService: NodeSetService)(implicit system: ActorSystem, ec: ExecutionContext) { 21 | 22 | implicit val timeout = Timeout(2.seconds) 23 | 24 | private val mediator = DistributedPubSub(system).mediator 25 | private val connectionStore = system.actorOf(ConnectionStore.props()) 26 | 27 | def throttle(serverName: String, delay: FiniteDuration) = { 28 | mediator ! Publish(Topics.Throttle, Messages.UpdateDelay(serverName, delay)) 29 | } 30 | 31 | def subscribe(connection: Connection) = async { 32 | if(await(roleIndexService.validate(connection))) 33 | mediator ! Publish(Topics.Subscription, Messages.Subscribe(connection)) 34 | else 35 | throw new RuntimeException( 36 | s"${connection.server} can not subscribe ${connection.topic} due to validation failure" 37 | ) 38 | } 39 | 40 | def unsubscribe(connection: Connection) = { 41 | mediator ! Publish(Topics.Subscription, Messages.Unsubscribe(connection)) 42 | } 43 | 44 | def allConnections = (connectionStore ? ConnectionStore.GetConnections).mapTo[ConnectionSet] 45 | } 46 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/app/AppSettings.scala: -------------------------------------------------------------------------------- 1 | package tmt.app 2 | 3 | import java.net.InetSocketAddress 4 | 5 | import akka.http.scaladsl.Http 6 | import akka.http.scaladsl.model.Uri 7 | import tmt.shared.models.{Node, Role} 8 | import scala.concurrent.duration.DurationLong 9 | import scala.util.Try 10 | 11 | class AppSettings(actorConfigs: ActorConfigs) { 12 | 13 | import actorConfigs._ 14 | 15 | val fileIoDispatcher = system.dispatchers.lookup("akka.stream.default-file-io-dispatcher") 16 | 17 | val config = system.settings.config 18 | val superPool = Http().superPool[Uri]() 19 | 20 | val maxTransferFiles = Try(config.getInt("max-transfer-files")).getOrElse(Int.MaxValue) 21 | 22 | val framesInputDir = config.getString("data-location.frames.input") 23 | val framesOutputDir = config.getString("data-location.frames.output") 24 | 25 | val scienceImagesInputDir = config.getString("data-location.science-images.input") 26 | val scienceImagesOutputDir = config.getString("data-location.science-images.output") 27 | 28 | val imageReadThrottle = config.getDuration("image-read-throttle").toMillis.millis 29 | 30 | val imageProcessingThreadPoolSize = config.getInt("image-processing.thread-pool-size") 31 | val imageProcessingParallelism = config.getInt("image-processing.parallelism") 32 | 33 | val env = config.getString("env") 34 | 35 | object binding { 36 | val name = config.getString("binding.name") 37 | val role = config.getString("binding.role") 38 | val hostname = config.getString("binding.hostname") 39 | val externalIp = config.getString("binding.externalIp") 40 | val httpAddress = new InetSocketAddress(hostname, 0) 41 | def roleMapping(httpPort: Int) = Node(Role.withName(role), name, externalIp, httpPort) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /frontend/app/controllers/StreamController.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import javax.inject.{Inject, Singleton} 4 | 5 | import common.AppSettings 6 | import play.api.mvc.{Action, Controller} 7 | import prickle.Pickle 8 | import services.{ClusterClientService, ConnectionSetService, NodeSetService} 9 | import templates.Page 10 | import tmt.shared.models.Connection 11 | 12 | import scala.async.Async._ 13 | import scala.concurrent.ExecutionContext 14 | import scala.concurrent.duration.DurationLong 15 | 16 | @Singleton 17 | class StreamController @Inject()( 18 | appSettings: AppSettings, 19 | clusterClientService: ClusterClientService, 20 | nodeSetService: NodeSetService, 21 | connectionSetService: ConnectionSetService 22 | )(implicit ec: ExecutionContext) extends Controller { 23 | 24 | def streams() = Action { 25 | Ok(new Page("showcase").render) 26 | } 27 | 28 | def nodes() = Action.async { 29 | async { 30 | Ok(Pickle.intoString(await(nodeSetService.onlineNodeSet))) 31 | } 32 | } 33 | 34 | def connections() = Action.async { 35 | async { 36 | val connectionDataSet = await(connectionSetService.connectionSet) 37 | Ok(Pickle.intoString(connectionDataSet)) 38 | } 39 | } 40 | 41 | def throttle(serverName: String, delay: Long) = Action { 42 | clusterClientService.throttle(serverName, delay.millis) 43 | Accepted("ok") 44 | } 45 | 46 | def subscribe(serverName: String, topic: String) = Action.async { 47 | async { 48 | await(clusterClientService.subscribe(Connection(serverName, topic))) 49 | Accepted("ok") 50 | } 51 | } 52 | 53 | def unsubscribe(serverName: String, topic: String) = Action { 54 | clusterClientService.unsubscribe(Connection(serverName, topic)) 55 | Accepted("ok") 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/framework/Framework.scala: -------------------------------------------------------------------------------- 1 | package tmt.framework 2 | 3 | import org.scalajs.dom.{Element, html} 4 | import rx._ 5 | import rx.core.Obs 6 | 7 | import scala.util.{Failure, Success} 8 | import scalatags.JsDom.all._ 9 | 10 | 11 | /** 12 | * A minimal binding between Scala.Rx and Scalatags and Scala-Js-Dom 13 | */ 14 | object Framework { 15 | 16 | /** 17 | * Wraps reactive strings in spans, so they can be referenced/replaced 18 | * when the Rx changes. 19 | */ 20 | implicit def RxStr[T](r: Rx[T])(implicit f: T => Frag): Frag = { 21 | rxMod(Rx(span(r()))) 22 | } 23 | 24 | /** 25 | * Sticks some Rx into a Scalatags fragment, which means hooking up an Obs 26 | * to propagate changes into the DOM via the element's ID. Monkey-patches 27 | * the Obs onto the element itself so we have a reference to kill it when 28 | * the element leaves the DOM (e.g. it gets deleted). 29 | */ 30 | implicit def rxMod[T <: html.Element](r: Rx[HtmlTag]): Frag = { 31 | def rSafe = r.toTry match { 32 | case Success(v) => v.render 33 | case Failure(e) => span(e.toString, backgroundColor := "red").render 34 | } 35 | var last = rSafe 36 | Obs(r, skipInitial = true){ 37 | val newLast = rSafe 38 | last.parentElement.replaceChild(newLast, last) 39 | last = newLast 40 | } 41 | bindNode(last) 42 | } 43 | implicit def RxAttrValue[T: AttrValue] = new AttrValue[Rx[T]]{ 44 | def apply(t: Element, a: Attr, r: Rx[T]): Unit = { 45 | Obs(r){ implicitly[AttrValue[T]].apply(t, a, r())} 46 | } 47 | } 48 | implicit def RxStyleValue[T: StyleValue] = new StyleValue[Rx[T]]{ 49 | def apply(t: Element, s: Style, r: Rx[T]): Unit = { 50 | Obs(r){ implicitly[StyleValue[T]].apply(t, s, r())} 51 | } 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/library/SourceExtensions.scala: -------------------------------------------------------------------------------- 1 | package tmt.library 2 | 3 | import akka.stream.Materializer 4 | import akka.stream.scaladsl._ 5 | import org.reactivestreams.Publisher 6 | 7 | import scala.concurrent.duration.FiniteDuration 8 | 9 | object SourceExtensions { 10 | 11 | implicit class RichSource[Out, Mat](val source: Source[Out, Mat]) extends AnyVal { 12 | 13 | def zip[Out2, Mat2](other: Source[Out2, Mat2]) = Source(source) { implicit b => src => 14 | import FlowGraph.Implicits._ 15 | 16 | val zipper = b.add(Zip[Out, Out2]()) 17 | 18 | src ~> zipper.in0 19 | other ~> zipper.in1 20 | 21 | zipper.out 22 | } 23 | 24 | def zipWithIndex = zip(Source(() => Iterator.from(0))) 25 | 26 | def throttle(duration: FiniteDuration) = throttleBy(Sources.ticks(duration)) 27 | def throttleBy[Out2, Mat2](other: Source[Out2, Mat2]) = source.zip(other).map(_._1) 28 | 29 | def multicast(implicit mat: Materializer) = Source(source.runWith(Sink.fanoutPublisher(2, 2))) 30 | 31 | def hotUnicast(implicit mat: Materializer) = hot(Sink.publisher) 32 | def hotMulticast(implicit mat: Materializer) = hot(Sink.fanoutPublisher(2, 2)) 33 | 34 | def hot(sink: Sink[Out, Publisher[Out]])(implicit mat: Materializer) = { 35 | val (actorRef, hotSource) = Connector.coupling(sink) 36 | source.runForeach(x => actorRef ! x) 37 | hotSource 38 | } 39 | 40 | } 41 | 42 | def merge[Out, Mat](sources: Seq[Source[Out, Mat]]) = 43 | if(sources.isEmpty) 44 | Source.empty[Out] 45 | else { 46 | Source() { implicit b => 47 | import FlowGraph.Implicits._ 48 | 49 | val merge = b.add(Merge[Out](sources.length)) 50 | 51 | sources.zipWithIndex.foreach { case (source, index) => source ~> merge.in(index) } 52 | 53 | merge.out 54 | } 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /shared/src/main/scala/tmt/shared/models/NodeSet.scala: -------------------------------------------------------------------------------- 1 | package tmt.shared.models 2 | 3 | import tmt.shared.Topics 4 | 5 | case class NodeSet(nodes: Seq[Node]) { 6 | def compatibleConsumers(producer: String) = nodes.collectFirst { 7 | case Node(producerRole, `producer`, _, _) => nodes.collect { 8 | case Node(role, name, _, _) if producerRole.output == role.input && name != producer => name 9 | } 10 | }.toSeq.flatten 11 | 12 | def getRole(name: String) = nodes.collectFirst { 13 | case Node(role, `name`, _, _) => role 14 | }.getOrElse(Role.Empty) 15 | 16 | def getNames(role: Role) = nodes.collect { 17 | case Node(`role`, name, _, _) => name 18 | } 19 | 20 | def getNames(outputType: ItemType) = nodes.collect { 21 | case Node(role, name, _, _) if role.output == outputType => name 22 | } 23 | 24 | def getWsUrl(name: String) = nodes.collectFirst { 25 | case n@Node(_, `name`, externalIp, httpPort) => s"ws://$externalIp:$httpPort/$name" 26 | } 27 | 28 | def getScienceImageUrl(name: String) = nodes.collectFirst { 29 | case n@Node(_, `name`, externalIp, httpPort) => s"http://$externalIp:$httpPort/${Topics.ScienceImages}" 30 | } 31 | 32 | def producers = nodes.collect { 33 | case Node(role, server, _, _) if role.output.nonEmpty => server 34 | } 35 | 36 | def pruneBy(onlineRoles: Set[String]) = NodeSet(nodes.filter(_.isOnline(onlineRoles))) 37 | } 38 | 39 | object NodeSet { 40 | def empty = NodeSet(Seq.empty) 41 | } 42 | 43 | case class Node(role: Role, name: String, externalIp: String, httpPort: Int) { 44 | def isOnline(onlineRoles: Set[String]) = onlineRoles(role.entryName) && onlineRoles(name) 45 | } 46 | 47 | case class NodeS(role: String, name: String, externalIp: String, httpPort: Int) { 48 | def node = Node(Role.withName(role), name, externalIp, httpPort) 49 | } 50 | 51 | object NodeS { 52 | def fromNode(node: Node) = NodeS(node.role.entryName, node.name, node.externalIp, node.httpPort) 53 | } 54 | -------------------------------------------------------------------------------- /frontend/app/actors/ConnectionStore.scala: -------------------------------------------------------------------------------- 1 | package actors 2 | 3 | import akka.actor.Props 4 | import akka.cluster.Cluster 5 | import akka.cluster.ClusterEvent._ 6 | import akka.cluster.pubsub.DistributedPubSub 7 | import akka.cluster.pubsub.DistributedPubSubMediator.{Publish, Subscribe} 8 | import akka.persistence.{SnapshotOffer, PersistentActor} 9 | import tmt.common.Messages 10 | import tmt.shared.Topics 11 | import tmt.shared.models.ConnectionSet 12 | 13 | import scala.concurrent.duration._ 14 | 15 | class ConnectionStore extends PersistentActor { 16 | 17 | def persistenceId = "connection-store" 18 | 19 | var connectionSet = ConnectionSet.empty 20 | 21 | val mediator = DistributedPubSub(context.system).mediator 22 | val cluster = Cluster(context.system) 23 | val scheduler = context.system.scheduler 24 | 25 | import context.dispatcher 26 | 27 | override def preStart() = { 28 | cluster.subscribe(self, classOf[MemberUp]) 29 | mediator ! Subscribe(Topics.Subscription, self) 30 | } 31 | 32 | def receiveRecover = { 33 | case SnapshotOffer(metadata, cs: ConnectionSet) => connectionSet = cs 34 | } 35 | 36 | def receiveCommand = { 37 | case event: Messages.Subscribe => addConnection(event) 38 | case event: Messages.Unsubscribe => removeConnection(event) 39 | case x: MemberUp => 40 | println("******: ", x) 41 | scheduler.scheduleOnce(2.second, mediator, Publish(Topics.Connections, connectionSet)) 42 | case ConnectionStore.GetConnections => sender() ! connectionSet 43 | } 44 | 45 | def addConnection(event: Messages.Subscribe) = { 46 | connectionSet = connectionSet.add(event.connection) 47 | saveSnapshot(connectionSet) 48 | } 49 | 50 | def removeConnection(event: Messages.Unsubscribe) = { 51 | connectionSet = connectionSet.remove(event.connection) 52 | saveSnapshot(connectionSet) 53 | } 54 | } 55 | 56 | object ConnectionStore { 57 | case object GetConnections 58 | 59 | def props() = Props(new ConnectionStore) 60 | } 61 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/server/RouteFactory.scala: -------------------------------------------------------------------------------- 1 | package tmt.server 2 | 3 | import akka.http.scaladsl.model.ws.BinaryMessage 4 | import akka.http.scaladsl.server.Directives._ 5 | import akka.http.scaladsl.server._ 6 | import akka.stream.scaladsl.{Sink, Source} 7 | import akka.util.ByteString 8 | import prickle.Pickle 9 | import tmt.app.{ActorConfigs, AppSettings, CustomDirectives, Types} 10 | import tmt.io.ScienceImageReadService 11 | import tmt.library.Cors 12 | import tmt.library.SourceExtensions.RichSource 13 | import tmt.marshalling.{BinaryMarshallers, BFormat} 14 | import tmt.shared.Topics 15 | import tmt.shared.models.Image 16 | 17 | class RouteFactory( 18 | scienceImageReadService: ScienceImageReadService, 19 | actorConfigs: ActorConfigs, 20 | publisher: Publisher, 21 | appSettings: AppSettings) extends CustomDirectives with BinaryMarshallers { 22 | 23 | import actorConfigs._ 24 | 25 | def scienceImages: Route = get { 26 | path(Topics.ScienceImages) { 27 | Cors.cors(complete(Pickle.intoString(scienceImageReadService.scienceImages))) 28 | } ~ 29 | pathPrefix(Topics.ScienceImages) { 30 | path(Rest) { name => 31 | complete(scienceImageReadService.send(name)) 32 | } 33 | } 34 | } 35 | 36 | def wavefront(topic: String, dataSource: Source[Image, Any]): Route = { 37 | val frames = dataSource.hotMulticast 38 | val bytes = frames.map(x => ByteString(x.bytes)) 39 | makeRoute(topic, frames, bytes) 40 | } 41 | 42 | def generic[T: Types.Stream: BFormat](topic: String, dataSource: Source[T, Any]): Route = { 43 | val items = dataSource.hotMulticast 44 | makeRoute(topic, items, items) 45 | } 46 | 47 | private def makeRoute[T: Types.Stream, S: BFormat]( 48 | topic: String, 49 | dataSource: Source[T, Any], 50 | socketSource: Source[S, Any]): Route = { 51 | 52 | publisher.publish(topic, dataSource) 53 | def messages = socketSource.map(x => BinaryMessage(BFormat[S].write(x))).hotMulticast 54 | path(topic) { 55 | get { 56 | handleWebsocketMessages(Sink.ignore, messages) ~ complete(dataSource) 57 | } 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/clients/clients.scala: -------------------------------------------------------------------------------- 1 | package tmt.clients 2 | 3 | import java.net.InetSocketAddress 4 | 5 | import akka.http.scaladsl.model._ 6 | import akka.stream.scaladsl.{FlattenStrategy, Source, Flow} 7 | import tmt.app.{ActorConfigs, AppSettings} 8 | import tmt.library.InetSocketAddressExtensions.RichInetSocketAddress 9 | import tmt.library.ResponseExtensions.RichResponse 10 | import tmt.marshalling.BFormat 11 | 12 | import scala.concurrent.duration.DurationInt 13 | import scala.util.Success 14 | 15 | class ProducingClient(address: InetSocketAddress, actorConfigs: ActorConfigs, settings: AppSettings) { 16 | import actorConfigs._ 17 | 18 | val flow: Flow[String, (MessageEntity, Uri), Unit] = Flow[String] 19 | .map(address.absoluteUri) 20 | .map(uri => HttpRequest(uri = uri) -> uri) 21 | .via(settings.superPool) 22 | .collect { case (Success(response), uri) if response.status == StatusCodes.OK => 23 | response.multicastEntity -> uri 24 | } 25 | 26 | def request[T: BFormat](routeName: String) = Source(List(routeName)) 27 | .via(flow) 28 | .map { case (entity, _) => entity.dataBytes } 29 | .flatten(FlattenStrategy.concat) 30 | .map(BFormat[T].read) 31 | } 32 | 33 | class ProducingClientFactory(actorConfigs: ActorConfigs, settings: AppSettings) { 34 | def make(address: InetSocketAddress) = new ProducingClient(address, actorConfigs, settings) 35 | } 36 | 37 | class ConsumingClient(address: InetSocketAddress, actorConfigs: ActorConfigs, settings: AppSettings) { 38 | import actorConfigs._ 39 | 40 | val flow: Flow[(MessageEntity, Uri), HttpResponse, Any] = Flow[(MessageEntity, Uri)] 41 | .collect { case (entity, uri) => 42 | HttpRequest(uri = address.update(uri), method = HttpMethods.POST, entity = entity) -> address.update(uri) 43 | } 44 | .via(settings.superPool) 45 | .mapAsync(1) { case (Success(response), name) => 46 | response.toStrict(1.second) 47 | } 48 | 49 | } 50 | 51 | class ConsumingClientFactory(actorConfigs: ActorConfigs, settings: AppSettings) { 52 | def make(address: InetSocketAddress) = new ConsumingClient(address, actorConfigs, settings) 53 | } 54 | -------------------------------------------------------------------------------- /shared/src/main/scala/tmt/shared/models/Role.scala: -------------------------------------------------------------------------------- 1 | package tmt.shared.models 2 | 3 | import prickle.{PicklerPair, CompositePickler} 4 | 5 | sealed abstract class Role(val entryName: String, val input: ItemType, val output: ItemType) 6 | 7 | object Role { 8 | case object ScienceImageSource extends Role("science-image-source", ItemType.Empty, ItemType.Empty) 9 | 10 | case object Wavefront extends Role("wavefront", ItemType.Empty, ItemType.Image) 11 | case object Copier extends Role("copier", ItemType.Image, ItemType.Empty) 12 | case object Filter extends Role("filter", ItemType.Image, ItemType.Image) 13 | 14 | case object Metadata extends Role("metadata", ItemType.Image, ItemType.Metadata) 15 | case object Metric extends Role("metric", ItemType.Metadata, ItemType.Metric) 16 | case object Rotator extends Role("rotator", ItemType.Image, ItemType.Image) 17 | 18 | case object Empty extends Role("empty", ItemType.Empty, ItemType.Empty) 19 | 20 | val values: Seq[Role] = Seq(ScienceImageSource, Wavefront, Copier, Filter, Metadata, Metric, Rotator, Empty) 21 | 22 | def withName(name: String) = values.find(_.entryName == name).getOrElse(Empty) 23 | 24 | implicit val rolePickler: PicklerPair[Role] = CompositePickler[Role] 25 | .concreteType[ScienceImageSource.type] 26 | .concreteType[Wavefront.type] 27 | .concreteType[Copier.type] 28 | .concreteType[Filter.type] 29 | .concreteType[Metadata.type] 30 | .concreteType[Metric.type] 31 | .concreteType[Rotator.type] 32 | .concreteType[Empty.type] 33 | 34 | } 35 | 36 | sealed abstract class ItemType(val entryName: String) { 37 | def nonEmpty = this != ItemType.Empty 38 | } 39 | 40 | object ItemType { 41 | case object Image extends ItemType("image") 42 | case object Metadata extends ItemType("metadata") 43 | case object Metric extends ItemType("metric") 44 | case object Empty extends ItemType("empty") 45 | 46 | val values: Seq[ItemType] = Seq(Image, Metadata, Metric, Empty) 47 | 48 | def withName(name: String) = values.find(_.entryName == name).getOrElse(Empty) 49 | 50 | implicit val itemTypePickler = CompositePickler[ItemType] 51 | .concreteType[Image.type] 52 | .concreteType[Metadata.type] 53 | .concreteType[Metric.type] 54 | .concreteType[Empty.type] 55 | } 56 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/server/MediaRoute.scala: -------------------------------------------------------------------------------- 1 | package tmt.server 2 | 3 | import akka.http.scaladsl.server.Directives._ 4 | import akka.http.scaladsl.server.Route 5 | import akka.stream.scaladsl.{Sink, Source} 6 | import akka.util.ByteString 7 | import tmt.app.CustomDirectives 8 | import tmt.io.{WavefrontReadService, WavefrontWriteService, ScienceImageReadService, ScienceImageWriteService} 9 | import tmt.marshalling.BinaryMarshallers 10 | import tmt.shared.models.Image 11 | 12 | class MediaRoute( 13 | wavefrontReadService: WavefrontReadService, 14 | wavefrontWriteService: WavefrontWriteService, 15 | scienceImageReadService: ScienceImageReadService, 16 | scienceImageWriteService: ScienceImageWriteService 17 | ) extends BinaryMarshallers with CustomDirectives { 18 | 19 | val route = movieRoute ~ imageRoute 20 | 21 | lazy val movieRoute: Route = { 22 | path("science-images" / "list") { 23 | get { 24 | complete(scienceImageReadService.listScienceImages) 25 | } 26 | } ~ 27 | pathPrefix("science-images") { 28 | path(Rest) { name => 29 | get { 30 | complete(scienceImageReadService.send(name)) 31 | } ~ 32 | post { 33 | entity(as[Source[ByteString, Any]]) { byteStrings => 34 | onSuccess(scienceImageWriteService.copy(name, byteStrings)) { 35 | complete("copied") 36 | } 37 | } 38 | } 39 | } 40 | } 41 | } 42 | 43 | lazy val imageRoute: Route = { 44 | path("images" / "bytes") { 45 | get { 46 | handleWebsocketMessages(Sink.ignore, wavefrontReadService.sendMessages) ~ 47 | complete(wavefrontReadService.sendBytes) 48 | } ~ 49 | post { 50 | entity(as[Source[ByteString, Any]]) { byteStrings => 51 | onSuccess(wavefrontWriteService.copyBytes(byteStrings)) { 52 | complete("copied") 53 | } 54 | } 55 | } 56 | } ~ 57 | path("images" / "objects") { 58 | get { 59 | complete(wavefrontReadService.sendImages) 60 | } ~ 61 | post { 62 | entity(as[Source[Image, Any]]) { images => 63 | onSuccess(wavefrontWriteService.copyImages(images)) { 64 | complete("copied") 65 | } 66 | } 67 | } 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/app/Assembly.scala: -------------------------------------------------------------------------------- 1 | package tmt.app 2 | 3 | import akka.actor.ActorSystem 4 | import akka.stream.ActorMaterializer 5 | import tmt.actors.{SubscriptionService, TickerService} 6 | import tmt.clients._ 7 | import tmt.io._ 8 | import tmt.server._ 9 | import tmt.shared.models.{Image, ImageMetadata} 10 | import tmt.transformations.{ImageProcessor, ImageTransformations, MetricsTransformations} 11 | 12 | class Assembly(role: String, serverName: String, env: String, seedName: Option[String]) { 13 | import com.softwaremill.macwire._ 14 | 15 | lazy val configLoader = wire[ConfigLoader] 16 | 17 | lazy val system = ActorSystem("ClusterSystem", configLoader.load(role, serverName, env, seedName)) 18 | lazy val ec = system.dispatcher 19 | lazy val mat = ActorMaterializer()(system) 20 | 21 | lazy val actorConfigs: ActorConfigs = wire[ActorConfigs] 22 | 23 | lazy val appSettings: AppSettings = wire[AppSettings] 24 | 25 | lazy val nodeInfoPublisher = wire[NodeInfoPublisher] 26 | 27 | lazy val producer = wire[Producer] 28 | 29 | lazy val wavefrontReadService: WavefrontReadService = wire[WavefrontReadService] 30 | lazy val wavefrontWriteService = wire[WavefrontWriteService] 31 | lazy val movieReadService = wire[ScienceImageReadService] 32 | lazy val movieWriteService = wire[ScienceImageWriteService] 33 | 34 | lazy val producingClientFactory = wire[ProducingClientFactory] 35 | lazy val consumingClientFactory = wire[ConsumingClientFactory] 36 | 37 | lazy val oneToOneTransferFactory = wire[OneToOneTransferFactory] 38 | lazy val oneToManyTransferFactory = wire[OneToManyTransferFactory] 39 | 40 | lazy val publisher = wire[Publisher] 41 | lazy val imageSubscriber = wire[SubscriptionService[Image]] 42 | lazy val metricSubscriber = wire[SubscriptionService[ImageMetadata]] 43 | 44 | lazy val imageTransformations = wire[ImageTransformations] 45 | lazy val metricsTransformations = wire[MetricsTransformations] 46 | lazy val imageProcessor = wire[ImageProcessor] 47 | 48 | lazy val routeFactory = wire[RouteFactory] 49 | lazy val routeInstances = wire[RouteInstances] 50 | lazy val serverFactory = wire[ServerFactory] 51 | 52 | lazy val tickerService = wire[TickerService] 53 | } 54 | -------------------------------------------------------------------------------- /project/Dependencies.scala: -------------------------------------------------------------------------------- 1 | import org.scalajs.sbtplugin.ScalaJSPlugin.autoImport._ 2 | import sbt._ 3 | 4 | object Dependencies { 5 | 6 | val sharedLibs = Def.setting { 7 | Seq( 8 | "me.chrons" %%% "boopickle" % "1.1.0", 9 | "com.softwaremill.macwire" %% "macros" % "1.0.7", 10 | "com.github.benhutchison" %%% "prickle" % "1.1.9", 11 | "org.scala-lang.modules" %% "scala-async" % "0.9.5" 12 | ) 13 | } 14 | 15 | val clientLibs = Def.setting { 16 | Seq( 17 | "com.lihaoyi" %%% "scalatags" % "0.5.2", 18 | "com.lihaoyi" %%% "scalarx" % "0.2.8", 19 | "org.scala-js" %%% "scalajs-dom" % "0.8.2", 20 | "be.doeraene" %%% "scalajs-jquery" % "0.8.1", 21 | "org.monifu" %%% "monifu" % "1.0-M11" 22 | ) 23 | } 24 | 25 | val commonLibs = Seq( 26 | "com.typesafe" % "config" % "1.3.0", 27 | "com.typesafe.akka" %% "akka-cluster" % Versions.Akka, 28 | "com.typesafe.akka" %% "akka-cluster-tools" % Versions.Akka, 29 | "com.typesafe.akka" %% "akka-cluster-metrics" % Versions.Akka, 30 | "com.typesafe.akka" %% "akka-distributed-data-experimental" % Versions.Akka 31 | ) 32 | 33 | val frontendLibs = Seq( 34 | "com.vmunier" %% "play-scalajs-scripts" % "0.3.0", 35 | "com.typesafe.akka" % "akka-slf4j_2.11" % Versions.Akka, 36 | "com.lihaoyi" %% "scalatags" % "0.5.2", 37 | "com.typesafe.akka" %% "akka-persistence" % Versions.Akka, 38 | "org.iq80.leveldb" % "leveldb" % "0.7", 39 | "org.fusesource.leveldbjni" % "leveldbjni-all" % "1.8", 40 | "org.webjars.bower" % "materialize" % "0.97.0" 41 | ) 42 | 43 | val backendLibs = Seq( 44 | "com.typesafe.akka" %% "akka-stream-experimental" % Versions.Streams, 45 | "com.typesafe.akka" %% "akka-http-core-experimental" % Versions.Streams, 46 | "com.typesafe.akka" %% "akka-http-experimental" % Versions.Streams, 47 | "net.codingwell" %% "scala-guice" % "4.0.0", 48 | "org.scala-lang.modules" %% "scala-async" % "0.9.5", 49 | "org.imgscalr" % "imgscalr-lib" % "4.2", 50 | 51 | //test 52 | "org.scalatest" %% "scalatest" % "2.2.5" % "test", 53 | "com.typesafe.akka" %% "akka-stream-testkit-experimental" % Versions.Streams % "test", 54 | "com.typesafe.akka" %% "akka-http-testkit-experimental" % Versions.Streams % "test" 55 | ) 56 | 57 | } 58 | 59 | object Versions { 60 | val Akka = "2.4.0" 61 | val Streams = "1.0" 62 | } 63 | -------------------------------------------------------------------------------- /backend/src/main/scala/tmt/library/Cors.scala: -------------------------------------------------------------------------------- 1 | package tmt.library 2 | 3 | import akka.http.scaladsl.model.HttpHeader 4 | import akka.http.scaladsl.model.HttpMethods._ 5 | import akka.http.scaladsl.model.HttpResponse 6 | import akka.http.scaladsl.model.headers._ 7 | import akka.http.scaladsl.server.Directive0 8 | import akka.http.scaladsl.server.Directives._ 9 | import akka.http.scaladsl.server.MethodRejection 10 | import akka.http.scaladsl.server.RejectionHandler 11 | 12 | object Cors { 13 | val corsAllowOrigins: List[String] = List("*") 14 | val corsAllowedHeaders: List[String] = List("Origin", "X-Requested-With", "Content-Type", "Accept", "Accept-Encoding", "Accept-Language", "Host", "Referer", "User-Agent") 15 | val corsAllowCredentials: Boolean = true 16 | val optionsCorsHeaders: List[HttpHeader] = List[HttpHeader]( 17 | `Access-Control-Allow-Headers`(corsAllowedHeaders.mkString(", ")), 18 | `Access-Control-Max-Age`(60 * 60 * 24 * 20), // cache pre-flight response for 20 days 19 | `Access-Control-Allow-Credentials`(corsAllowCredentials) 20 | ) 21 | 22 | private def corsRejectionHandler(allowOrigin: `Access-Control-Allow-Origin`) = RejectionHandler 23 | .newBuilder().handle { 24 | case MethodRejection(supported) => 25 | complete(HttpResponse().withHeaders( 26 | `Access-Control-Allow-Methods`(OPTIONS, supported) :: 27 | allowOrigin :: 28 | optionsCorsHeaders 29 | )) 30 | } 31 | .result() 32 | 33 | private def originToAllowOrigin(origin: Origin): Option[`Access-Control-Allow-Origin`] = 34 | if (corsAllowOrigins.contains("*") || corsAllowOrigins.contains(origin.value)) 35 | origin.origins.headOption.map(`Access-Control-Allow-Origin`.apply) 36 | else 37 | None 38 | 39 | def cors[T]: Directive0 = mapInnerRoute { route => context => 40 | ((context.request.method, context.request.header[Origin].flatMap(originToAllowOrigin)) match { 41 | case (OPTIONS, Some(allowOrigin)) => 42 | handleRejections(corsRejectionHandler(allowOrigin)) { 43 | respondWithHeaders(allowOrigin, `Access-Control-Allow-Credentials`(corsAllowCredentials)) { 44 | route 45 | } 46 | } 47 | case (_, Some(allowOrigin)) => 48 | respondWithHeaders(allowOrigin, `Access-Control-Allow-Credentials`(corsAllowCredentials)) { 49 | route 50 | } 51 | case (_, _) => 52 | route 53 | })(context) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /client/src/main/scala/tmt/views/SubscriptionView.scala: -------------------------------------------------------------------------------- 1 | package tmt.views 2 | 3 | import org.scalajs.dom.ext.Ajax 4 | import rx._ 5 | import tmt.app.ViewData 6 | import tmt.framework.Framework._ 7 | import tmt.framework.Helpers._ 8 | import tmt.shared.models._ 9 | 10 | import scala.concurrent.ExecutionContext 11 | import scalatags.JsDom.all._ 12 | 13 | class SubscriptionView(dataStore: ViewData)(implicit ec: ExecutionContext) extends View { 14 | 15 | val topicName = Var("") 16 | val serverName = Var("") 17 | val consumers = dataStore.consumersOf(topicName) 18 | 19 | val connection = Rx(Connection(serverName(), topicName())) 20 | 21 | val connectionSet = Var(ConnectionSet.empty) 22 | 23 | Obs(dataStore.connectionSet) { 24 | connectionSet() = dataStore.connectionSet() 25 | } 26 | 27 | val noData = Rx( 28 | topicName().isEmpty 29 | || serverName().isEmpty 30 | || consumers().isEmpty 31 | || dataStore.producers().isEmpty 32 | || !consumers().contains(serverName()) 33 | ) 34 | 35 | val disabledStyle = Rx(if(noData()) "disabled" else "") 36 | 37 | def viewTitle = h5("Connect") 38 | 39 | def viewContent = div( 40 | label("Output server"), 41 | makeSelection(dataStore.producers, topicName), 42 | 43 | label("Input server"), 44 | makeSelection(consumers, serverName) 45 | ) 46 | 47 | def viewAction = div( 48 | a(cls := Rx(s"btn-floating waves-effect waves-light ${disabledStyle()}"))( 49 | onclick := { () => if(!noData()) addConnection() }, 50 | i(cls := "material-icons")("add") 51 | ), 52 | Rx { 53 | val rows = connectionSet().connections.toSeq.map(makeRow) 54 | table(tbody(rows)) 55 | } 56 | ) 57 | 58 | def makeRow(c: Connection) = tr( 59 | td(c.topic), 60 | td("====>"), 61 | td(c.server), 62 | td( 63 | a(cls := "btn-floating waves-effect waves-light red")( 64 | onclick := { () => removeConnection(c) }, 65 | i(cls := "material-icons")("remove") 66 | ) 67 | ) 68 | ) 69 | 70 | def addConnection() = { 71 | subscribe(connection()).onSuccess { 72 | case x if x.status == 202 => connectionSet() = connectionSet().add(connection()) 73 | } 74 | } 75 | 76 | def removeConnection(connection: Connection) = { 77 | unsubscribe(connection) 78 | connectionSet() = connectionSet().remove(connection) 79 | } 80 | 81 | def subscribe(connection: Connection) = Ajax.post(s"${connection.server}/subscribe/${connection.topic}") 82 | 83 | def unsubscribe(connection: Connection) = Ajax.post(s"${connection.server}/unsubscribe/${connection.topic}") 84 | } 85 | -------------------------------------------------------------------------------- /activator.bat: -------------------------------------------------------------------------------- 1 | @REM activator launcher script 2 | @REM 3 | @REM Environment: 4 | @REM In order for Activator to work you must have Java available on the classpath 5 | @REM JAVA_HOME - location of a JDK home dir (optional if java on path) 6 | @REM CFG_OPTS - JVM options (optional) 7 | @REM Configuration: 8 | @REM activatorconfig.txt found in the ACTIVATOR_HOME or ACTIVATOR_HOME/ACTIVATOR_VERSION 9 | @setlocal enabledelayedexpansion 10 | 11 | @echo off 12 | 13 | set "var1=%~1" 14 | if defined var1 ( 15 | if "%var1%"=="help" ( 16 | echo. 17 | echo Usage activator [options] [command] 18 | echo. 19 | echo Commands: 20 | echo ui Start the Activator UI 21 | echo new [name] [template-id] Create a new project with [name] using template [template-id] 22 | echo list-templates Print all available template names 23 | echo help Print this message 24 | echo. 25 | echo Options: 26 | echo -jvm-debug [port] Turn on JVM debugging, open at the given port. Defaults to 9999 if no port given. 27 | echo. 28 | echo Environment variables ^(read from context^): 29 | echo JAVA_OPTS Environment variable, if unset uses "" 30 | echo SBT_OPTS Environment variable, if unset uses "" 31 | echo ACTIVATOR_OPTS Environment variable, if unset uses "" 32 | echo. 33 | echo Please note that in order for Activator to work you must have Java available on the classpath 34 | echo. 35 | goto :end 36 | ) 37 | ) 38 | 39 | if "%ACTIVATOR_HOME%"=="" ( 40 | set "ACTIVATOR_HOME=%~dp0" 41 | @REM remove trailing "\" from path 42 | set ACTIVATOR_HOME=!ACTIVATOR_HOME:~0,-1! 43 | ) 44 | 45 | set ERROR_CODE=0 46 | set APP_VERSION=1.3.2 47 | set ACTIVATOR_LAUNCH_JAR=activator-launch-%APP_VERSION%.jar 48 | 49 | rem Detect if we were double clicked, although theoretically A user could 50 | rem manually run cmd /c 51 | for %%x in (%cmdcmdline%) do if %%~x==/c set DOUBLECLICKED=1 52 | 53 | rem FIRST we load a config file of extra options (if there is one) 54 | set "CFG_FILE_HOME=%UserProfile%\.activator\activatorconfig.txt" 55 | set "CFG_FILE_VERSION=%UserProfile%\.activator\%APP_VERSION%\activatorconfig.txt" 56 | set CFG_OPTS= 57 | if exist %CFG_FILE_VERSION% ( 58 | FOR /F "tokens=* eol=# usebackq delims=" %%i IN ("%CFG_FILE_VERSION%") DO ( 59 | set DO_NOT_REUSE_ME=%%i 60 | rem ZOMG (Part #2) WE use !! here to delay the expansion of 61 | rem CFG_OPTS, otherwise it remains "" for this loop. 62 | set CFG_OPTS=!CFG_OPTS! !DO_NOT_REUSE_ME! 63 | ) 64 | ) 65 | if "%CFG_OPTS%"=="" ( 66 | if exist %CFG_FILE_HOME% ( 67 | FOR /F "tokens=* eol=# usebackq delims=" %%i IN ("%CFG_FILE_HOME%") DO ( 68 | set DO_NOT_REUSE_ME=%%i 69 | rem ZOMG (Part #2) WE use !! here to delay the expansion of 70 | rem CFG_OPTS, otherwise it remains "" for this loop. 71 | set CFG_OPTS=!CFG_OPTS! !DO_NOT_REUSE_ME! 72 | ) 73 | ) 74 | ) 75 | 76 | rem We use the value of the JAVACMD environment variable if defined 77 | set _JAVACMD=%JAVACMD% 78 | 79 | if "%_JAVACMD%"=="" ( 80 | if not "%JAVA_HOME%"=="" ( 81 | if exist "%JAVA_HOME%\bin\java.exe" set "_JAVACMD=%JAVA_HOME%\bin\java.exe" 82 | 83 | rem if there is a java home set we make sure it is the first picked up when invoking 'java' 84 | SET "PATH=%JAVA_HOME%\bin;%PATH%" 85 | ) 86 | ) 87 | 88 | if "%_JAVACMD%"=="" set _JAVACMD=java 89 | 90 | rem Detect if this java is ok to use. 91 | for /F %%j in ('"%_JAVACMD%" -version 2^>^&1') do ( 92 | if %%~j==java set JAVAINSTALLED=1 93 | if %%~j==openjdk set JAVAINSTALLED=1 94 | ) 95 | 96 | rem Detect the same thing about javac 97 | if "%_JAVACCMD%"=="" ( 98 | if not "%JAVA_HOME%"=="" ( 99 | if exist "%JAVA_HOME%\bin\javac.exe" set "_JAVACCMD=%JAVA_HOME%\bin\javac.exe" 100 | ) 101 | ) 102 | if "%_JAVACCMD%"=="" set _JAVACCMD=javac 103 | for /F %%j in ('"%_JAVACCMD%" -version 2^>^&1') do ( 104 | if %%~j==javac set JAVACINSTALLED=1 105 | ) 106 | 107 | rem BAT has no logical or, so we do it OLD SCHOOL! Oppan Redmond Style 108 | set JAVAOK=true 109 | if not defined JAVAINSTALLED set JAVAOK=false 110 | if not defined JAVACINSTALLED set JAVAOK=false 111 | 112 | if "%JAVAOK%"=="false" ( 113 | echo. 114 | echo A Java JDK is not installed or can't be found. 115 | if not "%JAVA_HOME%"=="" ( 116 | echo JAVA_HOME = "%JAVA_HOME%" 117 | ) 118 | echo. 119 | echo Please go to 120 | echo http://www.oracle.com/technetwork/java/javase/downloads/index.html 121 | echo and download a valid Java JDK and install before running Activator. 122 | echo. 123 | echo If you think this message is in error, please check 124 | echo your environment variables to see if "java.exe" and "javac.exe" are 125 | echo available via JAVA_HOME or PATH. 126 | echo. 127 | if defined DOUBLECLICKED pause 128 | exit /B 1 129 | ) 130 | 131 | rem Check what Java version is being used to determine what memory options to use 132 | for /f "tokens=3" %%g in ('java -version 2^>^&1 ^| findstr /i "version"') do ( 133 | set JAVA_VERSION=%%g 134 | ) 135 | 136 | rem Strips away the " characters 137 | set JAVA_VERSION=%JAVA_VERSION:"=% 138 | 139 | rem TODO Check if there are existing mem settings in JAVA_OPTS/CFG_OPTS and use those instead of the below 140 | for /f "delims=. tokens=1-3" %%v in ("%JAVA_VERSION%") do ( 141 | set MAJOR=%%v 142 | set MINOR=%%w 143 | set BUILD=%%x 144 | 145 | set META_SIZE=-XX:MetaspaceSize=64M -XX:MaxMetaspaceSize=256M 146 | if "!MINOR!" LSS "8" ( 147 | set META_SIZE=-XX:PermSize=64M -XX:MaxPermSize=256M 148 | ) 149 | 150 | set MEM_OPTS=!META_SIZE! 151 | ) 152 | 153 | rem We use the value of the JAVA_OPTS environment variable if defined, rather than the config. 154 | set _JAVA_OPTS=%JAVA_OPTS% 155 | if "%_JAVA_OPTS%"=="" set _JAVA_OPTS=%CFG_OPTS% 156 | 157 | set DEBUG_OPTS= 158 | 159 | rem Loop through the arguments, building remaining args in args variable 160 | set args= 161 | :argsloop 162 | if not "%~1"=="" ( 163 | rem Checks if the argument contains "-D" and if true, adds argument 1 with 2 and puts an equal sign between them. 164 | rem This is done since batch considers "=" to be a delimiter so we need to circumvent this behavior with a small hack. 165 | set arg1=%~1 166 | if "!arg1:~0,2!"=="-D" ( 167 | set "args=%args% "%~1"="%~2"" 168 | shift 169 | shift 170 | goto argsloop 171 | ) 172 | 173 | if "%~1"=="-jvm-debug" ( 174 | if not "%~2"=="" ( 175 | rem This piece of magic somehow checks that an argument is a number 176 | for /F "delims=0123456789" %%i in ("%~2") do ( 177 | set var="%%i" 178 | ) 179 | if defined var ( 180 | rem Not a number, assume no argument given and default to 9999 181 | set JPDA_PORT=9999 182 | ) else ( 183 | rem Port was given, shift arguments 184 | set JPDA_PORT=%~2 185 | shift 186 | ) 187 | ) else ( 188 | set JPDA_PORT=9999 189 | ) 190 | shift 191 | 192 | set DEBUG_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=!JPDA_PORT! 193 | goto argsloop 194 | ) 195 | rem else 196 | set "args=%args% "%~1"" 197 | shift 198 | goto argsloop 199 | ) 200 | 201 | :run 202 | 203 | if "!args!"=="" ( 204 | if defined DOUBLECLICKED ( 205 | set CMDS="ui" 206 | ) else set CMDS=!args! 207 | ) else set CMDS=!args! 208 | 209 | rem We add a / in front, so we get file:///C: instead of file://C: 210 | rem Java considers the later a UNC path. 211 | rem We also attempt a solid effort at making it URI friendly. 212 | rem We don't even bother with UNC paths. 213 | set JAVA_FRIENDLY_HOME_1=/!ACTIVATOR_HOME:\=/! 214 | set JAVA_FRIENDLY_HOME=/!JAVA_FRIENDLY_HOME_1: =%%20! 215 | 216 | rem Checks if the command contains spaces to know if it should be wrapped in quotes or not 217 | set NON_SPACED_CMD=%_JAVACMD: =% 218 | if "%_JAVACMD%"=="%NON_SPACED_CMD%" %_JAVACMD% %DEBUG_OPTS% %MEM_OPTS% %ACTIVATOR_OPTS% %SBT_OPTS% %_JAVA_OPTS% "-Dactivator.home=%JAVA_FRIENDLY_HOME%" -jar "%ACTIVATOR_HOME%\%ACTIVATOR_LAUNCH_JAR%" %CMDS% 219 | if NOT "%_JAVACMD%"=="%NON_SPACED_CMD%" "%_JAVACMD%" %DEBUG_OPTS% %MEM_OPTS% %ACTIVATOR_OPTS% %SBT_OPTS% %_JAVA_OPTS% "-Dactivator.home=%JAVA_FRIENDLY_HOME%" -jar "%ACTIVATOR_HOME%\%ACTIVATOR_LAUNCH_JAR%" %CMDS% 220 | 221 | if ERRORLEVEL 1 goto error 222 | goto end 223 | 224 | :error 225 | set ERROR_CODE=1 226 | 227 | :end 228 | 229 | @endlocal 230 | 231 | exit /B %ERROR_CODE% 232 | -------------------------------------------------------------------------------- /common/src/main/resources/akka-http-core-reference.conf: -------------------------------------------------------------------------------- 1 | ######################################## 2 | # akka-http-core Reference Config File # 3 | ######################################## 4 | 5 | # This is the reference config file that contains all the default settings. 6 | # Make your edits/overrides in your application.conf. 7 | 8 | akka.http { 9 | 10 | server { 11 | # The default value of the `Server` header to produce if no 12 | # explicit `Server`-header was included in a response. 13 | # If this value is the empty string and no header was included in 14 | # the request, no `Server` header will be rendered at all. 15 | server-header = akka-http/${akka.version} 16 | 17 | # The time after which an idle connection will be automatically closed. 18 | # Set to `infinite` to completely disable idle connection timeouts. 19 | idle-timeout = 60 s 20 | 21 | # The time period within which the TCP binding process must be completed. 22 | # Set to `infinite` to disable. 23 | bind-timeout = 1s 24 | 25 | # Enables/disables the addition of a `Remote-Address` header 26 | # holding the clients (remote) IP address. 27 | remote-address-header = off 28 | 29 | # Enables/disables the addition of a `Raw-Request-URI` header holding the 30 | # original raw request URI as the client has sent it. 31 | raw-request-uri-header = off 32 | 33 | # Enables/disables automatic handling of HEAD requests. 34 | # If this setting is enabled the server dispatches HEAD requests as GET 35 | # requests to the application and automatically strips off all message 36 | # bodies from outgoing responses. 37 | # Note that, even when this setting is off the server will never send 38 | # out message bodies on responses to HEAD requests. 39 | transparent-head-requests = on 40 | 41 | # Enables/disables the returning of more detailed error messages to 42 | # the client in the error response. 43 | # Should be disabled for browser-facing APIs due to the risk of XSS attacks 44 | # and (probably) enabled for internal or non-browser APIs. 45 | # Note that akka-http will always produce log messages containing the full 46 | # error details. 47 | verbose-error-messages = off 48 | 49 | # The initial size of the buffer to render the response headers in. 50 | # Can be used for fine-tuning response rendering performance but probably 51 | # doesn't have to be fiddled with in most applications. 52 | response-header-size-hint = 512 53 | 54 | # If this setting is empty the server only accepts requests that carry a 55 | # non-empty `Host` header. Otherwise it responds with `400 Bad Request`. 56 | # Set to a non-empty value to be used in lieu of a missing or empty `Host` 57 | # header to make the server accept such requests. 58 | # Note that the server will never accept HTTP/1.1 request without a `Host` 59 | # header, i.e. this setting only affects HTTP/1.1 requests with an empty 60 | # `Host` header as well as HTTP/1.0 requests. 61 | # Examples: `www.spray.io` or `example.com:8080` 62 | default-host-header = "" 63 | 64 | # Modify to tweak parsing settings on the server-side only. 65 | parsing = ${akka.http.parsing} 66 | } 67 | 68 | client { 69 | # The default value of the `User-Agent` header to produce if no 70 | # explicit `User-Agent`-header was included in a request. 71 | # If this value is the empty string and no header was included in 72 | # the request, no `User-Agent` header will be rendered at all. 73 | user-agent-header = akka-http/${akka.version} 74 | 75 | # The time period within which the TCP connecting process must be completed. 76 | connecting-timeout = 10s 77 | 78 | # The time after which an idle connection will be automatically closed. 79 | # Set to `infinite` to completely disable idle timeouts. 80 | idle-timeout = 60 s 81 | 82 | # The initial size of the buffer to render the request headers in. 83 | # Can be used for fine-tuning request rendering performance but probably 84 | # doesn't have to be fiddled with in most applications. 85 | request-header-size-hint = 512 86 | 87 | # The proxy configurations to be used for requests with the specified 88 | # scheme. 89 | proxy { 90 | # Proxy settings for unencrypted HTTP requests 91 | # Set to 'none' to always connect directly, 'default' to use the system 92 | # settings as described in http://docs.oracle.com/javase/6/docs/technotes/guides/net/proxies.html 93 | # or specify the proxy host, port and non proxy hosts as demonstrated 94 | # in the following example: 95 | # http { 96 | # host = myproxy.com 97 | # port = 8080 98 | # non-proxy-hosts = ["*.direct-access.net"] 99 | # } 100 | http = default 101 | 102 | # Proxy settings for HTTPS requests (currently unsupported) 103 | https = default 104 | } 105 | 106 | # Modify to tweak parsing settings on the client-side only. 107 | parsing = ${akka.http.parsing} 108 | } 109 | 110 | host-connection-pool { 111 | # The maximum number of parallel connections that a connection pool to a 112 | # single host endpoint is allowed to establish. Must be greater than zero. 113 | max-connections = 4 114 | 115 | # The maximum number of times failed requests are attempted again, 116 | # (if the request can be safely retried) before giving up and returning an error. 117 | # Set to zero to completely disable request retries. 118 | max-retries = 5 119 | 120 | # The maximum number of open requests accepted into the pool across all 121 | # materializations of any of its client flows. 122 | # Protects against (accidentally) overloading a single pool with too many client flow materializations. 123 | # Note that with N concurrent materializations the max number of open request in the pool 124 | # will never exceed N * max-connections * pipelining-limit. 125 | # Must be a power of 2 and > 0! 126 | max-open-requests = 32 127 | 128 | # The maximum number of requests that are dispatched to the target host in 129 | # batch-mode across a single connection (HTTP pipelining). 130 | # A setting of 1 disables HTTP pipelining, since only one request per 131 | # connection can be "in flight" at any time. 132 | # Set to higher values to enable HTTP pipelining. 133 | # This value must be > 0. 134 | # (Note that, independently of this setting, pipelining will never be done 135 | # on a connection that still has a non-idempotent request in flight. 136 | # See http://tools.ietf.org/html/rfc7230#section-6.3.2 for more info.) 137 | pipelining-limit = 1 138 | 139 | # The time after which an idle connection pool (without pending requests) 140 | # will automatically terminate itself. Set to `infinite` to completely disable idle timeouts. 141 | idle-timeout = 30 s 142 | 143 | # Modify to tweak client settings for host connection pools only. 144 | client = ${akka.http.client} 145 | } 146 | 147 | # The (default) configuration of the HTTP message parser for the server and the client. 148 | # IMPORTANT: These settings (i.e. children of `akka.http.parsing`) can't be directly 149 | # overridden in `application.conf` to change the parser settings for client and server 150 | # at the same time. Instead, override the concrete settings beneath 151 | # `akka.http.server.parsing` and `akka.http.client.parsing` 152 | # where these settings are copied to. 153 | parsing { 154 | # The limits for the various parts of the HTTP message parser. 155 | max-uri-length = 2k 156 | max-method-length = 16 157 | max-response-reason-length = 64 158 | max-header-name-length = 64 159 | max-header-value-length = 8k 160 | max-header-count = 64 161 | max-content-length = 8m 162 | max-chunk-ext-length = 256 163 | max-chunk-size = 1m 164 | 165 | # Sets the strictness mode for parsing request target URIs. 166 | # The following values are defined: 167 | # 168 | # `strict`: RFC3986-compliant URIs are required, 169 | # a 400 response is triggered on violations 170 | # 171 | # `relaxed`: all visible 7-Bit ASCII chars are allowed 172 | # 173 | # `relaxed-with-raw-query`: like `relaxed` but additionally 174 | # the URI query is not parsed, but delivered as one raw string 175 | # as the `key` value of a single Query structure element. 176 | # 177 | uri-parsing-mode = strict 178 | 179 | # Enables/disables the logging of warning messages in case an incoming 180 | # message (request or response) contains an HTTP header which cannot be 181 | # parsed into its high-level model class due to incompatible syntax. 182 | # Note that, independently of this settings, akka-http will accept messages 183 | # with such headers as long as the message as a whole would still be legal 184 | # under the HTTP specification even without this header. 185 | # If a header cannot be parsed into a high-level model instance it will be 186 | # provided as a `RawHeader`. 187 | # If logging is enabled it is performed with the configured 188 | # `error-logging-verbosity`. 189 | illegal-header-warnings = on 190 | 191 | # Configures the verbosity with which message (request or response) parsing 192 | # errors are written to the application log. 193 | # 194 | # Supported settings: 195 | # `off` : no log messages are produced 196 | # `simple`: a condensed single-line message is logged 197 | # `full` : the full error details (potentially spanning several lines) are logged 198 | error-logging-verbosity = full 199 | 200 | # limits for the number of different values per header type that the 201 | # header cache will hold 202 | header-cache { 203 | default = 12 204 | Content-MD5 = 0 205 | Date = 0 206 | If-Match = 0 207 | If-Modified-Since = 0 208 | If-None-Match = 0 209 | If-Range = 0 210 | If-Unmodified-Since = 0 211 | User-Agent = 32 212 | } 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /activator: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ### ------------------------------- ### 4 | ### Helper methods for BASH scripts ### 5 | ### ------------------------------- ### 6 | 7 | realpath () { 8 | ( 9 | TARGET_FILE="$1" 10 | 11 | cd "$(dirname "$TARGET_FILE")" 12 | TARGET_FILE=$(basename "$TARGET_FILE") 13 | 14 | COUNT=0 15 | while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ] 16 | do 17 | TARGET_FILE=$(readlink "$TARGET_FILE") 18 | cd "$(dirname "$TARGET_FILE")" 19 | TARGET_FILE=$(basename "$TARGET_FILE") 20 | COUNT=$(($COUNT + 1)) 21 | done 22 | 23 | if [ "$TARGET_FILE" == "." -o "$TARGET_FILE" == ".." ]; then 24 | cd "$TARGET_FILE" 25 | TARGET_FILEPATH= 26 | else 27 | TARGET_FILEPATH=/$TARGET_FILE 28 | fi 29 | 30 | # make sure we grab the actual windows path, instead of cygwin's path. 31 | if ! is_cygwin; then 32 | echo "$(pwd -P)/$TARGET_FILE" 33 | else 34 | echo $(cygwinpath "$(pwd -P)/$TARGET_FILE") 35 | fi 36 | ) 37 | } 38 | 39 | # TODO - Do we need to detect msys? 40 | 41 | # Uses uname to detect if we're in the odd cygwin environment. 42 | is_cygwin() { 43 | local os=$(uname -s) 44 | case "$os" in 45 | CYGWIN*) return 0 ;; 46 | *) return 1 ;; 47 | esac 48 | } 49 | 50 | # This can fix cygwin style /cygdrive paths so we get the 51 | # windows style paths. 52 | cygwinpath() { 53 | local file="$1" 54 | if is_cygwin; then 55 | echo $(cygpath -w $file) 56 | else 57 | echo $file 58 | fi 59 | } 60 | 61 | # Make something URI friendly 62 | make_url() { 63 | url="$1" 64 | local nospaces=${url// /%20} 65 | if is_cygwin; then 66 | echo "/${nospaces//\\//}" 67 | else 68 | echo "$nospaces" 69 | fi 70 | } 71 | 72 | # Detect if we should use JAVA_HOME or just try PATH. 73 | get_java_cmd() { 74 | if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then 75 | echo "$JAVA_HOME/bin/java" 76 | else 77 | echo "java" 78 | fi 79 | } 80 | 81 | echoerr () { 82 | echo 1>&2 "$@" 83 | } 84 | vlog () { 85 | [[ $verbose || $debug ]] && echoerr "$@" 86 | } 87 | dlog () { 88 | [[ $debug ]] && echoerr "$@" 89 | } 90 | execRunner () { 91 | # print the arguments one to a line, quoting any containing spaces 92 | [[ $verbose || $debug ]] && echo "# Executing command line:" && { 93 | for arg; do 94 | if printf "%s\n" "$arg" | grep -q ' '; then 95 | printf "\"%s\"\n" "$arg" 96 | else 97 | printf "%s\n" "$arg" 98 | fi 99 | done 100 | echo "" 101 | } 102 | 103 | exec "$@" 104 | } 105 | addJava () { 106 | dlog "[addJava] arg = '$1'" 107 | java_args=( "${java_args[@]}" "$1" ) 108 | } 109 | addApp () { 110 | dlog "[addApp] arg = '$1'" 111 | sbt_commands=( "${app_commands[@]}" "$1" ) 112 | } 113 | addResidual () { 114 | dlog "[residual] arg = '$1'" 115 | residual_args=( "${residual_args[@]}" "$1" ) 116 | } 117 | addDebugger () { 118 | addJava "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1" 119 | } 120 | addConfigOpts () { 121 | dlog "[addConfigOpts] arg = '$*'" 122 | for item in $* 123 | do 124 | addJava "$item" 125 | done 126 | } 127 | # a ham-fisted attempt to move some memory settings in concert 128 | # so they need not be messed around with individually. 129 | get_mem_opts () { 130 | local mem=${1:-1024} 131 | local meta=$(( $mem / 4 )) 132 | (( $meta > 256 )) || meta=256 133 | (( $meta < 1024 )) || meta=1024 134 | 135 | # default is to set memory options but this can be overridden by code section below 136 | memopts="-Xms${mem}m -Xmx${mem}m" 137 | if [[ "${java_version}" > "1.8" ]]; then 138 | extmemopts="-XX:MetaspaceSize=64m -XX:MaxMetaspaceSize=${meta}m" 139 | else 140 | extmemopts="-XX:PermSize=64m -XX:MaxPermSize=${meta}m" 141 | fi 142 | 143 | if [[ "${java_opts}" == *-Xmx* ]] || [[ "${java_opts}" == *-Xms* ]] || [[ "${java_opts}" == *-XX:MaxPermSize* ]] || [[ "${java_opts}" == *-XX:ReservedCodeCacheSize* ]] || [[ "${java_opts}" == *-XX:MaxMetaspaceSize* ]]; then 144 | # if we detect any of these settings in ${java_opts} we need to NOT output our settings. 145 | # The reason is the Xms/Xmx, if they don't line up, cause errors. 146 | memopts="" 147 | extmemopts="" 148 | fi 149 | 150 | echo "${memopts} ${extmemopts}" 151 | } 152 | require_arg () { 153 | local type="$1" 154 | local opt="$2" 155 | local arg="$3" 156 | if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then 157 | die "$opt requires <$type> argument" 158 | fi 159 | } 160 | is_function_defined() { 161 | declare -f "$1" > /dev/null 162 | } 163 | 164 | # If we're *not* running in a terminal, and we don't have any arguments, then we need to add the 'ui' parameter 165 | detect_terminal_for_ui() { 166 | [[ ! -t 0 ]] && [[ "${#residual_args}" == "0" ]] && { 167 | addResidual "ui" 168 | } 169 | # SPECIAL TEST FOR MAC 170 | [[ "$(uname)" == "Darwin" ]] && [[ "$HOME" == "$PWD" ]] && [[ "${#residual_args}" == "0" ]] && { 171 | echo "Detected MAC OSX launched script...." 172 | echo "Swapping to UI" 173 | addResidual "ui" 174 | } 175 | } 176 | 177 | # Processes incoming arguments and places them in appropriate global variables. called by the run method. 178 | process_args () { 179 | while [[ $# -gt 0 ]]; do 180 | case "$1" in 181 | -h|-help) usage; exit 1 ;; 182 | -v|-verbose) verbose=1 && shift ;; 183 | -d|-debug) debug=1 && shift ;; 184 | -mem) require_arg integer "$1" "$2" && app_mem="$2" && shift 2 ;; 185 | -jvm-debug) 186 | if echo "$2" | grep -E ^[0-9]+$ > /dev/null; then 187 | addDebugger "$2" && shift 188 | else 189 | addDebugger 9999 190 | fi 191 | shift ;; 192 | -java-home) require_arg path "$1" "$2" && java_cmd="$2/bin/java" && shift 2 ;; 193 | -D*) addJava "$1" && shift ;; 194 | -J*) addJava "${1:2}" && shift ;; 195 | *) addResidual "$1" && shift ;; 196 | esac 197 | done 198 | 199 | is_function_defined process_my_args && { 200 | myargs=("${residual_args[@]}") 201 | residual_args=() 202 | process_my_args "${myargs[@]}" 203 | } 204 | } 205 | 206 | # Actually runs the script. 207 | run() { 208 | # TODO - check for sane environment 209 | 210 | # process the combined args, then reset "$@" to the residuals 211 | process_args "$@" 212 | detect_terminal_for_ui 213 | set -- "${residual_args[@]}" 214 | argumentCount=$# 215 | 216 | #check for jline terminal fixes on cygwin 217 | if is_cygwin; then 218 | stty -icanon min 1 -echo > /dev/null 2>&1 219 | addJava "-Djline.terminal=jline.UnixTerminal" 220 | addJava "-Dsbt.cygwin=true" 221 | fi 222 | 223 | # run sbt 224 | execRunner "$java_cmd" \ 225 | "-Dactivator.home=$(make_url "$activator_home")" \ 226 | $(get_mem_opts $app_mem) \ 227 | ${java_opts[@]} \ 228 | ${java_args[@]} \ 229 | -jar "$app_launcher" \ 230 | "${app_commands[@]}" \ 231 | "${residual_args[@]}" 232 | 233 | local exit_code=$? 234 | if is_cygwin; then 235 | stty icanon echo > /dev/null 2>&1 236 | fi 237 | exit $exit_code 238 | } 239 | 240 | # Loads a configuration file full of default command line options for this script. 241 | loadConfigFile() { 242 | cat "$1" | sed '/^\#/d' 243 | } 244 | 245 | ### ------------------------------- ### 246 | ### Start of customized settings ### 247 | ### ------------------------------- ### 248 | usage() { 249 | cat < [options] 251 | 252 | Command: 253 | ui Start the Activator UI 254 | new [name] [template-id] Create a new project with [name] using template [template-id] 255 | list-templates Print all available template names 256 | -h | -help Print this message 257 | 258 | Options: 259 | -v | -verbose Make this runner chattier 260 | -d | -debug Set sbt log level to debug 261 | -mem Set memory options (default: $sbt_mem, which is $(get_mem_opts $sbt_mem)) 262 | -jvm-debug Turn on JVM debugging, open at the given port. 263 | 264 | # java version (default: java from PATH, currently $(java -version 2>&1 | grep version)) 265 | -java-home Alternate JAVA_HOME 266 | 267 | # jvm options and output control 268 | -Dkey=val Pass -Dkey=val directly to the java runtime 269 | -J-X Pass option -X directly to the java runtime 270 | (-J is stripped) 271 | 272 | # environment variables (read from context) 273 | JAVA_OPTS Environment variable, if unset uses "" 274 | SBT_OPTS Environment variable, if unset uses "" 275 | ACTIVATOR_OPTS Environment variable, if unset uses "" 276 | 277 | In the case of duplicated or conflicting options, the order above 278 | shows precedence: environment variables lowest, command line options highest. 279 | EOM 280 | } 281 | 282 | ### ------------------------------- ### 283 | ### Main script ### 284 | ### ------------------------------- ### 285 | 286 | declare -a residual_args 287 | declare -a java_args 288 | declare -a app_commands 289 | declare -r real_script_path="$(realpath "$0")" 290 | declare -r activator_home="$(realpath "$(dirname "$real_script_path")")" 291 | declare -r app_version="1.3.2" 292 | 293 | declare -r app_launcher="${activator_home}/activator-launch-${app_version}.jar" 294 | declare -r script_name=activator 295 | java_cmd=$(get_java_cmd) 296 | declare -r java_opts=( "${ACTIVATOR_OPTS[@]}" "${SBT_OPTS[@]}" "${JAVA_OPTS[@]}" "${java_opts[@]}" ) 297 | userhome="$HOME" 298 | if is_cygwin; then 299 | # cygwin sets home to something f-d up, set to real windows homedir 300 | userhome="$USERPROFILE" 301 | fi 302 | declare -r activator_user_home_dir="${userhome}/.activator" 303 | declare -r java_opts_config_home="${activator_user_home_dir}/activatorconfig.txt" 304 | declare -r java_opts_config_version="${activator_user_home_dir}/${app_version}/activatorconfig.txt" 305 | 306 | # Now check to see if it's a good enough version 307 | declare -r java_version=$("$java_cmd" -version 2>&1 | awk -F '"' '/version/ {print $2}') 308 | if [[ "$java_version" == "" ]]; then 309 | echo 310 | echo No java installations was detected. 311 | echo Please go to http://www.java.com/getjava/ and download 312 | echo 313 | exit 1 314 | elif [[ ! "$java_version" > "1.6" ]]; then 315 | echo 316 | echo The java installation you have is not up to date 317 | echo Activator requires at least version 1.6+, you have 318 | echo version $java_version 319 | echo 320 | echo Please go to http://www.java.com/getjava/ and download 321 | echo a valid Java Runtime and install before running Activator. 322 | echo 323 | exit 1 324 | fi 325 | 326 | # if configuration files exist, prepend their contents to the java args so it can be processed by this runner 327 | # a "versioned" config trumps one on the top level 328 | if [[ -f "$java_opts_config_version" ]]; then 329 | addConfigOpts $(loadConfigFile "$java_opts_config_version") 330 | elif [[ -f "$java_opts_config_home" ]]; then 331 | addConfigOpts $(loadConfigFile "$java_opts_config_home") 332 | fi 333 | 334 | run "$@" 335 | --------------------------------------------------------------------------------