├── project ├── build.properties └── plugins.sbt ├── version.sbt ├── src ├── test │ ├── resources │ │ ├── application.conf │ │ └── log4j.properties │ └── scala │ │ └── co │ │ └── coinsmith │ │ └── kafka │ │ └── cryptocoin │ │ ├── streaming │ │ ├── ExchangeProtocolActorSpec.scala │ │ ├── AkkaWebsocketSpec.scala │ │ ├── BitfinexWebsocketProtocolSpec.scala │ │ ├── OKCoinWebsocketProtocolSpec.scala │ │ └── BitstampPusherProtocolSpec.scala │ │ └── polling │ │ ├── HTTPPollingActorSpec.scala │ │ ├── OKCoinPollingActorSpec.scala │ │ ├── BitstampPollingActorSpec.scala │ │ └── BitfinexPollingActorSpec.scala └── main │ ├── scala │ └── co │ │ └── coinsmith │ │ └── kafka │ │ └── cryptocoin │ │ ├── streaming │ │ ├── package.scala │ │ ├── AkkaWebsocket.scala │ │ ├── BitstampStreamingActor.scala │ │ ├── OKCoinStreamingActor.scala │ │ └── BitfinexStreamingActor.scala │ │ ├── avro │ │ ├── GlobalScaleAndPrecision.scala │ │ └── InstantTypeMaps.scala │ │ ├── StreamingActor.scala │ │ ├── PollingActor.scala │ │ ├── KafkaCryptocoin.scala │ │ ├── polling │ │ ├── HTTPPollingActor.scala │ │ ├── BitfinexPollingActor.scala │ │ ├── OKCoinPollingActor.scala │ │ └── BitstampPollingActor.scala │ │ ├── models.scala │ │ └── producer │ │ └── KafkaCryptocoinProducer.scala │ └── resources │ ├── log4j.properties │ └── application.conf ├── .gitignore ├── .travis.yml ├── LICENSE ├── CHANGELOG.md ├── docker-compose.yml ├── docker.sbt └── README.md /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.15 2 | -------------------------------------------------------------------------------- /version.sbt: -------------------------------------------------------------------------------- 1 | version in ThisBuild := "0.0.3-SNAPSHOT" 2 | -------------------------------------------------------------------------------- /src/test/resources/application.conf: -------------------------------------------------------------------------------- 1 | kafka.cryptocoin { 2 | preprocess = false 3 | } 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # SBT 2 | target/ 3 | 4 | # Eclipse 5 | .project 6 | .classpath 7 | .settings/ 8 | 9 | # IDEA 10 | .idea/ 11 | -------------------------------------------------------------------------------- /src/main/scala/co/coinsmith/kafka/cryptocoin/streaming/package.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin 2 | 3 | 4 | package object streaming { 5 | case class Connect() 6 | } 7 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.3") 2 | addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.4.1") 3 | addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "0.8.5") 4 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO, stdout 2 | 3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n 6 | -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=ERROR, stdout 2 | 3 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 4 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n 6 | -------------------------------------------------------------------------------- /src/main/scala/co/coinsmith/kafka/cryptocoin/avro/GlobalScaleAndPrecision.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.avro 2 | 3 | import com.sksamuel.avro4s.ScaleAndPrecision 4 | 5 | object GlobalScaleAndPrecision { 6 | implicit val sp = ScaleAndPrecision(8, 20) 7 | } 8 | -------------------------------------------------------------------------------- /src/main/resources/application.conf: -------------------------------------------------------------------------------- 1 | kafka.cryptocoin { 2 | producer.config = ${?KAFKA_CRYPTOCOIN_PRODUCER_CONFIG} 3 | 4 | bootstrap-servers = ${?KAFKA_CRYPTOCOIN_BOOTSTRAP_SERVERS} 5 | schema-registry-url = ${?KAFKA_CRYPTOCOIN_SCHEMA_REGISTRY_URL} 6 | 7 | exchanges = ["bitfinex", "bitstamp", "okcoin"] 8 | preprocess = false 9 | } 10 | 11 | akka { 12 | loggers = ["akka.event.slf4j.Slf4jLogger"] 13 | loglevel = "INFO" 14 | logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" 15 | } 16 | -------------------------------------------------------------------------------- /src/test/scala/co/coinsmith/kafka/cryptocoin/streaming/ExchangeProtocolActorSpec.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.streaming 2 | 3 | import akka.actor.ActorSystem 4 | import akka.testkit.{ImplicitSender, TestKit} 5 | import org.scalatest.{BeforeAndAfterAll, FlatSpecLike} 6 | 7 | 8 | abstract class ExchangeProtocolActorSpec(_system: ActorSystem) extends TestKit(_system) 9 | with FlatSpecLike with BeforeAndAfterAll with ImplicitSender { 10 | override def afterAll { 11 | TestKit.shutdownActorSystem(system) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | scala: 3 | - 2.12.1 4 | 5 | branches: 6 | only: 7 | - develop 8 | - master 9 | 10 | sudo: required 11 | services: 12 | - docker 13 | 14 | before_deploy: 15 | - docker login -e="$DOCKER_EMAIL" -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" 16 | - git checkout $TRAVIS_BRANCH 17 | - if [ "$TRAVIS_BRANCH" == "master" ]; then git tag v$(sbt --error "export version" 2>/dev/null); fi 18 | 19 | deploy: 20 | provider: script 21 | skip_cleanup: true 22 | script: sbt ++$TRAVIS_SCALA_VERSION dockerBuildAndPush 23 | on: 24 | all_branches: true 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2015 Brandon Bradley 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /src/main/scala/co/coinsmith/kafka/cryptocoin/avro/InstantTypeMaps.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.avro 2 | 3 | import java.time.Instant 4 | 5 | import com.sksamuel.avro4s.{FromValue, ToSchema, ToValue} 6 | import org.apache.avro.Schema 7 | import org.apache.avro.Schema.Field 8 | 9 | 10 | object InstantTypeMaps { 11 | implicit object InstantToSchema extends ToSchema[Instant] { 12 | override val schema: Schema = Schema.create(Schema.Type.STRING) 13 | } 14 | 15 | implicit object InstantToValue extends ToValue[Instant] { 16 | override def apply(value: Instant): String = value.toString 17 | } 18 | 19 | implicit object InstantFromValue extends FromValue[Instant] { 20 | override def apply(value: Any, field: Field): Instant = Instant.parse(value.toString) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | CHANGELOG 2 | ========= 3 | 4 | v0.0.3 5 | ------ 6 | 7 | * exchanges can be disabled via configuration 8 | 9 | 10 | v0.0.2 11 | ------ 12 | 13 | * reimplementation using Akka Streams for polling and Actors for streaming 14 | * removal of [XChange](https://github.com/timmolter/XChange) usage due to inconsistent streaming implementations 15 | * updated to new Kafka Producer API 16 | * updated to Kafka 0.10.0.0 17 | * Avro data format for Kafka topics 18 | * consistent Avro schemas for similar data elements across different exchanges 19 | * topic per exchange and data type 20 | 21 | 22 | v0.0.1 23 | ------ 24 | 25 | * Added OKCoin polling 26 | * Added Polling supports order book data 27 | * Added Bitstamp streaming 28 | * Streaming collects order book snapshots, depth updates, and trades 29 | 30 | 31 | v0.0.0 32 | ------ 33 | 34 | * Initial release 35 | * Bitstamp and BitFinex support 36 | * Polls ticker data every 30 seconds 37 | -------------------------------------------------------------------------------- /src/test/scala/co/coinsmith/kafka/cryptocoin/polling/HTTPPollingActorSpec.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.polling 2 | 3 | import java.time.Instant 4 | 5 | import akka.NotUsed 6 | import akka.actor.ActorSystem 7 | import akka.http.scaladsl.model.ResponseEntity 8 | import akka.stream.scaladsl.{Flow, Keep} 9 | import akka.stream.testkit.scaladsl.{TestSink, TestSource} 10 | import akka.testkit.TestKit 11 | import org.json4s.JsonAST.JValue 12 | import org.scalatest.{BeforeAndAfterAll, FlatSpecLike} 13 | 14 | 15 | class HTTPPollingActorSpec(_system: ActorSystem) extends TestKit(_system) 16 | with FlatSpecLike with BeforeAndAfterAll { 17 | 18 | override def afterAll = { 19 | TestKit.shutdownActorSystem(system) 20 | } 21 | 22 | def testExchangeFlowPubSub(flow: Flow[(Instant, ResponseEntity), (String, JValue), NotUsed]) = 23 | TestSource.probe[(Instant, ResponseEntity)] 24 | .via(flow) 25 | .toMat(TestSink.probe[(String, JValue)])(Keep.both) 26 | } 27 | -------------------------------------------------------------------------------- /src/main/scala/co/coinsmith/kafka/cryptocoin/StreamingActor.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin 2 | 3 | import akka.actor.{Actor, ActorLogging, Props} 4 | import co.coinsmith.kafka.cryptocoin.streaming.{BitfinexStreamingActor, BitstampStreamingActor, OKCoinStreamingActor} 5 | import com.typesafe.config.ConfigFactory 6 | 7 | 8 | class StreamingActor extends Actor with ActorLogging { 9 | val conf = ConfigFactory.load 10 | val configuredExchanges = conf.getStringList("kafka.cryptocoin.exchanges") 11 | 12 | val supportedExchanges = Map( 13 | "bitfinex" -> Props[BitfinexStreamingActor], 14 | "bitstamp" -> Props[BitstampStreamingActor], 15 | "okcoin" -> Props[OKCoinStreamingActor] 16 | ) 17 | 18 | val exchangeActors = supportedExchanges filterKeys { name => 19 | configuredExchanges contains name 20 | } map { case (name, props) => 21 | name -> context.actorOf(props, name) 22 | } 23 | 24 | if (exchangeActors.isEmpty) { 25 | log.warning("Streaming has no active exchanges.") 26 | } 27 | 28 | def receive = { 29 | case _ => 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/scala/co/coinsmith/kafka/cryptocoin/PollingActor.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin 2 | 3 | import akka.actor.{Actor, ActorLogging, Props} 4 | import com.typesafe.config.ConfigFactory 5 | import polling.{BitfinexPollingActor, BitstampPollingActor, OKCoinPollingActor} 6 | 7 | 8 | class PollingActor extends Actor with ActorLogging { 9 | val conf = ConfigFactory.load 10 | val configuredExchanges = conf.getStringList("kafka.cryptocoin.exchanges") 11 | 12 | val supportedExchanges = Map( 13 | "bitfinex" -> Props[BitfinexPollingActor], 14 | "bitstamp" -> Props[BitstampPollingActor], 15 | "okcoin" -> Props[OKCoinPollingActor] 16 | ) 17 | 18 | val exchangeActors = supportedExchanges filterKeys { name => 19 | configuredExchanges contains name 20 | } map { case (name, props) => 21 | name -> context.actorOf(props, name) 22 | } 23 | 24 | if (exchangeActors.isEmpty) { 25 | log.warning("Polling has no active exchanges.") 26 | } 27 | 28 | exchangeActors foreach { _._2 ! "start" } 29 | 30 | def receive = { 31 | case _ => 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | zookeeper: 4 | image: "confluentinc/cp-zookeeper:3.2.1" 5 | ports: 6 | - '2181' 7 | environment: 8 | ZOOKEEPER_CLIENT_PORT: 2181 9 | 10 | broker: 11 | image: "confluentinc/cp-kafka:3.2.1" 12 | ports: 13 | - '9092' 14 | depends_on: 15 | - zookeeper 16 | environment: 17 | KAFKA_BROKER_ID: 1 18 | KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' 19 | KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://broker:9092" 20 | 21 | schema-registry: 22 | image: "confluentinc/cp-schema-registry:3.2.1" 23 | ports: 24 | - '8081' 25 | depends_on: 26 | - broker 27 | environment: 28 | - SCHEMA_REGISTRY_HOST_NAME=schema-registry 29 | - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 30 | - SCHEMA_REGISTRY_LISTENERS=http://0.0.0.0:8081 31 | 32 | kafka-cryptocoin: 33 | image: "coinsmith/kafka-cryptocoin:develop" 34 | environment: 35 | - KAFKA_CRYPTOCOIN_SCHEMA_REGISTRY_URL=http://schema-registry:8081 36 | - KAFKA_CRYPTOCOIN_BOOTSTRAP_SERVERS=broker:9092 37 | depends_on: 38 | - schema-registry 39 | -------------------------------------------------------------------------------- /src/main/scala/co/coinsmith/kafka/cryptocoin/KafkaCryptocoin.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin 2 | 3 | import java.io.IOException 4 | import java.net.{MalformedURLException, URL} 5 | 6 | import akka.actor.{ActorSystem, Props} 7 | import akka.event.Logging 8 | import com.typesafe.config.ConfigFactory 9 | 10 | 11 | object KafkaCryptocoin { 12 | implicit val system = ActorSystem("KafkaCryptocoinSystem") 13 | val log = Logging(system.eventStream, this.getClass.getName) 14 | 15 | val conf = ConfigFactory.load 16 | val schemaRegistryUrl = conf.getString("kafka.cryptocoin.schema-registry-url") 17 | 18 | def isSchemaRegistryAvailable: Boolean = { 19 | val connection = new URL(schemaRegistryUrl).openConnection 20 | try { 21 | connection.connect 22 | return true 23 | } catch { 24 | case e: MalformedURLException => 25 | throw new RuntimeException("Schema Registry URL is malformed.") 26 | case e: IOException => 27 | log.error(e, "Schema registry at {} is unreachable.", schemaRegistryUrl) 28 | return false 29 | } 30 | } 31 | 32 | def main(args: Array[String]) { 33 | while(isSchemaRegistryAvailable == false) { 34 | log.info("Retrying connection to schema registry in three seconds.") 35 | Thread.sleep(3000) 36 | } 37 | 38 | val streamingActor = system.actorOf(Props[StreamingActor], "streaming") 39 | val pollingActor = system.actorOf(Props[PollingActor], "polling") 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /docker.sbt: -------------------------------------------------------------------------------- 1 | import java.util.Date 2 | 3 | docker <<= docker.dependsOn(sbt.Keys.`package`.in(Compile, packageBin)) 4 | 5 | dockerfile in docker := { 6 | val jarFile = artifactPath.in(Compile, packageBin).value 7 | val classpath = (managedClasspath in Compile).value 8 | val mainclass = mainClass.in(Compile, packageBin).value.getOrElse(sys.error("Expected exactly one main class")) 9 | val jarTargetName = { 10 | val filename = jarFile.getName 11 | val (name, ext) = filename.splitAt(filename lastIndexOf ".") 12 | git.gitCurrentBranch.value match { 13 | case _ if git.gitUncommittedChanges.value => 14 | git.defaultFormatDateVersion(Some(name), new Date) + ext 15 | case "master" => version.value 16 | case _ => name + "-" + git.gitHeadCommit.value.get.take(8) + ext 17 | } 18 | } 19 | val jarTarget = s"/app/${jarTargetName}" 20 | val classpathString = classpath.files.map("/app/" + _.getName) 21 | .mkString(":") + ":" + jarTarget 22 | new Dockerfile { 23 | from("openjdk") 24 | add(classpath.files, "/app/") 25 | add(jarFile, jarTarget) 26 | entryPointShell("exec", "java", "$JAVA_OPTS", "-cp", classpathString, mainclass) 27 | } 28 | } 29 | 30 | val dockerTags = settingKey[Seq[Option[String]]]("Docker tags to build") 31 | dockerTags := { 32 | val branchTag = git.gitCurrentBranch.value match { 33 | case "master" => "latest" 34 | case "develop" => "develop" 35 | case _ => version.value 36 | } 37 | 38 | git.gitCurrentTags.value.map { Option(_) } :+ Option(branchTag) 39 | } 40 | 41 | imageNames in docker := dockerTags.value.map { t => 42 | ImageName( 43 | namespace = Some(organization.value), 44 | repository = name.value, 45 | tag = t 46 | ) 47 | } 48 | -------------------------------------------------------------------------------- /src/test/scala/co/coinsmith/kafka/cryptocoin/streaming/AkkaWebsocketSpec.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.streaming 2 | 3 | import java.net.URI 4 | import java.time.Instant 5 | 6 | import akka.actor.ActorSystem 7 | import akka.http.scaladsl.Http 8 | import akka.http.scaladsl.model.ws.{Message, TextMessage} 9 | import akka.http.scaladsl.server.Directives 10 | import akka.stream.ActorMaterializer 11 | import akka.stream.scaladsl.Flow 12 | import akka.testkit.{TestKit, TestProbe} 13 | import org.json4s.JsonAST.JObject 14 | import org.scalatest.{BeforeAndAfterAll, FlatSpecLike} 15 | 16 | 17 | class AkkaWebsocketSpec extends TestKit(ActorSystem("AkkaWebsocketSpecSystem")) 18 | with FlatSpecLike with BeforeAndAfterAll { 19 | implicit val materializer = ActorMaterializer() 20 | 21 | import Directives._ 22 | 23 | val testWebSocketService = 24 | Flow[Message] 25 | .collect { 26 | case m: TextMessage => m 27 | } 28 | 29 | val route = 30 | path("test") { 31 | get { 32 | handleWebSocketMessages(testWebSocketService) 33 | } 34 | } 35 | 36 | val bindingFuture = Http().bindAndHandle(route, "localhost", 8080) 37 | 38 | override def afterAll { 39 | TestKit.shutdownActorSystem(system) 40 | 41 | import system.dispatcher // for the future transformations 42 | bindingFuture 43 | .flatMap(_.unbind()) // trigger unbinding from the port 44 | .onComplete(_ => system.terminate()) // and shutdown when done 45 | } 46 | 47 | "AkkaWebsocket" should "send timestamp and message to receiver" in { 48 | val probe = TestProbe() 49 | val messages = List(TextMessage("{}")) 50 | val websocket = new AkkaWebsocket(new URI("ws://localhost:8080/test"), messages, probe.ref) 51 | 52 | websocket.connect 53 | probe.expectMsgPF() { 54 | case (t: Instant, "{}") => 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/main/scala/co/coinsmith/kafka/cryptocoin/polling/HTTPPollingActor.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.polling 2 | 3 | import scala.concurrent.Future 4 | import scala.concurrent.duration._ 5 | import java.time.Instant 6 | 7 | import akka.actor.{Actor, ActorLogging} 8 | import akka.http.scaladsl.model.{HttpResponse, ResponseEntity, StatusCodes} 9 | import akka.stream.ActorMaterializer 10 | import akka.stream.scaladsl.Flow 11 | import akka.util.ByteString 12 | import co.coinsmith.kafka.cryptocoin.ExchangeEvent 13 | import co.coinsmith.kafka.cryptocoin.producer.KafkaCryptocoinProducer 14 | import com.typesafe.config.ConfigFactory 15 | import org.json4s.DefaultFormats 16 | import org.json4s.jackson.JsonMethods._ 17 | 18 | 19 | abstract class HTTPPollingActor extends Actor with ActorLogging { 20 | val exchange: String 21 | 22 | implicit val formats = DefaultFormats 23 | implicit val materializer = ActorMaterializer() 24 | import context.system 25 | import context.dispatcher 26 | 27 | val conf = ConfigFactory.load 28 | val preprocess = conf.getBoolean("kafka.cryptocoin.preprocess") 29 | 30 | def responseEntityToString(entity: ResponseEntity): Future[String] = 31 | entity.dataBytes.runFold(ByteString(""))(_ ++ _).map(_.utf8String) 32 | 33 | def convertFlow[T : Manifest] = Flow[ExchangeEvent].map { e => 34 | val json = parse(e.data) 35 | (e.timestamp, json.extract[T]) 36 | } 37 | 38 | val periodicBehavior : Receive = { 39 | case "start" => 40 | context.system.scheduler.schedule(2 seconds, 30 seconds, self, "tick") 41 | context.system.scheduler.schedule(2 seconds, 30 seconds, self, "orderbook") 42 | } 43 | 44 | val responseBehavior : Receive = { 45 | case HttpResponse(StatusCodes.OK, headers, entity, _) => 46 | val timeCollected = Instant.now 47 | responseEntityToString(entity).foreach { data => 48 | val event = ExchangeEvent(timeCollected, KafkaCryptocoinProducer.uuid, exchange, data) 49 | KafkaCryptocoinProducer.send("polling.raw", exchange, ExchangeEvent.format.to(event)) 50 | } 51 | case response @ HttpResponse(code, _, _, _) => 52 | throw new Exception(s"Request failed with response ${response}") 53 | response.discardEntityBytes() 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/scala/co/coinsmith/kafka/cryptocoin/models.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin 2 | 3 | import java.time.Instant 4 | import java.util.UUID 5 | 6 | import co.coinsmith.kafka.cryptocoin.avro.InstantTypeMaps._ 7 | import co.coinsmith.kafka.cryptocoin.avro.GlobalScaleAndPrecision._ 8 | import com.sksamuel.avro4s.RecordFormat 9 | 10 | 11 | case class ProducerKey(producerUUID: UUID, exchange: String) 12 | object ProducerKey { 13 | val format = RecordFormat[ProducerKey] 14 | } 15 | 16 | case class ExchangeEvent(timestamp: Instant, producerUUID: UUID, exchange: String, data: String) 17 | object ExchangeEvent { 18 | val format = RecordFormat[ExchangeEvent] 19 | } 20 | 21 | case class Tick(last: BigDecimal, bid: BigDecimal, ask: BigDecimal, timeCollected: Instant, 22 | high: Option[BigDecimal] = None, low: Option[BigDecimal] = None, open: Option[BigDecimal] = None, 23 | volume: Option[BigDecimal] = None, vwap: Option[BigDecimal] = None, 24 | bidVolume: Option[BigDecimal] = None, askVolume: Option[BigDecimal] = None, 25 | lastDailyChange: Option[BigDecimal] = None, lastDailyChangePercent: Option[BigDecimal] = None, 26 | timestamp: Option[Instant] = None) 27 | object Tick { 28 | val format = RecordFormat[Tick] 29 | } 30 | 31 | case class Order(price: BigDecimal, volume: BigDecimal, id: Option[Long] = None, 32 | timestamp: Option[Instant] = None, timeCollected: Option[Instant] = None) 33 | object Order { 34 | val format = RecordFormat[Order] 35 | 36 | def apply(price: String, volume: String) = new Order(BigDecimal(price), BigDecimal(volume)) 37 | } 38 | 39 | case class OrderBook(bids: List[Order], asks: List[Order], 40 | timestamp: Option[Instant] = None, timeCollected: Option[Instant] = None) 41 | object OrderBook { 42 | val format = RecordFormat[OrderBook] 43 | } 44 | 45 | case class Trade(price: BigDecimal, volume: BigDecimal, timestamp: Instant, timeCollected: Instant, 46 | tpe: Option[String] = None, tid: Option[Long] = None, 47 | bidoid: Option[Long] = None, askoid: Option[Long] = None, 48 | seq: Option[String] = None) 49 | object Trade { 50 | val format = RecordFormat[Trade] 51 | } 52 | -------------------------------------------------------------------------------- /src/test/scala/co/coinsmith/kafka/cryptocoin/streaming/BitfinexWebsocketProtocolSpec.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.streaming 2 | 3 | import java.time.Instant 4 | 5 | import akka.actor.ActorSystem 6 | import akka.testkit.{EventFilter, TestActorRef} 7 | import org.json4s.JsonDSL.WithBigDecimal._ 8 | 9 | 10 | class BitfinexWebsocketProtocolSpec extends ExchangeProtocolActorSpec(ActorSystem("BitfinexWebsocketProtocolSpecSystem")) { 11 | "BitfinexWebsocketProtocol" should "have a channel after subscription message" in { 12 | val actorRef = TestActorRef[BitfinexWebsocketProtocol] 13 | val actor = actorRef.underlyingActor 14 | 15 | val timeCollected = Instant.ofEpochSecond(10L) 16 | val msg = ("event" -> "subscribed") ~ 17 | ("channel" ->"book") ~ 18 | ("chanId" -> 67) ~ 19 | ("prec" -> "R0") ~ 20 | ("freq" -> "F0") ~ 21 | ("len" -> "100") ~ 22 | ("pair" -> "BTCUSD") 23 | 24 | actorRef ! (timeCollected, msg) 25 | assert(actor.subscribed == Map(BigInt(67) -> "book")) 26 | } 27 | 28 | it should "not have a channel after unsubscribing from it" in { 29 | val actorRef = TestActorRef[BitfinexWebsocketProtocol] 30 | val actor = actorRef.underlyingActor 31 | actor.subscribed += (BigInt(67) -> "book") 32 | 33 | val timeCollected = Instant.ofEpochSecond(10L) 34 | val msg = ("event" -> "unsubscribed") ~ 35 | ("status" -> "OK") ~ 36 | ("chanId" -> 67) 37 | 38 | actorRef ! (timeCollected, msg) 39 | assert(actor.subscribed == Map.empty[BigInt, String]) 40 | expectMsg(Unsubscribed("book")) 41 | } 42 | 43 | it should "unsubscribe from all channels when the trading engine resync ends" in { 44 | val actorRef = TestActorRef[BitfinexWebsocketProtocol] 45 | val actor = actorRef.underlyingActor 46 | actor.subscribed += (BigInt(67) -> "book") 47 | actor.subscribed += (BigInt(68) -> "ticker") 48 | 49 | val timeCollected = Instant.ofEpochSecond(10L) 50 | val msg = ("event" -> "info") ~ 51 | ("code" -> 20061) ~ 52 | ("msg" -> "Resync from the Trading Engine ended") 53 | 54 | actorRef ! (timeCollected, msg) 55 | expectMsg(("event" -> "unsubscribe") ~ ("chanId" -> 67)) 56 | expectMsg(("event" -> "unsubscribe") ~ ("chanId" -> 68)) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Kafka Cryptocoin 2 | ================ 3 | 4 | Kafka producer for data collection from cryptocurrency exchanges 5 | 6 | [![Build Status](https://travis-ci.org/blbradley/kafka-cryptocoin.svg?branch=develop)](https://travis-ci.org/blbradley/kafka-cryptocoin) 7 | 8 | 9 | Configuration 10 | ------------- 11 | 12 | ### Environment Variables 13 | 14 | * `KAFKA_CRYPTOCOIN_BOOTSTRAP_SERVERS`: Comma separated list of Kafka brokers. Port is required. 15 | * `KAFKA_CRYPTOCOIN_SCHEMA_REGISTRY_URL`: URL for Kafka Schema Registry 16 | * `KAFKA_CRYPTOCOIN_PRODUCER_CONFIG`: Path to Kafka producer properties file (optional) 17 | 18 | ### application.conf 19 | 20 | Main config is at `kafka.cryptocoin`. 21 | 22 | * `exchanges`: List of enabled exchanges. Default is all supported exchanges. 23 | 24 | 25 | Kafka 26 | ----- 27 | 28 | You must have a running Kafka broker and Kafka Schema Registry. 29 | These are part of [Confluent Platform 3.0](http://docs.confluent.io/3.0.0/index.html). 30 | You may go to the [Quickstart](http://docs.confluent.io/3.0.0/quickstart.html) 31 | to get them running locally for development. 32 | 33 | 34 | Usage 35 | ----- 36 | 37 | Start the the required services. Then, run: 38 | 39 | export KAFKA_CRYPTOCOIN_BOOTSTRAP_SERVERS=localhost:9092 40 | export KAFKA_CRYPTOCOIN_SCHEMA_REGISTRY_URL=http://localhost:8081 41 | sbt run 42 | 43 | ### Running the tests 44 | 45 | sbt test 46 | 47 | 48 | Docker 49 | ------ 50 | 51 | ### Docker Compose 52 | 53 | This repo includes a Compose file that will run the (development version of the) app and all services required for demo purposes. 54 | 55 | docker-compose up 56 | 57 | Prepare for lots of output. The app should wait for the required services to start. 58 | 59 | 60 | ### Run Docker image 61 | 62 | Images for development and mainline versions are built and pushed to [Docker Hub](https://hub.docker.com/r/coinsmith/kafka-cryptocoin) 63 | when a pull request is merged. Merges into git branches `develop` or `master` are 64 | images published on Docker Hub with tags `develop` and `latest` respectively. 65 | Project releases are published with tags equal to the git release tag. 66 | 67 | 68 | This downloads the latest development version and assumes your required services 69 | are running locally. 70 | 71 | docker run --net=host \ 72 | -e KAFKA_CRYPTOCOIN_BOOTSTRAP_SERVERS=localhost:9092 \ 73 | -e KAFKA_CRYPTOCOIN_SCHEMA_REGISTRY_URL=http://localhost:8081 \ 74 | coinsmith/kafka-cryptocoin:develop 75 | 76 | 77 | ### Build a Docker image 78 | 79 | sbt docker 80 | 81 | Then, you can run it as specified above. Images built from code will be tagged as the 82 | current snapshot version. 83 | -------------------------------------------------------------------------------- /src/main/scala/co/coinsmith/kafka/cryptocoin/polling/BitfinexPollingActor.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.polling 2 | 3 | import java.time.Instant 4 | 5 | import akka.http.scaladsl.Http 6 | import akka.http.scaladsl.model.HttpRequest 7 | import akka.stream.scaladsl.Flow 8 | import co.coinsmith.kafka.cryptocoin.producer.KafkaCryptocoinProducer 9 | import co.coinsmith.kafka.cryptocoin.{Order, OrderBook, Tick} 10 | 11 | 12 | case class BitfinexPollingTick(mid: String, bid: String, ask: String, last_price: String, timestamp: String) 13 | object BitfinexPollingTick { 14 | implicit def toTick(tick: BitfinexPollingTick)(implicit timeCollected: Instant) = { 15 | val Array(seconds, nanos) = tick.timestamp.split('.').map { _.toLong } 16 | Tick( 17 | tick.last_price.toDouble, tick.bid.toDouble, tick.ask.toDouble, timeCollected, 18 | timestamp = Some(Instant.ofEpochSecond(seconds, nanos)) 19 | ) 20 | } 21 | } 22 | 23 | case class BitfinexPollingOrder(price: String, amount: String, timestamp: String) 24 | 25 | case class BitfinexPollingOrderBook(bids: List[BitfinexPollingOrder], asks: List[BitfinexPollingOrder]) 26 | object BitfinexPollingOrderBook { 27 | def toOrder(o: BitfinexPollingOrder) = 28 | Order(BigDecimal(o.price), BigDecimal(o.amount), timestamp = Some(Instant.ofEpochSecond(o.timestamp.toDouble.toLong))) 29 | 30 | implicit def toOrderBook(ob: BitfinexPollingOrderBook)(implicit timeCollected: Instant) = 31 | OrderBook( 32 | ob.bids map toOrder, 33 | ob.asks map toOrder, 34 | timeCollected = Some(timeCollected) 35 | ) 36 | } 37 | 38 | class BitfinexPollingActor extends HTTPPollingActor { 39 | import akka.pattern.pipe 40 | import context.system 41 | import context.dispatcher 42 | 43 | val exchange = "bitfinex" 44 | val topicPrefix = "bitfinex.polling.btcusd." 45 | val http = Http(context.system) 46 | 47 | val tickFlow = Flow[(Instant, BitfinexPollingTick)].map { case (t, tick) => 48 | implicit val timeCollected = t 49 | ("ticks", Tick.format.to(tick)) 50 | } 51 | 52 | val orderbookFlow = Flow[(Instant, BitfinexPollingOrderBook)].map { case (t, ob) => 53 | implicit val timeCollected = t 54 | ("orderbook", OrderBook.format.to(ob)) 55 | } 56 | 57 | def receive = periodicBehavior orElse responseBehavior orElse { 58 | case (topic: String, value: Object) => 59 | KafkaCryptocoinProducer.send(topicPrefix + topic, value) 60 | case "tick" => 61 | http.singleRequest(HttpRequest(uri = "https://api.bitfinex.com/v1/pubticker/btcusd")) pipeTo self 62 | case "orderbook" => 63 | http.singleRequest(HttpRequest(uri = "https://api.bitfinex.com/v1/book/btcusd")) pipeTo self 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/scala/co/coinsmith/kafka/cryptocoin/polling/OKCoinPollingActor.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.polling 2 | 3 | import java.time.Instant 4 | 5 | import akka.http.scaladsl.Http 6 | import akka.http.scaladsl.model.HttpRequest 7 | import akka.stream.scaladsl.Flow 8 | import co.coinsmith.kafka.cryptocoin.producer.KafkaCryptocoinProducer 9 | import co.coinsmith.kafka.cryptocoin.{Order, OrderBook, Tick} 10 | 11 | case class OKCoinPollingTick( 12 | buy: String, 13 | high: String, 14 | last: String, 15 | low: String, 16 | sell: String, 17 | vol: String 18 | ) 19 | case class OKCoinPollingDatedTick(date: String, ticker: OKCoinPollingTick) 20 | object OKCoinPollingDatedTick { 21 | implicit def toTick(datedTick: OKCoinPollingDatedTick)(implicit timeCollected: Instant) = { 22 | val tick = datedTick.ticker 23 | Tick(tick.last.toDouble, tick.buy.toDouble, tick.sell.toDouble, timeCollected, 24 | Some(tick.high.toDouble), Some(tick.low.toDouble), None, 25 | volume = Some(tick.vol.toDouble), vwap = None, 26 | timestamp = Some(Instant.ofEpochSecond(datedTick.date.toLong)) 27 | ) 28 | } 29 | } 30 | 31 | case class OKCoinPollingOrderBook(asks: List[List[Double]], bids: List[List[Double]]) 32 | object OKCoinPollingOrderBook { 33 | val toOrder = { o: List[Double] => Order(o(0), o(1)) } 34 | 35 | implicit def toOrderBook(ob: OKCoinPollingOrderBook)(implicit timeCollected: Instant) = 36 | OrderBook( 37 | ob.bids map toOrder, 38 | ob.asks map toOrder, 39 | None, 40 | Some(timeCollected) 41 | ) 42 | } 43 | 44 | class OKCoinPollingActor extends HTTPPollingActor { 45 | import akka.pattern.pipe 46 | import context.system 47 | import context.dispatcher 48 | 49 | val exchange = "okcoin" 50 | val topicPrefix = "okcoin.polling.btcusd." 51 | val http = Http(context.system) 52 | 53 | val tickFlow = Flow[(Instant, OKCoinPollingDatedTick)].map { case (t, tick) => 54 | implicit val timeCollected = t 55 | ("ticks", Tick.format.to(tick)) 56 | } 57 | 58 | val orderbookFlow = Flow[(Instant, OKCoinPollingOrderBook)].map { case (t, ob) => 59 | implicit val timeCollected = t 60 | ("orderbook", OrderBook.format.to(ob)) 61 | } 62 | 63 | def receive = periodicBehavior orElse responseBehavior orElse { 64 | case (topic: String, value: Object) => 65 | KafkaCryptocoinProducer.send(topicPrefix + topic, value) 66 | case "tick" => 67 | http.singleRequest(HttpRequest(uri = "https://www.okcoin.cn/api/v1/ticker.do?symbol=btc_cny")) pipeTo self 68 | case "orderbook" => 69 | http.singleRequest(HttpRequest(uri = "https://www.okcoin.cn/api/v1/depth.do??symbol=btc_cny")) pipeTo self 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/scala/co/coinsmith/kafka/cryptocoin/polling/BitstampPollingActor.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.polling 2 | 3 | import java.time.Instant 4 | 5 | import akka.http.scaladsl.Http 6 | import akka.http.scaladsl.model.HttpRequest 7 | import akka.stream.scaladsl.Flow 8 | import co.coinsmith.kafka.cryptocoin.producer.KafkaCryptocoinProducer 9 | import co.coinsmith.kafka.cryptocoin.{Order, OrderBook, Tick} 10 | 11 | case class BitstampPollingTick( 12 | high: String, 13 | last: String, 14 | timestamp: String, 15 | bid: String, 16 | vwap: String, 17 | volume: String, 18 | low: String, 19 | ask: String, 20 | open: Double 21 | ) 22 | object BitstampPollingTick { 23 | implicit def toTick(tick: BitstampPollingTick)(implicit timeCollected: Instant) = 24 | Tick( 25 | tick.last.toDouble, tick.bid.toDouble, tick.ask.toDouble, timeCollected, 26 | Some(tick.high.toDouble), Some(tick.low.toDouble), Some(tick.open), 27 | volume = Some(tick.volume.toDouble), vwap = Some(tick.vwap.toDouble), 28 | timestamp = Some(Instant.ofEpochSecond(tick.timestamp.toLong)) 29 | ) 30 | } 31 | 32 | case class BitstampPollingOrderBook( 33 | timestamp: String, 34 | bids: List[List[String]], 35 | asks: List[List[String]] 36 | ) 37 | object BitstampPollingOrderBook { 38 | val toOrder = { o: List[String] => Order(o(0), o(1)) } 39 | 40 | implicit def toOrderBook(ob: BitstampPollingOrderBook)(implicit timeCollected: Instant) = 41 | OrderBook( 42 | ob.bids map toOrder, 43 | ob.asks map toOrder, 44 | Some(Instant.ofEpochSecond(ob.timestamp.toLong)), 45 | Some(timeCollected) 46 | ) 47 | } 48 | 49 | class BitstampPollingActor extends HTTPPollingActor { 50 | import akka.pattern.pipe 51 | import context.system 52 | import context.dispatcher 53 | 54 | val exchange = "bitstamp" 55 | val topicPrefix = "bitstamp.polling.btcusd." 56 | val http = Http(context.system) 57 | 58 | val tickFlow = Flow[(Instant, BitstampPollingTick)].map { case (t, tick) => 59 | implicit val timeCollected = t 60 | ("ticks", Tick.format.to(tick)) 61 | } 62 | 63 | val orderbookFlow = Flow[(Instant, BitstampPollingOrderBook)].map { case (t, ob) => 64 | implicit val timeCollected = t 65 | ("orderbook", OrderBook.format.to(ob)) 66 | } 67 | 68 | def receive = periodicBehavior orElse responseBehavior orElse { 69 | case (topic: String, value: Object) => 70 | KafkaCryptocoinProducer.send(topicPrefix + topic, value) 71 | case "tick" => 72 | http.singleRequest(HttpRequest(uri = "https://www.bitstamp.net/api/v2/ticker/btcusd/")) pipeTo self 73 | case "orderbook" => 74 | http.singleRequest(HttpRequest(uri = "https://www.bitstamp.net/api/v2/order_book/btcusd/")) pipeTo self 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/main/scala/co/coinsmith/kafka/cryptocoin/producer/KafkaCryptocoinProducer.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.producer 2 | 3 | import scala.util.{Failure, Success, Try} 4 | import java.io.FileInputStream 5 | import java.util.{Properties, UUID} 6 | 7 | import akka.Done 8 | import akka.actor.ActorSystem 9 | import akka.event.Logging 10 | import akka.kafka.ProducerSettings 11 | import akka.kafka.scaladsl.Producer 12 | import akka.stream.ActorMaterializer 13 | import akka.stream.scaladsl.Source 14 | import co.coinsmith.kafka.cryptocoin.KafkaCryptocoin 15 | import com.typesafe.config.ConfigFactory 16 | import io.confluent.kafka.serializers.KafkaAvroSerializer 17 | import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord} 18 | 19 | object KafkaCryptocoinProducer { 20 | val logger = Logging(KafkaCryptocoin.system.eventStream, this.getClass.getName) 21 | 22 | val uuid = UUID.randomUUID 23 | logger.info("Producer UUID: {}", uuid) 24 | 25 | val conf = ConfigFactory.load 26 | val brokers = conf.getString("kafka.cryptocoin.bootstrap-servers") 27 | val schemaRegistryUrl = conf.getString("kafka.cryptocoin.schema-registry-url") 28 | 29 | val props = new Properties 30 | props.put("acks", "all") 31 | props.put("retires", Int.MaxValue.toString) 32 | props.put("max.block.ms", Long.MaxValue.toString) 33 | props.put("max.in.flight.requests.per.connection", "1") 34 | 35 | val producerConfigPath = "kafka.cryptocoin.producer.config" 36 | if (conf.hasPath(producerConfigPath)) { 37 | val filename = conf.getString(producerConfigPath) 38 | val propsFile = new FileInputStream(filename) 39 | props.load(propsFile) 40 | } 41 | 42 | props.put("bootstrap.servers", brokers) 43 | props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "io.confluent.kafka.serializers.KafkaAvroSerializer") 44 | props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "io.confluent.kafka.serializers.KafkaAvroSerializer") 45 | props.put("schema.registry.url", schemaRegistryUrl) 46 | val producer = new KafkaProducer[Object, Object](props) 47 | 48 | val settings = ProducerSettings(KafkaCryptocoin.system, new KafkaAvroSerializer, new KafkaAvroSerializer) 49 | val producerSink = Producer.plainSink(settings, producer) 50 | 51 | def producerComplete(td: Try[Done]) = td match { 52 | case Success(d) => 53 | case Failure(ex) => throw ex 54 | } 55 | 56 | def send(topic: String, key: Object, value: Object)(implicit system: ActorSystem, materializer: ActorMaterializer) = { 57 | import system.dispatcher 58 | val data = new ProducerRecord[Object, Object](topic, key, value) 59 | val closed = Source.single(data).runWith(producerSink) 60 | closed.onComplete(producerComplete) 61 | } 62 | 63 | def send(topic: String, value: Object)(implicit system: ActorSystem, materializer: ActorMaterializer): Unit = 64 | send(topic, null, value) 65 | } 66 | -------------------------------------------------------------------------------- /src/main/scala/co/coinsmith/kafka/cryptocoin/streaming/AkkaWebsocket.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.streaming 2 | 3 | import scala.concurrent.{ExecutionContext, Future, Promise} 4 | import java.net.URI 5 | import java.time.Instant 6 | 7 | import akka.Done 8 | import akka.actor.{ActorRef, ActorSystem} 9 | import akka.event.Logging 10 | import akka.http.scaladsl.Http 11 | import akka.http.scaladsl.model.ws.{Message, TextMessage, WebSocketRequest} 12 | import akka.stream.scaladsl.{Broadcast, Flow, GraphDSL, Keep, Sink, Source, SourceQueueWithComplete, Zip} 13 | import akka.stream.{ActorMaterializer, FlowShape, OverflowStrategy} 14 | import org.json4s.JsonAST.JValue 15 | import org.json4s.jackson.JsonMethods.parse 16 | 17 | 18 | class AkkaWebsocket(uri: URI, messages: List[TextMessage], receiver: ActorRef)(implicit system: ActorSystem) { 19 | implicit val ec = system.dispatcher 20 | implicit val materializer = ActorMaterializer() 21 | val log = Logging(system.eventStream, this.getClass.getName) 22 | var queue: SourceQueueWithComplete[Message] = _ 23 | 24 | // emit initial message and then keep the connection open 25 | // val source: Source[Message, Promise[Option[Message]]] = 26 | // Source(messages).concatMat(Source.maybe[Message])(Keep.right) 27 | val source = Source.queue[Message](100, OverflowStrategy.backpressure) 28 | 29 | val receiverSink = Sink.foreach[(Instant, String)] { receiver ! _ } 30 | 31 | val websocketFlow: Flow[Message, Message, (Future[Done], SourceQueueWithComplete[Message])] = 32 | Flow.fromGraph(GraphDSL.create(receiverSink, source)((_,_)) { implicit b => 33 | (sink, source) => 34 | import GraphDSL.Implicits._ 35 | 36 | val bcast = b.add(Broadcast[Message](2)) 37 | val zip = b.add(Zip[Instant, String]()) 38 | val stringFlow = b.add(Flow[Message].mapAsync(1)(messageToString)) 39 | 40 | bcast.out(0) ~> Flow[Message].map(_ => Instant.now) ~> zip.in0 41 | bcast.out(1) ~> stringFlow ~> zip.in1 42 | 43 | zip.out ~> sink 44 | 45 | FlowShape(bcast.in, source.out) 46 | }) 47 | 48 | def connect: Unit = { 49 | val (upgradeResponse, (sinkClosed, queue)) = 50 | Http().singleWebSocketRequest( 51 | WebSocketRequest(uri.toString), 52 | websocketFlow) 53 | 54 | // send initial messages 55 | for (msg <- messages) { 56 | queue.offer(msg) 57 | } 58 | 59 | // expose queue to class 60 | this.queue = queue 61 | 62 | sinkClosed.onComplete { _ => 63 | log.info("Reconnecting in five seconds to {}", uri) 64 | Thread.sleep(5000) 65 | connect 66 | } 67 | } 68 | 69 | def messageToString(m: Message)(implicit ec: ExecutionContext): Future[String] = m match { 70 | case TextMessage.Strict(m) => Future(m) 71 | case TextMessage.Streamed(ms) => ms.runFold("")(_ + _) 72 | case m => throw new RuntimeException("Received unhandled websocket message type.") 73 | } 74 | 75 | def disconnect = { 76 | queue.complete 77 | } 78 | 79 | def send(msg: Message) = { 80 | queue.offer(msg) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/test/scala/co/coinsmith/kafka/cryptocoin/streaming/OKCoinWebsocketProtocolSpec.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.streaming 2 | 3 | import java.time.Instant 4 | 5 | import akka.actor.ActorSystem 6 | import akka.testkit.TestActorRef 7 | import co.coinsmith.kafka.cryptocoin.{Order, OrderBook, Tick, Trade} 8 | import org.json4s.JsonAST.JArray 9 | import org.json4s.JsonDSL.WithBigDecimal._ 10 | 11 | 12 | class OKCoinWebsocketProtocolSpec extends ExchangeProtocolActorSpec(ActorSystem("OKCoinWebsocketProtocolSpecSystem")) { 13 | val actorRef = TestActorRef[OKCoinWebsocketProtocol] 14 | 15 | "OKCoinWebsocketProtocol" should "process a ticker message" in { 16 | val timeCollected = Instant.ofEpochSecond(10L) 17 | val timestamp = 1463444493398L 18 | val json = ("buy" -> "2984.41") ~ 19 | ("high" -> "3004.07") ~ 20 | ("last" -> "2984.40") ~ 21 | ("low" -> "2981.0") ~ 22 | ("sell" -> "2984.42") ~ 23 | ("timestamp" -> timestamp.toString) ~ 24 | ("vol" -> "639,976.04") 25 | val data = Data(timeCollected, "ok_sub_spotcny_btc_ticker", json) 26 | 27 | val expected = Tick( 28 | 2984.40, 2984.41, 2984.42, timeCollected, 29 | Some(3004.07), Some(2981.0), None, 30 | volume = Some(639976.04), 31 | timestamp = Some(Instant.ofEpochMilli(timestamp)) 32 | ) 33 | 34 | actorRef ! data 35 | expectMsg(("ticks", Tick.format.to(expected))) 36 | } 37 | 38 | it should "process an orderbook message" in { 39 | val timeCollected = Instant.ofEpochSecond(10L) 40 | val json = ("bids" -> List( 41 | List(3841.52, 0.372), 42 | List(3841.46, 0.548), 43 | List(3841.4, 0.812) 44 | )) ~ ("asks" -> List( 45 | List(3844.75, 0.04), 46 | List(3844.71, 5.181), 47 | List(3844.63, 3.143) 48 | )) ~ ("timestamp" -> "1465496881515") 49 | val data = Data(timeCollected, "ok_sub_spotcny_btc_depth_60", json) 50 | 51 | val bids = List( 52 | Order(3841.52, 0.372), 53 | Order(3841.46, 0.548), 54 | Order(3841.4, 0.812) 55 | ) 56 | val asks = List( 57 | Order(3844.75, 0.04), 58 | Order(3844.71, 5.181), 59 | Order(3844.63, 3.143) 60 | ) 61 | val timestamp = Instant.ofEpochMilli(1465496881515L) 62 | val expected = OrderBook(bids, asks, Some(timestamp), Some(timeCollected)) 63 | 64 | actorRef ! data 65 | expectMsg(("orderbook", OrderBook.format.to(expected))) 66 | } 67 | 68 | it should "process a trade message" in { 69 | // OKCoin only returns the time of the trade 70 | // timestamp should pull date from time collected 71 | val timeCollected = Instant.ofEpochSecond(1464117326L) 72 | val json = JArray(List( 73 | "2949439265" 74 | ,"2968.55" 75 | ,"0.02", 76 | "03:15:24", 77 | "ask" 78 | )) 79 | val data = Data(timeCollected, "ok_sub_spotcny_btc_trades", JArray(List(json))) 80 | 81 | val timestamp = Instant.ofEpochSecond(1464117324L) 82 | val expected = Trade(2968.55, 0.02, timestamp, timeCollected, Some("ask"), Some(2949439265L)) 83 | 84 | actorRef ! data 85 | expectMsg(("trades", Trade.format.to(expected))) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/test/scala/co/coinsmith/kafka/cryptocoin/streaming/BitstampPusherProtocolSpec.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.streaming 2 | 3 | import java.time.Instant 4 | 5 | import akka.actor.ActorSystem 6 | import akka.testkit.TestActorRef 7 | import co.coinsmith.kafka.cryptocoin.{Order, OrderBook, Trade} 8 | import org.json4s.JsonDSL.WithBigDecimal._ 9 | 10 | class BitstampPusherProtocolSpec extends ExchangeProtocolActorSpec(ActorSystem("BitstampPusherProtocolSpecSystem")) { 11 | val actorRef = TestActorRef[BitstampPusherProtocol] 12 | 13 | "BitstampPusherProtocol" should "process a trade message" in { 14 | val timeCollected = Instant.ofEpochSecond(10L) 15 | val timestamp = 1471452789L 16 | val json = ("buy_order_id" -> 146107417) ~ 17 | ("timestamp" -> timestamp.toString) ~ 18 | ("price" -> 567.0) ~ 19 | ("amount" -> 0.2) ~ 20 | ("sell_order_id" -> 146106449) ~ 21 | ("type" -> 0) ~ 22 | ("id" -> 11881674) 23 | val expected = Trade( 24 | 567.0, 0.2, Instant.ofEpochSecond(timestamp), timeCollected, 25 | Some("bid"), Some(11881674), 26 | Some(146107417), Some(146106449) 27 | ) 28 | 29 | actorRef ! ("live_trades", "trade", timeCollected, json) 30 | expectMsg(("trades", Trade.format.to(expected))) 31 | } 32 | 33 | it should "process an orderbook message" in { 34 | val json = ("bids" -> List( 35 | List("452.50000000", "5.00000000"), 36 | List("452.07000000", "6.63710000"), 37 | List("452.00000000", "3.75000000") 38 | )) ~ ("asks" -> List( 39 | List("452.97000000", "12.10000000"), 40 | List("452.98000000", "6.58530000"), 41 | List("453.00000000", "12.54279453") 42 | )) 43 | 44 | val timeCollected = Instant.ofEpochSecond(10L) 45 | val bids = List( 46 | Order("452.50000000", "5.00000000"), 47 | Order("452.07000000", "6.63710000"), 48 | Order("452.00000000", "3.75000000") 49 | ) 50 | val asks = List( 51 | Order("452.97000000", "12.10000000"), 52 | Order("452.98000000", "6.58530000"), 53 | Order("453.00000000", "12.54279453") 54 | ) 55 | val expected = OrderBook(bids, asks, None, Some(timeCollected)) 56 | 57 | actorRef ! ("order_book", "data", timeCollected, json) 58 | expectMsg(("orderbook", OrderBook.format.to(expected))) 59 | } 60 | 61 | it should "process an orderbook diff message" in { 62 | val timestamp = 1463009242L 63 | val json = ("timestamp" -> timestamp.toString) ~ 64 | ("bids" -> List( 65 | List("451.89000000", "6.57270000"), 66 | List("451.84000000", "0") 67 | )) ~ 68 | ("asks" -> List( 69 | List("453.32000000", "8.77550000"), 70 | List("453.68000000", "0.25324645"), 71 | List("458.90000000", "0") 72 | )) 73 | 74 | val timeCollected = Instant.ofEpochSecond(10L) 75 | val bids = List( 76 | Order("451.89000000", "6.57270000"), 77 | Order("451.84000000", "0") 78 | ) 79 | val asks = List( 80 | Order("453.32000000", "8.77550000"), 81 | Order("453.68000000", "0.25324645"), 82 | Order("458.90000000", "0") 83 | ) 84 | val expected = OrderBook(bids, asks, Some(Instant.ofEpochSecond(timestamp)), Some(timeCollected)) 85 | 86 | actorRef ! ("diff_order_book", "data", timeCollected, json) 87 | expectMsg(("orderbook.updates", OrderBook.format.to(expected))) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/test/scala/co/coinsmith/kafka/cryptocoin/polling/OKCoinPollingActorSpec.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.polling 2 | 3 | import java.time.Instant 4 | import java.util.UUID 5 | 6 | import akka.actor.ActorSystem 7 | import akka.http.scaladsl.model.ContentTypes 8 | import akka.stream.ActorMaterializer 9 | import akka.stream.scaladsl.Keep 10 | import akka.stream.testkit.scaladsl.{TestSink, TestSource} 11 | import akka.testkit.TestActorRef 12 | import co.coinsmith.kafka.cryptocoin.{ExchangeEvent, Order, OrderBook, Tick} 13 | import org.apache.avro.generic.GenericRecord 14 | import org.json4s.JsonDSL.WithBigDecimal._ 15 | import org.json4s.jackson.JsonMethods._ 16 | 17 | 18 | class OKCoinPollingActorSpec 19 | extends HTTPPollingActorSpec(ActorSystem("OKCoinPollingActorSpecSystem")) { 20 | implicit val materializer = ActorMaterializer() 21 | 22 | val actorRef = TestActorRef[OKCoinPollingActor] 23 | val actor = actorRef.underlyingActor 24 | 25 | "OKCoinPollingActor" should "process a ticker message" in { 26 | val timeCollected = Instant.ofEpochSecond(10L) 27 | val timestamp = Instant.ofEpochSecond(1462489329L) 28 | val uuid = UUID.randomUUID 29 | val json = ("date" -> timestamp.getEpochSecond.toString) ~ ("ticker" -> 30 | ("buy" -> "2906.58") ~ 31 | ("high" -> "2915.0") ~ 32 | ("last" -> "2906.64") ~ 33 | ("low" -> "2885.6") ~ 34 | ("sell" -> "2906.63") ~ 35 | ("vol" ->"635178.4712")) 36 | val contentType = ContentTypes.`text/html(UTF-8)` 37 | val data = compact(render(json)) 38 | val event = ExchangeEvent(timeCollected, uuid, actor.exchange, data) 39 | 40 | val expected = Tick( 41 | 2906.64, 2906.58, 2906.63, timeCollected, 42 | Some(2915.0), Some(2885.6), None, 43 | volume = Some(635178.4712), 44 | timestamp = Some(timestamp) 45 | ) 46 | 47 | val (pub, sub) = TestSource.probe[ExchangeEvent] 48 | .via(actor.convertFlow[OKCoinPollingDatedTick]) 49 | .via(actor.tickFlow) 50 | .toMat(TestSink.probe[(String, GenericRecord)])(Keep.both) 51 | .run 52 | pub.sendNext(event) 53 | sub.requestNext(("ticks", Tick.format.to(expected))) 54 | } 55 | 56 | it should "process an orderbook message" in { 57 | val timeCollected = Instant.ofEpochSecond(10L) 58 | val uuid = UUID.randomUUID 59 | val json = ("asks" -> List( 60 | List(2915.15, 0.032), 61 | List(2915.14, 1.701), 62 | List(2915.08,0.172) 63 | )) ~ ("bids" -> List( 64 | List(2906.83, 1.912), 65 | List(2906.76, 0.01), 66 | List(2906.75, 0.082) 67 | )) 68 | val data = compact(render(json)) 69 | val event = ExchangeEvent(timeCollected, uuid, actor.exchange, data) 70 | 71 | val bids = List( 72 | Order(2906.83, 1.912), 73 | Order(2906.76, 0.01), 74 | Order(2906.75, 0.082) 75 | ) 76 | val asks = List( 77 | Order(2915.15, 0.032), 78 | Order(2915.14, 1.701), 79 | Order(2915.08, 0.172) 80 | ) 81 | 82 | val expected = OrderBook(bids, asks, timeCollected = Some(timeCollected)) 83 | 84 | val (pub, sub) = TestSource.probe[ExchangeEvent] 85 | .via(actor.convertFlow[OKCoinPollingOrderBook]) 86 | .via(actor.orderbookFlow) 87 | .toMat(TestSink.probe[(String, GenericRecord)])(Keep.both) 88 | .run 89 | pub.sendNext(event) 90 | sub.requestNext(("orderbook", OrderBook.format.to(expected))) 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/test/scala/co/coinsmith/kafka/cryptocoin/polling/BitstampPollingActorSpec.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.polling 2 | 3 | import java.time.Instant 4 | import java.util.UUID 5 | 6 | import akka.actor.ActorSystem 7 | import akka.http.scaladsl.model.ContentTypes 8 | import akka.stream.ActorMaterializer 9 | import akka.stream.scaladsl.Keep 10 | import akka.stream.testkit.scaladsl.{TestSink, TestSource} 11 | import akka.testkit.TestActorRef 12 | import co.coinsmith.kafka.cryptocoin.{ExchangeEvent, Order, OrderBook, Tick} 13 | import org.apache.avro.generic.GenericRecord 14 | import org.json4s.JsonDSL.WithBigDecimal._ 15 | import org.json4s.jackson.JsonMethods._ 16 | 17 | 18 | class BitstampPollingActorSpec 19 | extends HTTPPollingActorSpec(ActorSystem("BitstampPollingActorSpecSystem")) { 20 | implicit val materializer = ActorMaterializer() 21 | 22 | val actorRef = TestActorRef[BitstampPollingActor] 23 | val actor = actorRef.underlyingActor 24 | 25 | "BitstampPollingActor" should "process a ticker message" in { 26 | val timeCollected = Instant.ofEpochSecond(10L) 27 | val timestamp = Instant.ofEpochSecond(1459297128) 28 | val uuid = UUID.randomUUID 29 | val json = ("high" -> "424.37") ~ 30 | ("last" -> "415.24") ~ 31 | ("timestamp" -> timestamp.getEpochSecond.toString) ~ 32 | ("bid" -> "414.34") ~ 33 | ("vwap" -> "415.41") ~ 34 | ("volume" -> "5961.02582305") ~ 35 | ("low" -> "407.22") ~ 36 | ("ask" -> "415.24") ~ 37 | ("open" -> 415.43) 38 | val data = compact(render(json)) 39 | val event = ExchangeEvent(timeCollected, uuid, actor.exchange, data) 40 | 41 | val expected = Tick( 42 | 415.24, 414.34, 415.24, timeCollected, 43 | Some(424.37), Some(407.22), Some(415.43), 44 | Some(5961.02582305), Some(415.41), 45 | timestamp = Some(timestamp) 46 | ) 47 | 48 | val (pub, sub) = TestSource.probe[ExchangeEvent] 49 | .via(actor.convertFlow[BitstampPollingTick]) 50 | .via(actor.tickFlow) 51 | .toMat(TestSink.probe[(String, GenericRecord)])(Keep.both) 52 | .run 53 | pub.sendNext(event) 54 | sub.requestNext(("ticks", Tick.format.to(expected))) 55 | } 56 | 57 | it should "process an orderbook message" in { 58 | val timeCollected = Instant.ofEpochSecond(10L) 59 | val uuid = UUID.randomUUID 60 | val timestamp = 1461605735L 61 | val json = ("timestamp" -> timestamp.toString) ~ 62 | ("bids" -> List( 63 | List("462.49", "0.03010000"), 64 | List("462.48", "4.03000000"), 65 | List("462.47", "16.49799877") 66 | )) ~ ("asks" -> List( 67 | List("462.50", "9.12686646"), 68 | List("462.51", "0.05981955"), 69 | List("462.88", "1.00000000") 70 | )) 71 | val contentType = ContentTypes.`application/json` 72 | val data = compact(render(json)) 73 | val event = ExchangeEvent(timeCollected, uuid, actor.exchange, data) 74 | 75 | val bids = List( 76 | Order("462.49", "0.03010000"), 77 | Order("462.48", "4.03000000"), 78 | Order("462.47", "16.49799877") 79 | ) 80 | val asks = List( 81 | Order("462.50", "9.12686646"), 82 | Order("462.51", "0.05981955"), 83 | Order("462.88", "1.00000000") 84 | ) 85 | val expected = OrderBook(bids, asks, Some(Instant.ofEpochSecond(timestamp)), Some(timeCollected)) 86 | 87 | val (pub, sub) = TestSource.probe[ExchangeEvent] 88 | .via(actor.convertFlow[BitstampPollingOrderBook]) 89 | .via(actor.orderbookFlow) 90 | .toMat(TestSink.probe[(String, GenericRecord)])(Keep.both) 91 | .run 92 | pub.sendNext(event) 93 | sub.requestNext(("orderbook", OrderBook.format.to(expected))) 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/test/scala/co/coinsmith/kafka/cryptocoin/polling/BitfinexPollingActorSpec.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.polling 2 | 3 | import java.time.Instant 4 | import java.util.UUID 5 | 6 | import akka.actor.ActorSystem 7 | import akka.stream.ActorMaterializer 8 | import akka.stream.scaladsl.Keep 9 | import akka.stream.testkit.scaladsl.{TestSink, TestSource} 10 | import akka.testkit.TestActorRef 11 | import co.coinsmith.kafka.cryptocoin.{ExchangeEvent, Order, OrderBook, Tick} 12 | import org.apache.avro.generic.GenericRecord 13 | import org.json4s.JsonDSL.WithBigDecimal._ 14 | import org.json4s.jackson.JsonMethods._ 15 | 16 | 17 | class BitfinexPollingActorSpec 18 | extends HTTPPollingActorSpec(ActorSystem("BitfinexPollingActorSpecSystem")) { 19 | implicit val materializer = ActorMaterializer() 20 | 21 | val actorRef = TestActorRef[BitfinexPollingActor] 22 | val actor = actorRef.underlyingActor 23 | 24 | "BitfinexPollingActor" should "process a ticker message" in { 25 | val timeCollected = Instant.ofEpochSecond(10L) 26 | val timestamp = Instant.ofEpochSecond(1461608354L, 95383854L) 27 | val uuid = UUID.randomUUID 28 | val json = ("mid" -> "464.845") ~ 29 | ("bid" -> "464.8") ~ 30 | ("ask" -> "464.89") ~ 31 | ("last_price" -> "464.9") ~ 32 | ("timestamp" -> "1461608354.095383854") 33 | val data = compact(render(json)) 34 | val event = ExchangeEvent(timeCollected, uuid, actor.exchange, data) 35 | 36 | val expected = Tick(464.9, 464.8, 464.89, timeCollected, timestamp = Some(timestamp)) 37 | 38 | val (pub, sub) = TestSource.probe[ExchangeEvent] 39 | .via(actor.convertFlow[BitfinexPollingTick]) 40 | .via(actor.tickFlow) 41 | .toMat(TestSink.probe[(String, GenericRecord)])(Keep.both) 42 | .run 43 | pub.sendNext(event) 44 | sub.requestNext(("ticks", Tick.format.to(expected))) 45 | } 46 | 47 | it should "process an orderbook message" in { 48 | val timeCollected = Instant.ofEpochSecond(10L) 49 | val uuid = UUID.randomUUID 50 | val json = ("bids" -> List( 51 | ("price" -> "464.11") ~ ("amount" -> "43.98077206") ~ ("timestamp" -> "1461607939.0"), 52 | ("price" -> "463.87") ~ ("amount" -> "21.3389") ~ ("timestamp" -> "1461607927.0"), 53 | ("price" -> "463.86") ~ ("amount" -> "12.5686") ~ ("timestamp" -> "1461607887.0") 54 | )) ~ ("asks" -> List( 55 | ("price" -> "464.12") ~ ("amount" -> "1.457") ~ ("timestamp" -> "1461607308.0"), 56 | ("price" -> "464.49") ~ ("amount" -> "4.07481358") ~ ("timestamp" -> "1461607942.0"), 57 | ("price" -> "464.63") ~ ("amount" -> "4.07481358") ~ ("timestamp" -> "1461607944.0") 58 | )) 59 | val data = compact(render(json)) 60 | val event = ExchangeEvent(timeCollected, uuid, actor.exchange, data) 61 | 62 | val bids = List( 63 | Order(464.11, 43.98077206, timestamp = Some(Instant.ofEpochSecond(1461607939L))), 64 | Order(463.87, 21.3389, timestamp = Some(Instant.ofEpochSecond(1461607927L))), 65 | Order(463.86, 12.5686, timestamp = Some(Instant.ofEpochSecond(1461607887L))) 66 | ) 67 | val asks = List( 68 | Order(464.12, 1.457, timestamp = Some(Instant.ofEpochSecond(1461607308L))), 69 | Order(464.49, 4.07481358, timestamp = Some(Instant.ofEpochSecond(1461607942L))), 70 | Order(464.63, 4.07481358, timestamp = Some(Instant.ofEpochSecond(1461607944L))) 71 | ) 72 | val expected = OrderBook(bids, asks, timeCollected = Some(timeCollected)) 73 | 74 | val (pub, sub) = TestSource.probe[ExchangeEvent] 75 | .via(actor.convertFlow[BitfinexPollingOrderBook]) 76 | .via(actor.orderbookFlow) 77 | .toMat(TestSink.probe[(String, GenericRecord)])(Keep.both) 78 | .run 79 | pub.sendNext(event) 80 | sub.requestNext(("orderbook", OrderBook.format.to(expected))) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/main/scala/co/coinsmith/kafka/cryptocoin/streaming/BitstampStreamingActor.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.streaming 2 | 3 | import java.time.Instant 4 | import java.util.UUID 5 | 6 | import akka.actor.{Actor, ActorLogging, ActorRef, Props} 7 | import akka.stream.ActorMaterializer 8 | import co.coinsmith.kafka.cryptocoin.{Order, OrderBook, ProducerKey, Trade} 9 | import co.coinsmith.kafka.cryptocoin.avro.InstantTypeMaps._ 10 | import co.coinsmith.kafka.cryptocoin.producer.KafkaCryptocoinProducer 11 | import com.pusher.client.Pusher 12 | import com.pusher.client.channel.ChannelEventListener 13 | import com.pusher.client.connection.{ConnectionEventListener, ConnectionState, ConnectionStateChange} 14 | import com.sksamuel.avro4s.RecordFormat 15 | import com.typesafe.config.ConfigFactory 16 | import org.json4s.DefaultFormats 17 | import org.json4s.JsonAST._ 18 | import org.json4s.JsonDSL.WithBigDecimal._ 19 | import org.json4s.jackson.JsonMethods._ 20 | 21 | 22 | case class BitstampStreamingTrade(id: Long, amount: Double, price: Double, 23 | tpe: Int, timestamp: String, 24 | buy_order_id: Long, sell_order_id: Long) 25 | object BitstampStreamingTrade { 26 | implicit def toTrade(trade: BitstampStreamingTrade)(implicit timeCollected: Instant) = 27 | Trade( 28 | trade.price, 29 | trade.amount, 30 | Instant.ofEpochSecond(trade.timestamp.toLong), 31 | timeCollected, 32 | Some(trade.tpe match { 33 | case 0 => "bid" 34 | case 1 => "ask" 35 | }), 36 | Some(trade.id), 37 | Some(trade.buy_order_id), 38 | Some(trade.sell_order_id)) 39 | } 40 | 41 | case class BitstampStreamingOrderBook( 42 | bids: List[List[String]], 43 | asks: List[List[String]], 44 | timestamp: Option[String] 45 | ) 46 | object BitstampStreamingOrderBook { 47 | val toOrder = { o: List[String] => Order(o(0), o(1)) } 48 | 49 | implicit def toOrderBook(ob: BitstampStreamingOrderBook)(implicit timeCollected: Instant) = 50 | OrderBook( 51 | ob.bids map toOrder, 52 | ob.asks map toOrder, 53 | ob.timestamp map { _.toLong } map Instant.ofEpochSecond, 54 | Some(timeCollected) 55 | ) 56 | } 57 | 58 | case class PusherEvent(channelName: String, eventName: String, timestamp: Instant, producerUUID: UUID, msg: String) 59 | object PusherEvent { 60 | val format = RecordFormat[PusherEvent] 61 | } 62 | 63 | class BitstampPusherActor extends Actor with ActorLogging { 64 | var receiver: ActorRef = _ 65 | 66 | val pusher = new Pusher("de504dc5763aeef9ff52") 67 | val connectionEventListener = new ConnectionEventListener { 68 | override def onError(s: String, s1: String, e: Exception) = { 69 | log.error(e, "There was a problem connecting!") 70 | } 71 | 72 | override def onConnectionStateChange(change: ConnectionStateChange) = { 73 | log.info("State changed from " + change.getPreviousState + " to " + change.getCurrentState) 74 | change.getCurrentState match { 75 | case ConnectionState.DISCONNECTED => self ! Connect 76 | case _ => 77 | } 78 | } 79 | } 80 | val channelEventListener = new ChannelEventListener { 81 | override def onSubscriptionSucceeded(channelName: String) = { 82 | log.info("Subscribed to channel {}", channelName) 83 | } 84 | 85 | override def onEvent(channelName: String, eventName: String, data: String) = { 86 | val timeCollected = Instant.now 87 | log.debug("Received event {} on channel {}: {}", channelName, eventName, data) 88 | receiver ! PusherEvent(channelName, eventName, timeCollected, KafkaCryptocoinProducer.uuid, data) 89 | } 90 | } 91 | 92 | pusher.subscribe("live_trades", channelEventListener, "trade") 93 | pusher.subscribe("order_book", channelEventListener, "data") 94 | pusher.subscribe("diff_order_book", channelEventListener, "data") 95 | 96 | def connect: Unit = { 97 | pusher.connect(connectionEventListener, ConnectionState.ALL) 98 | } 99 | 100 | def receive = { 101 | case actor: ActorRef => receiver = actor 102 | case Connect => connect 103 | } 104 | } 105 | 106 | class BitstampPusherProtocol extends Actor { 107 | implicit val formats = DefaultFormats 108 | 109 | def receive = { 110 | case pe: PusherEvent => self forward (pe.channelName, pe.eventName, pe.timestamp, parse(pe.msg)) 111 | case ("live_trades", "trade", t: Instant, json: JValue) => 112 | implicit val timeCollected = t 113 | // some json processing required due to 'type' as a key name 114 | val trade = (json transformField{ 115 | case ("type", v) => ("tpe", v) 116 | }).extract[BitstampStreamingTrade] 117 | sender ! ("trades", Trade.format.to(trade)) 118 | case ("order_book", "data", t: Instant, json: JValue) => 119 | implicit val timeCollected = t 120 | val ob = json.extract[BitstampStreamingOrderBook] 121 | sender ! ("orderbook", OrderBook.format.to(ob)) 122 | case ("diff_order_book", "data", t: Instant, json: JValue) => 123 | implicit val timeCollected = t 124 | val diff = json.extract[BitstampStreamingOrderBook] 125 | sender ! ("orderbook.updates", OrderBook.format.to(diff)) 126 | } 127 | } 128 | 129 | class BitstampStreamingActor extends Actor with ActorLogging { 130 | implicit val actorSystem = context.system 131 | implicit val materializer = ActorMaterializer() 132 | 133 | val topicPrefix = "bitstamp.streaming.btcusd." 134 | 135 | val conf = ConfigFactory.load 136 | val preprocess = conf.getBoolean("kafka.cryptocoin.preprocess") 137 | 138 | val pusher = context.actorOf(Props[BitstampPusherActor]) 139 | val protocol = context.actorOf(Props[BitstampPusherProtocol]) 140 | 141 | override def preStart = { 142 | pusher ! self 143 | pusher ! Connect 144 | } 145 | 146 | def receive = { 147 | case (topic: String, value: Object) => 148 | KafkaCryptocoinProducer.send(topicPrefix + topic, value) 149 | case pe : PusherEvent => 150 | val key = ProducerKey(KafkaCryptocoinProducer.uuid, "bitstamp") 151 | KafkaCryptocoinProducer.send("streaming.pusher.raw", ProducerKey.format.to(key), PusherEvent.format.to(pe)) 152 | 153 | if (preprocess) { 154 | protocol ! pe 155 | } 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /src/main/scala/co/coinsmith/kafka/cryptocoin/streaming/OKCoinStreamingActor.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.streaming 2 | 3 | import scala.concurrent.duration._ 4 | import java.net.URI 5 | import java.time._ 6 | 7 | import akka.actor.{Actor, ActorLogging, Props} 8 | import akka.http.scaladsl.model.ws.TextMessage 9 | import akka.stream.ActorMaterializer 10 | import co.coinsmith.kafka.cryptocoin.producer.KafkaCryptocoinProducer 11 | import co.coinsmith.kafka.cryptocoin._ 12 | import com.typesafe.config.ConfigFactory 13 | import org.json4s.DefaultFormats 14 | import org.json4s.JsonAST._ 15 | import org.json4s.JsonDSL.WithBigDecimal._ 16 | import org.json4s.jackson.JsonMethods._ 17 | 18 | 19 | case class Data(timeCollected: Instant, channel: String, data: JValue) 20 | 21 | case class OKCoinStreamingTick(buy: String, high: String, last: String, low: String, 22 | sell: String, timestamp: String, vol: String) 23 | object OKCoinStreamingTick { 24 | implicit def toTick(tick: OKCoinStreamingTick)(implicit timeCollected: Instant) = 25 | Tick( 26 | tick.last.toDouble, tick.buy.toDouble, tick.sell.toDouble, timeCollected, 27 | Some(tick.high.toDouble), Some(tick.low.toDouble), 28 | volume = Some(tick.vol.replace(",", "").toDouble), 29 | timestamp = Some(Instant.ofEpochMilli(tick.timestamp.toLong)) 30 | ) 31 | } 32 | 33 | case class OKCoinStreamingOrderBook(bids: List[List[Double]], asks: List[List[Double]], timestamp: String) 34 | object OKCoinStreamingOrderBook { 35 | val toOrder = { o: List[Double] => Order(o(0), o(1)) } 36 | 37 | implicit def toOrderBook(ob: OKCoinStreamingOrderBook)(implicit timeCollected: Instant) = 38 | OrderBook( 39 | ob.bids map toOrder, ob.asks map toOrder, 40 | Some(Instant.ofEpochMilli(ob.timestamp.toLong)), Some(timeCollected) 41 | ) 42 | } 43 | 44 | class OKCoinWebsocketProtocol extends Actor with ActorLogging { 45 | implicit val formats = DefaultFormats 46 | 47 | val conf = ConfigFactory.load 48 | val preprocess = conf.getBoolean("kafka.cryptocoin.preprocess") 49 | 50 | def adjustTimestamp(timeCollected: Instant, time: String) = { 51 | val zone = ZoneId.of("Asia/Shanghai") 52 | val collectedZoned = ZonedDateTime.ofInstant(timeCollected, ZoneOffset.UTC) 53 | .withZoneSameInstant(zone) 54 | var tradeZoned = LocalTime.parse(time).atDate(collectedZoned.toLocalDate).atZone(zone) 55 | if ((tradeZoned compareTo collectedZoned) > 0) { 56 | // correct date if time collected happens right after midnight 57 | tradeZoned = tradeZoned minusDays 1 58 | } 59 | 60 | tradeZoned.withZoneSameInstant(ZoneOffset.UTC).toInstant 61 | } 62 | 63 | def toTrade(trade: List[String])(implicit timeCollected: Instant) = 64 | Trade( 65 | trade(1).toDouble, 66 | trade(2).toDouble, 67 | adjustTimestamp(timeCollected, trade(3)), 68 | timeCollected, 69 | Some(trade(4)), 70 | Some(trade(0).toLong) 71 | ) 72 | 73 | def receive = { 74 | case (t: Instant, events: JArray) => 75 | // OKCoin websocket responses are an array of multiple events 76 | events match { 77 | case JArray(arr) => arr.foreach { event => self forward (t, event) } 78 | case _ => new Exception("Message did not contain array.") 79 | } 80 | case (t: Instant, JObject(JField("channel", JString(channel)) :: 81 | JField("success", JString(success)) :: Nil)) => 82 | log.info("Added channel {} at time {}.", channel, t) 83 | 84 | case (t: Instant, JObject(JField("channel", JString(channel)) :: 85 | JField("success", JString(success)) :: 86 | JField("error_code", JInt(errorCode)) :: Nil)) => 87 | log.error("Adding channel {} failed at time {}. Error code {}.", channel, t, errorCode) 88 | 89 | case (t: Instant, JObject(JField("channel", JString(channel)) :: JField("data", data) :: Nil)) 90 | if preprocess => 91 | self forward Data(t, channel, data) 92 | 93 | case Data(t, "ok_sub_spotcny_btc_ticker", data) => 94 | implicit val timeCollected = t 95 | val tick = data.extract[OKCoinStreamingTick] 96 | sender ! ("ticks", Tick.format.to(tick)) 97 | 98 | case Data(t, "ok_sub_spotcny_btc_depth_60", data) => 99 | implicit val timeCollected = t 100 | val ob = data.extract[OKCoinStreamingOrderBook] 101 | sender ! ("orderbook", OrderBook.format.to(ob)) 102 | 103 | case Data(t, "ok_sub_spotcny_btc_trades", data: JArray) => 104 | implicit val timeCollected = t 105 | val trades = data.extract[List[List[String]]] map toTrade 106 | trades foreach { trade => 107 | sender ! ("trades", Trade.format.to(trade)) 108 | } 109 | } 110 | } 111 | 112 | class OKCoinStreamingActor extends Actor with ActorLogging { 113 | implicit val actorSystem = context.system 114 | implicit val materializer = ActorMaterializer() 115 | implicit val dispatcher = actorSystem.dispatcher 116 | 117 | val topicPrefix = "okcoin.streaming.btcusd." 118 | val uri = new URI("wss://real.okcoin.cn:10440/websocket/okcoinapi") 119 | 120 | val channels = List( 121 | ("event" -> "addChannel") ~ ("channel" -> "ok_sub_spotcny_btc_ticker"), 122 | ("event" -> "addChannel") ~ ("channel" -> "ok_sub_spotcny_btc_depth_60"), 123 | ("event" -> "addChannel") ~ ("channel" -> "ok_sub_spotcny_btc_trades") 124 | ) 125 | val messages = List(TextMessage(compact(render(channels)))) 126 | 127 | val websocket = new AkkaWebsocket(uri, messages, self) 128 | val protocol = context.actorOf(Props[OKCoinWebsocketProtocol]) 129 | websocket.connect 130 | 131 | val pingEvent = ("event" -> "ping") 132 | actorSystem.scheduler.schedule(30 seconds, 30 seconds) { 133 | val msg = TextMessage(compact(render(pingEvent))) 134 | websocket.send(msg) 135 | } 136 | 137 | def receive = { 138 | case (topic: String, value: Object) => 139 | KafkaCryptocoinProducer.send(topicPrefix + topic, value) 140 | case (t: Instant, msg: String) => 141 | val exchange = "okcoin" 142 | val key = ProducerKey(KafkaCryptocoinProducer.uuid, exchange) 143 | val event = ExchangeEvent(t, KafkaCryptocoinProducer.uuid, exchange, msg) 144 | KafkaCryptocoinProducer.send("streaming.websocket.raw", ProducerKey.format.to(key), ExchangeEvent.format.to(event)) 145 | 146 | protocol ! (t, parse(msg)) 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /src/main/scala/co/coinsmith/kafka/cryptocoin/streaming/BitfinexStreamingActor.scala: -------------------------------------------------------------------------------- 1 | package co.coinsmith.kafka.cryptocoin.streaming 2 | 3 | import scala.math.BigInt 4 | import java.net.URI 5 | import java.time.Instant 6 | 7 | import akka.actor.SupervisorStrategy.Escalate 8 | import akka.actor.{Actor, ActorLogging, AllForOneStrategy, Props} 9 | import akka.http.scaladsl.model.ws.TextMessage 10 | import akka.stream.ActorMaterializer 11 | import co.coinsmith.kafka.cryptocoin._ 12 | import co.coinsmith.kafka.cryptocoin.producer.KafkaCryptocoinProducer 13 | import com.typesafe.config.ConfigFactory 14 | import org.json4s.DefaultFormats 15 | import org.json4s.JsonAST._ 16 | import org.json4s.JsonDSL.WithBigDecimal._ 17 | import org.json4s.jackson.JsonMethods._ 18 | 19 | 20 | case class Unsubscribed(channel: String) 21 | 22 | class BitfinexWebsocketProtocol extends Actor with ActorLogging { 23 | implicit val formats = DefaultFormats 24 | 25 | val conf = ConfigFactory.load 26 | val preprocess = conf.getBoolean("kafka.cryptocoin.preprocess") 27 | 28 | var subscribed = Map.empty[BigInt, String] 29 | def getChannelName(channelId: BigInt) = subscribed(channelId) 30 | 31 | def isEvent(event: JValue): Boolean = event.findField { 32 | case ("event", eventName: JString) => true 33 | case _ => false 34 | } match { 35 | case None => false 36 | case _ => true 37 | } 38 | 39 | def toTick(arr: List[Double])(implicit timeCollected: Instant) = 40 | Tick( 41 | arr(6), arr(0), arr(2), timeCollected, 42 | Some(arr(8)), Some(arr(9)), None, 43 | Some(arr(7)), None, 44 | Some(arr(1)), Some(arr(3)), 45 | Some(arr(4)), Some(arr(5)) 46 | ) 47 | 48 | def toOrder(order: List[Double])(implicit timeCollected: Instant) = 49 | Order(order(1), order(2), Some(order(0).toLong), Some(timeCollected)) 50 | 51 | def toDouble(v: JValue) = v match { 52 | case JInt(i) => i.toDouble 53 | case JDouble(d) => d 54 | case _ => throw new Exception(s"Could not convert to double: $v") 55 | } 56 | 57 | // price and volume can be JInt or JDouble 58 | def toTrade(trade: JValue)(implicit timeCollected: Instant) = trade match { 59 | case JArray(JString(seq) :: JInt(id) :: JInt(timestamp) :: price :: volume :: Nil) => 60 | Trade(toDouble(price), toDouble(volume), Instant.ofEpochSecond(timestamp.toLong), timeCollected, tid = Some(id.toLong), seq = Some(seq)) 61 | case JArray(JString(seq) :: JInt(timestamp) :: price :: volume :: Nil) => 62 | Trade(toDouble(price), toDouble(volume), Instant.ofEpochSecond(timestamp.toLong), timeCollected, seq = Some(seq)) 63 | case JArray(JInt(id) :: JInt(timestamp) :: price :: volume :: Nil) => 64 | Trade(toDouble(price), toDouble(volume), Instant.ofEpochSecond(timestamp.toLong), timeCollected, tid = Some(id.toLong)) 65 | case _ => throw new Exception(s"Trade snapshot processing error for $trade") 66 | } 67 | 68 | def handleInfoEvent(code: Int, msg: String) = code match { 69 | case 20060 => log.warning(msg) 70 | case 20061 => 71 | log.warning(msg) 72 | for (id <- subscribed.keys) { 73 | log.warning("Sending unsubscribe request for channel {}", getChannelName(id),id) 74 | sender ! ("event" -> "unsubscribe") ~ ("chanId" -> id) 75 | } 76 | } 77 | 78 | def handleErrorEvent(code: Int, msg: String) = code match { 79 | case 10400 => log.error("During unsubscription: {}", msg) 80 | } 81 | 82 | def receive = { 83 | case (t, JObject(JField("event", JString("subscribed")) :: 84 | JField("channel", JString(channelName)) :: 85 | JField("chanId", JInt(channelId)) :: 86 | xs)) => 87 | log.info("Received subscription event response for channel {} with ID {}", channelName, channelId) 88 | subscribed += (channelId -> channelName) 89 | case (t, JObject(JField("event", JString("info")) :: 90 | JField("code", JInt(code)) :: 91 | JField("msg", JString(msg)) :: Nil)) => handleInfoEvent(code.toInt, msg) 92 | case (t, JObject(JField("event", JString("unsubscribed")) :: 93 | JField("status", JString("OK")) :: 94 | JField("chanId", JInt(channelId)) :: Nil)) => 95 | val channelName = getChannelName(channelId) 96 | log.warning("Unsubscribe response for channel {}: OK", getChannelName(channelId)) 97 | sender ! Unsubscribed(channelName) 98 | subscribed = subscribed - channelId 99 | case (t, JObject(JField("event", JString("error")) :: 100 | JField("code", JInt(code)) :: 101 | JField("msg", JString(msg)) :: Nil)) => handleErrorEvent(code.toInt, msg) 102 | case (t, event: JValue) if isEvent(event) => 103 | log.info("Received event message: {}", compact(render(event))) 104 | case (t: Instant, JArray(JInt(channelId) :: JString("hb") :: Nil)) => 105 | log.debug("Received heartbeat message for channel ID {}", channelId) 106 | case (t: Instant, JArray(JInt(channelId) :: JArray(data) :: Nil)) if preprocess => 107 | implicit val timeCollected = t 108 | getChannelName(channelId) match { 109 | case "book" => 110 | val orders = JArray(data).extract[List[List[Double]]] map toOrder 111 | val ob = OrderBook( 112 | orders filter { _.volume > 0 }, 113 | orders filter { _.volume < 0 }, 114 | timeCollected = Some(t) 115 | ) 116 | sender ! ("orderbook.snapshots", OrderBook.format.to(ob)) 117 | case "trades" => data map toTrade foreach { trade => 118 | sender ! ("trades.snapshots", Trade.format.to(trade)) 119 | } 120 | } 121 | case (t: Instant, JArray(JInt(channelId) :: JString(updateType) :: xs)) if preprocess => 122 | implicit val timeCollected = t 123 | val topic = updateType match { 124 | case "tu" => "trades" 125 | case "te" => "trades.executions" 126 | } 127 | val trade = toTrade(JArray(xs)) 128 | sender ! (topic, Trade.format.to(trade)) 129 | case (t: Instant, JArray(JInt(channelId) :: xs)) if preprocess => 130 | implicit val timeCollected = t 131 | getChannelName(channelId) match { 132 | case "book" => 133 | val order = toOrder(JArray(xs).extract[List[Double]]) 134 | sender ! ("orderbook", Order.format.to(order)) 135 | case "ticker" => 136 | val tick = toTick(JArray(xs).extract[List[Double]]) 137 | sender ! ("ticker", Tick.format.to(tick)) 138 | } 139 | } 140 | } 141 | 142 | class BitfinexStreamingActor extends Actor with ActorLogging { 143 | implicit val actorSystem = context.system 144 | implicit val materializer = ActorMaterializer() 145 | 146 | val topicPrefix = "bitfinex.streaming.btcusd." 147 | val uri = new URI("wss://api2.bitfinex.com:3000/ws") 148 | 149 | override val supervisorStrategy = AllForOneStrategy() { 150 | case _: Exception => Escalate 151 | case t => super.supervisorStrategy.decider.applyOrElse(t, (_: Any) => Escalate) 152 | } 153 | 154 | val channelSubscriptionMessages = Map( 155 | "book" -> ("event" -> "subscribe") ~ ("channel" -> "book") ~ ("pair" -> "BTCUSD") 156 | ~ ("prec" -> "R0") ~ ("len" -> "100"), 157 | "trades" -> ("event" -> "subscribe") ~ ("channel" -> "trades") ~ ("pair" -> "BTCUSD"), 158 | "ticker" -> ("event" -> "subscribe") ~ ("channel" -> "ticker") ~ ("pair" -> "BTCUSD") 159 | ).mapValues(j => compact(render(j))).mapValues(s => TextMessage(s)) 160 | 161 | val websocket = new AkkaWebsocket(uri, channelSubscriptionMessages.values.toList, self) 162 | val protocol = context.actorOf(Props[BitfinexWebsocketProtocol]) 163 | websocket.connect 164 | 165 | def receive = { 166 | case json: JValue => 167 | websocket.send(TextMessage(compact(render(json)))) 168 | case e: Unsubscribed => 169 | websocket.send(channelSubscriptionMessages(e.channel)) 170 | case (topic: String, value: Object) => 171 | KafkaCryptocoinProducer.send(topicPrefix + topic, value) 172 | case (t: Instant, msg: String) => 173 | val exchange = "bitfinex" 174 | val key = ProducerKey(KafkaCryptocoinProducer.uuid, exchange) 175 | val event = ExchangeEvent(t, KafkaCryptocoinProducer.uuid, exchange, msg) 176 | KafkaCryptocoinProducer.send("streaming.websocket.raw", ProducerKey.format.to(key), ExchangeEvent.format.to(event)) 177 | 178 | protocol ! (t, parse(msg)) 179 | } 180 | } 181 | --------------------------------------------------------------------------------