├── project ├── build.properties ├── project │ └── plugins.sbt ├── Aliases.scala ├── plugins.sbt ├── Deps.scala └── Settings.scala ├── version.sbt ├── .gitignore ├── core └── src │ ├── main │ ├── scala │ │ └── jupyter │ │ │ └── kernel │ │ │ ├── interpreter │ │ │ ├── InterpreterKernel.scala │ │ │ ├── DisplayData.scala │ │ │ ├── Interpreter.scala │ │ │ └── InterpreterHandler.scala │ │ │ ├── stream │ │ │ ├── Streams.scala │ │ │ └── ZMQStreams.scala │ │ │ ├── Message.scala │ │ │ ├── protocol │ │ │ ├── HMAC.scala │ │ │ ├── Enumerate.scala │ │ │ ├── ParsedMessage.scala │ │ │ ├── OptimizedPrinter.scala │ │ │ └── Formats.scala │ │ │ ├── KernelSpecs.scala │ │ │ └── server │ │ │ ├── Server.scala │ │ │ ├── InterpreterServer.scala │ │ │ └── ServerApp.scala │ └── scala-2.10 │ │ └── com │ │ └── typesafe │ │ └── scalalogging │ │ └── package.scala │ └── test │ └── scala │ └── jupyter │ └── kernel │ └── interpreter │ ├── JsonTests.scala │ ├── DisplayDataTests.scala │ ├── InterpreterHandlerTests.scala │ └── Helpers.scala ├── protocol └── src │ └── main │ └── scala │ └── jupyter │ └── kernel │ └── protocol │ ├── Kernel.scala │ ├── Header.scala │ ├── Protocol.scala │ ├── StdinReply.scala │ ├── StdinRequest.scala │ ├── Connection.scala │ ├── Channel.scala │ ├── Comm.scala │ ├── Publish.scala │ ├── ShellRequest.scala │ └── ShellReply.scala ├── api └── src │ └── main │ └── scala │ └── jupyter │ └── api │ ├── Publish.scala │ ├── Comm.scala │ ├── Display.scala │ └── internals │ └── Base64.scala ├── .travis.yml └── README.md /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.15 2 | -------------------------------------------------------------------------------- /version.sbt: -------------------------------------------------------------------------------- 1 | version in ThisBuild := "0.4.2-SNAPSHOT" 2 | -------------------------------------------------------------------------------- /project/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-RC1") 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | target-2.10/ 3 | /.idea/ 4 | /.idea_modules/ 5 | logs/ 6 | notebook/notebooks/ 7 | /notebooks 8 | -------------------------------------------------------------------------------- /core/src/main/scala/jupyter/kernel/interpreter/InterpreterKernel.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.interpreter 2 | 3 | trait InterpreterKernel { 4 | def apply(): Interpreter 5 | } 6 | -------------------------------------------------------------------------------- /project/Aliases.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import sbt.Keys._ 3 | 4 | object Aliases { 5 | 6 | def libs = libraryDependencies 7 | 8 | def root = file(".") 9 | 10 | } 11 | -------------------------------------------------------------------------------- /core/src/main/scala-2.10/com/typesafe/scalalogging/package.scala: -------------------------------------------------------------------------------- 1 | package com.typesafe 2 | 3 | package object scalalogging { 4 | type LazyLogging = com.typesafe.scalalogging.slf4j.LazyLogging 5 | } 6 | -------------------------------------------------------------------------------- /protocol/src/main/scala/jupyter/kernel/protocol/Kernel.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.protocol 2 | 3 | final case class Kernel( 4 | argv: List[String], 5 | display_name: String, 6 | language: String 7 | ) 8 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") 2 | addSbtPlugin("org.xerial.sbt" % "sbt-pack" % "0.8.0") 3 | addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-RC1") 4 | addSbtPlugin("com.dwijnand" % "sbt-travisci" % "1.1.0") 5 | -------------------------------------------------------------------------------- /protocol/src/main/scala/jupyter/kernel/protocol/Header.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.protocol 2 | 3 | final case class Header( 4 | msg_id: String, 5 | username: String, 6 | session: String, 7 | msg_type: String, 8 | version: Option[String] 9 | ) 10 | -------------------------------------------------------------------------------- /protocol/src/main/scala/jupyter/kernel/protocol/Protocol.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.protocol 2 | 3 | object Protocol { 4 | val versionMajor = 5 5 | val versionMinor = 0 6 | 7 | val versionStrOpt: Option[String] = Some(s"$versionMajor.$versionMinor") 8 | } 9 | -------------------------------------------------------------------------------- /protocol/src/main/scala/jupyter/kernel/protocol/StdinReply.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.protocol 2 | 3 | sealed abstract class StdinReply extends Product with Serializable 4 | 5 | object StdinReply { 6 | 7 | final case class Input( 8 | value: String 9 | ) extends StdinReply 10 | 11 | } 12 | -------------------------------------------------------------------------------- /protocol/src/main/scala/jupyter/kernel/protocol/StdinRequest.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.protocol 2 | 3 | sealed abstract class StdinRequest extends Product with Serializable 4 | 5 | object StdinRequest { 6 | 7 | final case class Input( 8 | prompt: String, 9 | password: Boolean 10 | ) extends StdinRequest 11 | 12 | } 13 | -------------------------------------------------------------------------------- /protocol/src/main/scala/jupyter/kernel/protocol/Connection.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.protocol 2 | 3 | final case class Connection( 4 | ip: String, 5 | transport: String, 6 | stdin_port: Int, 7 | control_port: Int, 8 | hb_port: Int, 9 | shell_port: Int, 10 | iopub_port: Int, 11 | key: String, 12 | signature_scheme: Option[String] 13 | ) 14 | -------------------------------------------------------------------------------- /core/src/main/scala/jupyter/kernel/stream/Streams.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel 2 | package stream 3 | 4 | import jupyter.kernel.protocol.Channel 5 | 6 | import scalaz.concurrent.Task 7 | import scalaz.stream.{ Process, Sink } 8 | 9 | final case class Streams( 10 | processes: Channel => (Process[Task, Either[String, Message]], Sink[Task, Message]), 11 | stop: () => Unit 12 | ) 13 | -------------------------------------------------------------------------------- /protocol/src/main/scala/jupyter/kernel/protocol/Channel.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.protocol 2 | 3 | sealed abstract class Channel extends Product with Serializable 4 | 5 | object Channel { 6 | case object Requests extends Channel 7 | case object Control extends Channel 8 | case object Publish extends Channel 9 | case object Input extends Channel 10 | 11 | val channels = Seq(Requests, Control, Publish, Input) 12 | } 13 | -------------------------------------------------------------------------------- /api/src/main/scala/jupyter/api/Publish.scala: -------------------------------------------------------------------------------- 1 | package jupyter.api 2 | 3 | import java.util.UUID 4 | 5 | trait Publish extends Display { 6 | def stdout(text: String): Unit 7 | def stderr(text: String): Unit 8 | 9 | /** Opens a communication channel server -> client */ 10 | def comm(id: String = UUID.randomUUID().toString): Comm 11 | 12 | /** Registers a client -> server message handler */ 13 | def commHandler(target: String)(handler: CommChannelMessage => Unit): Unit 14 | } 15 | -------------------------------------------------------------------------------- /project/Deps.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import sbt.Keys._ 3 | 4 | object Deps { 5 | 6 | def argonaut = "io.argonaut" %% "argonaut" % "6.2" 7 | def argonautShapeless = "com.github.alexarchambault" %% "argonaut-shapeless_6.2" % "1.2.0-M5" 8 | def jeromq = "org.zeromq" % "jeromq" % "0.3.6" 9 | def macroParadise = "org.scalamacros" % "paradise" % "2.1.0" cross CrossVersion.full 10 | def scalaLogging = Def.setting { 11 | scalaBinaryVersion.value match { 12 | case "2.10" => 13 | "com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2" 14 | case _ => 15 | "com.typesafe.scala-logging" %% "scala-logging" % "3.5.0" 16 | } 17 | } 18 | def scalazStream = "org.scalaz.stream" %% "scalaz-stream" % "0.8.6a" 19 | def utest = "com.lihaoyi" %% "utest" % "0.4.4" 20 | 21 | } 22 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | scala: 3 | - 2.10.6 4 | - 2.11.11 5 | - 2.12.2 6 | jdk: 7 | - oraclejdk8 8 | script: sbt ++${TRAVIS_SCALA_VERSION} test $(if [[ "${TRAVIS_PULL_REQUEST}" == 9 | "false" && "${TRAVIS_BRANCH}" == "master" ]]; then echo "publish"; fi) 10 | sudo: false 11 | cache: 12 | directories: 13 | - $HOME/.coursier 14 | - $HOME/.ivy2/cache 15 | - $HOME/.sbt 16 | env: 17 | global: 18 | - secure: PEpYuctOXRqzaZ2knhj6PmKT732uFXjjDqnzNHk5WOIxIWqO7huI3Ewz90prdr94I2Umz5A6DARgAKFobp4liyom93xU86mPD2Kcszqo7xCD/HtCZ0/XWMWp8we40+K1gfT9tBzP0H4glPLRjfK6LRtc6dvJeRYaEa1kiqdq5gs= 19 | - secure: c7OeQKpSD9Tf5JYLpbtP32n+S2R5RA/zisl6Y114Aeh/00XM6/7eE7fDtvdg0t07/nI7BguyJK7qxg8/n9k0P4GZoB6KNxdYhnEYzbPHfhycDmaZn9hv0LQ4FazRjFuL5kIvklia5RVWp9Fw8zL7b9SHnjCxdIkR9TZ+q4MaI+4= 20 | branches: 21 | only: 22 | - master 23 | -------------------------------------------------------------------------------- /api/src/main/scala/jupyter/api/Comm.scala: -------------------------------------------------------------------------------- 1 | package jupyter.api 2 | 3 | sealed abstract class CommChannelMessage extends Product with Serializable 4 | 5 | final case class CommOpen(target: String, data: String) extends CommChannelMessage 6 | final case class CommMessage(data: String) extends CommChannelMessage 7 | final case class CommClose(data: String) extends CommChannelMessage 8 | 9 | trait Comm { 10 | def id: String 11 | 12 | def send(msg: CommChannelMessage): Unit 13 | 14 | final def open(target: String, data: String): Unit = 15 | send(CommOpen(target, data)) 16 | final def message(data: String): Unit = 17 | send(CommMessage(data)) 18 | final def close(data: String): Unit = 19 | send(CommClose(data)) 20 | 21 | def onMessage(f: CommChannelMessage => Unit): Unit 22 | def onSentMessage(f: CommChannelMessage => Unit): Unit 23 | } 24 | -------------------------------------------------------------------------------- /core/src/main/scala/jupyter/kernel/interpreter/DisplayData.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.interpreter 2 | 3 | import argonaut.{Json, Parse} 4 | 5 | final case class DisplayData(mimeType: String, data: String) { 6 | 7 | private def hasJsonMimeType: Boolean = 8 | mimeType == "application/json" || 9 | (mimeType.startsWith("application/") && mimeType.endsWith("+json")) 10 | 11 | def jsonField: (String, Json) = { 12 | 13 | def asString = Json.jString(data) 14 | def asJsonOption = Parse.parse(data).right.toOption 15 | 16 | val json = 17 | if (hasJsonMimeType) 18 | asJsonOption.getOrElse(asString) 19 | else 20 | asString 21 | 22 | mimeType -> json 23 | } 24 | } 25 | 26 | object DisplayData { 27 | 28 | def text(text: String): DisplayData = 29 | DisplayData("text/plain", text) 30 | 31 | val empty: DisplayData = text("") 32 | } 33 | -------------------------------------------------------------------------------- /protocol/src/main/scala/jupyter/kernel/protocol/Comm.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.protocol 2 | 3 | import argonaut.Json 4 | 5 | sealed abstract class Comm extends Product with Serializable { 6 | def comm_id: String 7 | def data: Json 8 | } 9 | 10 | object Comm { 11 | 12 | // Spec says: If the target_name key is not found on the receiving side, then it should immediately reply with a comm_close message to avoid an inconsistent state. 13 | final case class Open( 14 | comm_id: String, 15 | target_name: String, 16 | data: Json, 17 | target_module: Option[String] = None // spec says: used to select a module that is responsible for handling the target_name. 18 | ) extends Comm 19 | 20 | final case class Message( 21 | comm_id: String, 22 | data: Json 23 | ) extends Comm 24 | 25 | final case class Close( 26 | comm_id: String, 27 | data: Json 28 | ) extends Comm 29 | 30 | } 31 | -------------------------------------------------------------------------------- /core/src/main/scala/jupyter/kernel/Message.scala: -------------------------------------------------------------------------------- 1 | package jupyter 2 | package kernel 3 | 4 | import argonaut._, Argonaut._ 5 | import protocol.{ Header, ParsedMessage } 6 | import protocol.Formats.decodeHeader 7 | 8 | final case class Message( 9 | idents: List[Seq[Byte]], 10 | header: String, 11 | parentHeader: String, 12 | metaData: String, 13 | content: String 14 | ) { 15 | 16 | def msgType: Either[String, String] = header.decodeEither[Header].right.map(_.msg_type) 17 | 18 | def decodeAs[T: DecodeJson]: Either[String, ParsedMessage[T]] = 19 | for { 20 | header <- header.decodeEither[Header].right 21 | metaData <- metaData.decodeEither[Map[String, String]].right 22 | content <- content.decodeEither[T].right 23 | } yield { 24 | val parentHeaderOpt = parentHeader.decodeEither[Header].right.toOption 25 | ParsedMessage(idents, header, parentHeaderOpt, metaData, content) 26 | } 27 | 28 | class AsHelper[T] { 29 | def apply[U](f: ParsedMessage[T] => U)(implicit decodeJson: DecodeJson[T]): Either[String, U] = 30 | decodeAs[T].right.map(f) 31 | } 32 | 33 | def as[T] = new AsHelper[T] 34 | 35 | } 36 | -------------------------------------------------------------------------------- /core/src/test/scala/jupyter/kernel/interpreter/JsonTests.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.interpreter 2 | 3 | import jupyter.kernel.protocol.Formats._ 4 | import argonaut._, Argonaut._, ArgonautShapeless._ 5 | import jupyter.kernel.protocol.ShellReply 6 | import utest._ 7 | 8 | object JsonTests extends TestSuite { 9 | 10 | val tests = TestSuite { 11 | 'reply - { 12 | 'statusField - { 13 | val okReply = ShellReply.Execute(3, Map.empty) 14 | val errorReply = ShellReply.Error("name", "value", List("t1", "t2")) 15 | val abortReply = ShellReply.Abort() 16 | 17 | def statusOf(json: Json): Option[String] = { 18 | final case class WithStatus(status: String) 19 | json.asJson.as[WithStatus].toOption.map(_.status) 20 | } 21 | 22 | val okStatusOpt = statusOf(okReply.asJson) 23 | val errorStatusOpt = statusOf(errorReply.asJson) 24 | val abortStatusOpt = statusOf(abortReply.asJson) 25 | 26 | assert(okStatusOpt == Some("ok")) 27 | assert(errorStatusOpt == Some("error")) 28 | assert(abortStatusOpt == Some("abort")) 29 | } 30 | } 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /core/src/main/scala/jupyter/kernel/protocol/HMAC.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.protocol 2 | 3 | import javax.crypto.Mac 4 | import javax.crypto.spec.SecretKeySpec 5 | 6 | // Adapted from the implementation of IScala 7 | 8 | sealed abstract class HMAC { 9 | def apply(args: String*): String 10 | } 11 | 12 | object HMAC { 13 | 14 | private def instance(f: Seq[String] => String): HMAC = 15 | new HMAC { 16 | def apply(args: String*) = f(args) 17 | } 18 | 19 | private val empty = instance(_ => "") 20 | 21 | def apply(key: String, algorithm: Option[String] = None): HMAC = 22 | if (key.isEmpty) 23 | empty 24 | else { 25 | val algorithm0 = algorithm.getOrElse("hmac-sha256").replace("-", "") 26 | val mac = Mac.getInstance(algorithm0) 27 | val keySpec = new SecretKeySpec(key.getBytes("UTF-8"), algorithm0) 28 | 29 | mac.init(keySpec) 30 | 31 | def hex(bytes: Seq[Byte]) = bytes.map(s => f"$s%02x").mkString 32 | 33 | instance { args => 34 | mac.synchronized { 35 | for (s <- args) 36 | mac.update(s.getBytes("UTF-8")) 37 | 38 | hex(mac.doFinal()) 39 | } 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Jupyter kernel 2 | 3 | *Helper library to write [Jupyter / IPython](https://jupyter.org/) kernels on the JVM - Scala agnostic parts of [jupyter-scala](https://github.com/alexarchambault/jupyter-scala.git)* 4 | 5 | [![Build Status](https://travis-ci.org/alexarchambault/jupyter-kernel.svg?branch=master)](https://travis-ci.org/alexarchambault/jupyter-kernel) 6 | [![Maven Central](https://img.shields.io/maven-central/v/org.jupyter-scala/kernel_2.11.svg)](https://maven-badges.herokuapp.com/maven-central/org.jupyter-scala/kernel_2.11) 7 | [![Scaladoc](https://javadoc-badge.appspot.com/org.jupyter-scala/kernel_2.11.svg?label=scaladoc)](https://javadoc-badge.appspot.com/org.jupyter-scala/kernel_2.11) 8 | 9 | `jupyter-kernel` is a library that helps writing kernels for 10 | [Jupyter / IPython](https://jupyter.org/) on the JVM. 11 | It mainly targets Scala for now, but could be used for other 12 | languages as well. 13 | It tries to implement the `5.0` version of the IPython 14 | [messaging protocol](https://jupyter-client.readthedocs.io/en/latest/messaging.html). 15 | 16 | ### Limitations 17 | 18 | It currently has a poor test coverage, although that point should be easy to address. 19 | 20 | ### Notice 21 | 22 | Released under the LGPL version 3 license. 23 | -------------------------------------------------------------------------------- /api/src/main/scala/jupyter/api/Display.scala: -------------------------------------------------------------------------------- 1 | package jupyter.api 2 | 3 | import internals.Base64._ 4 | 5 | trait Display { 6 | 7 | def display(items: (String, String)*): Unit 8 | 9 | final def html(html: String): Unit = 10 | display("text/html" -> html) 11 | final def markdown(md: String): Unit = 12 | display("text/markdown" -> md) 13 | final def md(md: String): Unit = 14 | markdown(md) 15 | final def svg(svg: String): Unit = 16 | display("image/svg+xml" -> svg) 17 | final def png(data: Array[Byte]): Unit = 18 | display("image/png" -> data.toBase64) 19 | final def png(data: java.awt.image.BufferedImage): Unit = { 20 | val baos = new java.io.ByteArrayOutputStream 21 | javax.imageio.ImageIO.write(data, "png", baos) 22 | png(baos.toByteArray) 23 | } 24 | final def jpg(data: Array[Byte]): Unit = 25 | display("image/jpeg" -> data.toBase64) 26 | final def jpg(data: java.awt.image.BufferedImage): Unit = { 27 | val baos = new java.io.ByteArrayOutputStream 28 | javax.imageio.ImageIO.write(data, "jpg", baos) 29 | jpg(baos.toByteArray) 30 | } 31 | final def latex(latex: String): Unit = 32 | display("text/latex" -> latex) 33 | final def pdf(data: Array[Byte]): Unit = 34 | display("application/pdf" -> data.toBase64) 35 | final def javascript(code: String): Unit = 36 | display("application/javascript" -> code) 37 | final def js(code: String): Unit = 38 | javascript(code) 39 | } -------------------------------------------------------------------------------- /core/src/main/scala/jupyter/kernel/protocol/Enumerate.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.protocol 2 | 3 | import shapeless.{ :+:, ::, CNil, Coproduct, Generic, HList, HNil, Inl, Inr, Strict } 4 | 5 | sealed abstract class Enumerate[T] { 6 | def apply(): Seq[T] 7 | } 8 | 9 | object Enumerate { 10 | def apply[T](implicit enum: Enumerate[T]): Enumerate[T] = enum 11 | 12 | private def instance[T](values: => Seq[T]): Enumerate[T] = 13 | new Enumerate[T] { 14 | def apply() = values 15 | } 16 | 17 | implicit val boolean: Enumerate[Boolean] = 18 | instance(Seq(true, false)) 19 | 20 | implicit val hnil: Enumerate[HNil] = 21 | instance(Seq(HNil)) 22 | implicit def hcons[H, T <: HList] 23 | (implicit 24 | head: Strict[Enumerate[H]], 25 | tail: Enumerate[T] 26 | ): Enumerate[H :: T] = 27 | instance { 28 | for { 29 | h <- head.value() 30 | t <- tail() 31 | } yield h :: t 32 | } 33 | 34 | implicit val cnil: Enumerate[CNil] = 35 | instance(Seq()) 36 | implicit def ccons[H, T <: Coproduct] 37 | (implicit 38 | head: Strict[Enumerate[H]], 39 | tail: Enumerate[T] 40 | ): Enumerate[H :+: T] = 41 | instance(head.value().map(Inl(_)) ++ tail().map(Inr(_))) 42 | 43 | implicit def generic[F, G] 44 | (implicit 45 | gen: Generic.Aux[F, G], 46 | underlying: Strict[Enumerate[G]] 47 | ): Enumerate[F] = 48 | instance(underlying.value().map(gen.from)) 49 | } 50 | -------------------------------------------------------------------------------- /core/src/main/scala/jupyter/kernel/KernelSpecs.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel 2 | 3 | import java.io.File 4 | 5 | object KernelSpecs { 6 | 7 | private lazy val isWindows = 8 | sys.props 9 | .get("os.name") 10 | .exists(_.startsWith("Windows")) 11 | 12 | private lazy val isOSX = 13 | sys.props 14 | .get("os.name") 15 | .toSeq 16 | .contains("Mac OS X") 17 | 18 | lazy val homeDirOption = 19 | sys.props 20 | .get("user.home") 21 | .filter(_.nonEmpty) 22 | .orElse( 23 | sys.env 24 | .get("HOME") 25 | .filter(_.nonEmpty) 26 | ) 27 | 28 | def userKernelSpecDirectory: Option[File] = 29 | if (isWindows) 30 | sys.env 31 | .get("APPDATA") 32 | .map(_ + "/jupyter/kernels") 33 | .map(new File(_)) 34 | else 35 | homeDirOption.map { homeDir => 36 | val path = 37 | if (isOSX) 38 | "Library/Jupyter/kernels" 39 | else 40 | ".local/share/jupyter/kernels" 41 | 42 | new File(homeDir + "/" + path) 43 | } 44 | 45 | def systemKernelSpecDirectories: Seq[File] = { 46 | 47 | val paths = 48 | if (isWindows) 49 | sys.env 50 | .get("PROGRAMDATA") 51 | .map(_ + "/jupyter/kernels") 52 | .toSeq 53 | else 54 | Seq( 55 | "/usr/share/jupyter/kernels", 56 | "/usr/local/share/jupyter/kernels" 57 | ) 58 | 59 | paths.map(new File(_)) 60 | } 61 | 62 | } -------------------------------------------------------------------------------- /protocol/src/main/scala/jupyter/kernel/protocol/Publish.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.protocol 2 | 3 | import argonaut.Json 4 | 5 | sealed abstract class Publish extends Product with Serializable 6 | 7 | object Publish { 8 | 9 | final case class Stream( 10 | name: String, 11 | text: String 12 | ) extends Publish 13 | 14 | final case class DisplayData( 15 | data: Map[String, Json], // keys are always string, except if key is "application/json" 16 | metadata: Map[String, Json] 17 | ) extends Publish 18 | 19 | final case class ExecuteInput( 20 | code: String, 21 | execution_count: Int 22 | ) extends Publish 23 | 24 | final case class ExecuteResult( 25 | execution_count: Int, 26 | data: Map[String, Json], // same as DisplayData 27 | metadata: Map[String, Json] 28 | ) extends Publish 29 | 30 | final case class Error( 31 | ename: String, 32 | evalue: String, 33 | traceback: List[String] 34 | ) extends Publish 35 | 36 | // Spec says: Busy and idle messages should be sent before/after handling every message, not just execution. 37 | final case class Status( 38 | execution_state: ExecutionState0 39 | ) extends Publish 40 | 41 | sealed abstract class ExecutionState0(val label: String) extends Product with Serializable 42 | 43 | object ExecutionState0 { 44 | case object Busy extends ExecutionState0("busy") 45 | case object Idle extends ExecutionState0("idle") 46 | case object Starting extends ExecutionState0("starting") 47 | } 48 | 49 | final case class ClearOutput( 50 | wait0: Boolean // beware: just "wait" in the spec, but "wait" can't be used as an identifier here 51 | ) extends Publish 52 | 53 | } 54 | -------------------------------------------------------------------------------- /core/src/main/scala/jupyter/kernel/protocol/ParsedMessage.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.protocol 2 | 3 | import argonaut._, Argonaut.{ EitherDecodeJson => _, EitherEncodeJson => _, _ } 4 | 5 | import java.util.UUID 6 | 7 | import jupyter.kernel.Message 8 | import jupyter.kernel.protocol.Formats.{ decodeHeader, encodeHeader } 9 | 10 | final case class ParsedMessage[Content]( 11 | idents: List[Seq[Byte]], 12 | header: Header, 13 | parent_header: Option[Header], 14 | metadata: Map[String, String], 15 | content: Content 16 | ) { 17 | 18 | private def replyHeader(msgType: String): Header = 19 | header.copy(msg_id = UUID.randomUUID().toString, msg_type = msgType) 20 | 21 | private def replyMsg[ReplyContent: EncodeJson]( 22 | idents: List[Seq[Byte]], 23 | msgType: String, 24 | content: ReplyContent, 25 | metadata: Map[String, String] 26 | ): Message = 27 | ParsedMessage(idents, replyHeader(msgType), Some(header), metadata, content).toMessage 28 | 29 | def publish[PubContent: EncodeJson]( 30 | msgType: String, 31 | content: PubContent, 32 | metadata: Map[String, String] = Map.empty, 33 | ident: String = null 34 | ): Message = 35 | replyMsg(List(Option(ident).getOrElse(msgType).getBytes("UTF-8")), msgType, content, metadata) 36 | 37 | def reply[ReplyContent: EncodeJson]( 38 | msgType: String, 39 | content: ReplyContent, 40 | metadata: Map[String, String] = Map.empty 41 | ): Message = 42 | replyMsg(idents, msgType, content, metadata) 43 | 44 | def toMessage(implicit encode: EncodeJson[Content]): Message = 45 | Message( 46 | idents, 47 | OptimizedPrinter.noSpaces(header.asJson), 48 | parent_header.fold("{}")(x => OptimizedPrinter.noSpaces(x.asJson)), 49 | OptimizedPrinter.noSpaces(metadata.asJson), 50 | OptimizedPrinter.noSpaces(content.asJson) 51 | ) 52 | } -------------------------------------------------------------------------------- /core/src/test/scala/jupyter/kernel/interpreter/DisplayDataTests.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.interpreter 2 | 3 | import argonaut.{Json, Parse} 4 | import utest._ 5 | 6 | object DisplayDataTests extends TestSuite { 7 | val tests = TestSuite { 8 | 9 | 'textplain - { 10 | val data = DisplayData("text/plain", "foo") 11 | val expectedField = "text/plain" -> Json.jString("foo") 12 | val field = data.jsonField 13 | 14 | assert(field == expectedField) 15 | } 16 | 17 | 'applicationjson - { 18 | 19 | val someJson = Json.obj( 20 | "a" -> Json.jNumber(2), 21 | "b" -> Json.jBool(true), 22 | "c" -> Json.obj( 23 | "list" -> Json.array((1 to 3).map(Json.jNumber): _*) 24 | ) 25 | ) 26 | val someJsonStr = someJson.nospaces 27 | 28 | 'default - { 29 | val mimeType = "application/json" 30 | 31 | val data = DisplayData(mimeType, someJsonStr) 32 | val expectedField = mimeType -> someJson 33 | val field = data.jsonField 34 | 35 | assert(field == expectedField) 36 | } 37 | 38 | 'suffix - { 39 | val mimeType = "application/foo.bar+json" 40 | 41 | val data = DisplayData(mimeType, someJsonStr) 42 | val expectedField = mimeType -> someJson 43 | val field = data.jsonField 44 | 45 | assert(field == expectedField) 46 | } 47 | 48 | 'stringFallback - { 49 | val mimeType = "application/json" 50 | 51 | val value = someJsonStr.dropRight(2) 52 | assert(Parse.parse(value).isLeft) // value should be invalid JSON 53 | 54 | val data = DisplayData(mimeType, value) 55 | val expectedField = mimeType -> Json.jString(value) 56 | val field = data.jsonField 57 | 58 | assert(field == expectedField) 59 | } 60 | } 61 | 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /core/src/main/scala/jupyter/kernel/interpreter/Interpreter.scala: -------------------------------------------------------------------------------- 1 | package jupyter 2 | package kernel.interpreter 3 | 4 | import argonaut.Json 5 | import jupyter.api.Publish 6 | import jupyter.kernel.protocol.{ ParsedMessage, ShellReply } 7 | 8 | trait Interpreter { 9 | def init(): Unit = {} 10 | def initialized: Boolean = true 11 | def publish(publish: ParsedMessage[_] => Publish): Unit = {} 12 | 13 | def interpret( 14 | line: String, 15 | output: Option[(String => Unit, String => Unit)], 16 | storeHistory: Boolean, 17 | current: Option[ParsedMessage[_]] 18 | ): Interpreter.Result 19 | def isComplete(code: String): Option[Interpreter.IsComplete] = 20 | None 21 | def complete(code: String, pos: Int): (Int, Int, Seq[String]) = 22 | (pos, pos, Nil) 23 | def executionCount: Int 24 | 25 | def languageInfo: ShellReply.KernelInfo.LanguageInfo 26 | def implementation = ("", "") 27 | def banner = "" 28 | def resultDisplay = false 29 | 30 | def helpLinks: Seq[(String, String)] = Nil 31 | } 32 | 33 | object Interpreter { 34 | 35 | sealed abstract class IsComplete extends Product with Serializable 36 | 37 | object IsComplete { 38 | case object Complete extends IsComplete 39 | final case class Incomplete(indent: String) extends IsComplete 40 | case object Invalid extends IsComplete 41 | } 42 | 43 | sealed abstract class Result extends Product with Serializable 44 | sealed abstract class Success extends Result 45 | sealed abstract class Failure extends Result 46 | 47 | final case class Value(data: Seq[DisplayData]) extends Success { 48 | 49 | lazy val jsonMap: Map[String, Json] = 50 | data.map(_.jsonField).toMap 51 | } 52 | 53 | case object NoValue extends Success 54 | 55 | final case class Exception( 56 | name: String, 57 | msg: String, 58 | stackTrace: List[String] 59 | ) extends Failure { 60 | def traceBack = s"$name: $msg" :: stackTrace.map(" " + _) 61 | } 62 | 63 | final case class Error(message: String) extends Failure 64 | 65 | case object Cancelled extends Failure 66 | } 67 | -------------------------------------------------------------------------------- /core/src/test/scala/jupyter/kernel/interpreter/InterpreterHandlerTests.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.interpreter 2 | 3 | import argonaut.Json 4 | 5 | import jupyter.kernel.Message 6 | import jupyter.kernel.protocol._ 7 | import jupyter.kernel.protocol.Formats._ 8 | 9 | import utest._ 10 | 11 | object InterpreterHandlerTests extends TestSuite { 12 | import Helpers._ 13 | 14 | val connectReply = randomConnectReply() 15 | def session(msgs: (Req[_], Seq[Reply])*) = Helpers.session(echoInterpreter(), Some("5.0"), connectReply)(msgs: _*) 16 | 17 | val tests = TestSuite{ 18 | 'malformedHeader{ 19 | val response = InterpreterHandler(echoInterpreter(), randomConnectReply(), (_, _) => (), Message(Nil, "{", "", "{}", "{}")) 20 | assertMatch(response) { 21 | case Left(_) => 22 | } 23 | } 24 | 25 | 'emptyRequest{ 26 | // FIXME Provide a *real* header 27 | val response = InterpreterHandler(echoInterpreter(), randomConnectReply(), (_, _) => (), Message(Nil, "{}", "", "{}", "{}")) 28 | assertMatch(response) { 29 | case Left(_) => 30 | } 31 | } 32 | 33 | 'connectReply{ 34 | session( 35 | Req("connect_request", ShellRequest.Connect) -> Seq( 36 | ReqReply("connect_reply", connectReply) 37 | ) 38 | ) 39 | } 40 | 41 | 'kernelInfo{ 42 | session( 43 | Req("kernel_info_request", ShellRequest.KernelInfo) -> Seq( 44 | ReqReply("kernel_info_reply", ShellReply.KernelInfo("5.0", "", "", echoInterpreter().languageInfo, "")) 45 | ) 46 | ) 47 | } 48 | 49 | 'execute{ 50 | session( 51 | Req("execute_request", ShellRequest.Execute("meh", Map.empty, silent = Some(false), None, allow_stdin = Some(false), None)) -> Seq( 52 | PubReply(List("execute_input"), "execute_input", Publish.ExecuteInput("meh", 1)), 53 | PubReply(List("status"), "status", Publish.Status(Publish.ExecutionState0.Busy)), 54 | PubReply(List("execute_result"), "execute_result", Publish.ExecuteResult(1, Map("text/plain" -> Json.jString("meh")), Map.empty)), 55 | ReqReply("execute_reply", ShellReply.Execute(1, Map.empty)), 56 | PubReply(List("status"), "status", Publish.Status(Publish.ExecutionState0.Idle)) 57 | ) 58 | ) 59 | } 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /project/Settings.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import sbt.Keys._ 3 | 4 | import Aliases._ 5 | 6 | object Settings { 7 | 8 | lazy val shared = Seq( 9 | organization := "org.jupyter-scala", 10 | scalacOptions ++= Seq("-deprecation", "-unchecked", "-feature"), 11 | resolvers ++= Seq( 12 | Resolver.sonatypeRepo("releases"), 13 | "Scalaz Bintray Repo" at "https://dl.bintray.com/scalaz/releases" 14 | ), 15 | libs ++= { 16 | if (scalaBinaryVersion.value == "2.10") 17 | Seq(compilerPlugin(Deps.macroParadise)) 18 | else 19 | Nil 20 | } 21 | ) ++ publishSettings 22 | 23 | lazy val testSettings = Seq( 24 | libs += Deps.utest % "test", 25 | testFrameworks += new TestFramework("utest.runner.Framework") 26 | ) 27 | 28 | lazy val publishSettings = Seq( 29 | publishMavenStyle := true, 30 | publishTo := { 31 | val nexus = "https://oss.sonatype.org/" 32 | if (isSnapshot.value) 33 | Some("snapshots" at nexus + "content/repositories/snapshots") 34 | else 35 | Some("releases" at nexus + "service/local/staging/deploy/maven2") 36 | }, 37 | licenses := Seq("LGPL-3.0" -> url("http://www.gnu.org/licenses/lgpl.txt")), 38 | scmInfo := Some(ScmInfo( 39 | url("https://github.com/alexarchambault/jupyter-kernel"), 40 | "git@github.com:alexarchambault/jupyter-kernel.git" 41 | )), 42 | homepage := Some(url("https://github.com/alexarchambault/jupyter-kernel")), 43 | pomExtra := { 44 | 45 | 46 | alexarchambault 47 | Alexandre Archambault 48 | https://github.com/alexarchambault 49 | 50 | 51 | }, 52 | credentials ++= { 53 | Seq("SONATYPE_USER", "SONATYPE_PASS").map(sys.env.get) match { 54 | case Seq(Some(user), Some(pass)) => 55 | Seq(Credentials("Sonatype Nexus Repository Manager", "oss.sonatype.org", user, pass)) 56 | case _ => 57 | Nil 58 | } 59 | }, 60 | scalacOptions ++= { 61 | scalaBinaryVersion.value match { 62 | case "2.10" | "2.11" => 63 | Seq("-target:jvm-1.7") 64 | case _ => 65 | Nil 66 | } 67 | } 68 | ) 69 | 70 | lazy val dontPublish = Seq( 71 | publish := (), 72 | publishLocal := (), 73 | publishArtifact := false 74 | ) 75 | 76 | lazy val kernelPrefix = { 77 | name := "kernel-" + name.value 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /api/src/main/scala/jupyter/api/internals/Base64.scala: -------------------------------------------------------------------------------- 1 | package jupyter.api.internals 2 | 3 | import scala.collection.immutable.HashMap 4 | 5 | /** 6 | * Base64 encoder 7 | * @author Mark Lister 8 | * This software is distributed under the 2-Clause BSD license. See the 9 | * LICENSE file in the root of the repository. 10 | * 11 | * Copyright (c) 2014 - 2015 Mark Lister 12 | * 13 | * The repo for this Base64 encoder lives at https://github.com/marklister/base64 14 | * Please send your issues, suggestions and pull requests there. 15 | * 16 | */ 17 | 18 | private[api] object Base64 { 19 | private[this] val zero = Array(0, 0).map(_.toByte) 20 | class B64Scheme(val encodeTable: IndexedSeq[Char]) { 21 | lazy val decodeTable = HashMap(encodeTable.zipWithIndex: _ *) 22 | } 23 | 24 | lazy val base64 = new B64Scheme(('A' to 'Z') ++ ('a' to 'z') ++ ('0' to '9') ++ Seq('+', '/')) 25 | lazy val base64Url = new B64Scheme(base64.encodeTable.dropRight(2) ++ Seq('-', '_')) 26 | 27 | implicit class Encoder(b: Array[Byte]) { 28 | 29 | lazy val pad = (3 - b.length % 3) % 3 30 | 31 | def toBase64(implicit scheme: B64Scheme = base64): String = { 32 | def sixBits(x: Array[Byte]): Array[Int] = { 33 | val a = (x(0) & 0xfc) >> 2 34 | val b = ((x(0) & 0x3) << 4) | ((x(1) & 0xf0) >> 4) 35 | val c = ((x(1) & 0xf) << 2) | ((x(2) & 0xc0) >> 6) 36 | val d = (x(2)) & 0x3f 37 | Array(a, b, c, d) 38 | } 39 | ((b ++ zero.take(pad)).grouped(3) 40 | .flatMap(sixBits) 41 | .map(scheme.encodeTable) 42 | .toArray 43 | .dropRight(pad) :+ "=" * pad) 44 | .mkString 45 | } 46 | } 47 | 48 | implicit class Decoder(s: String) { 49 | lazy val cleanS = s.reverse.dropWhile(_ == '=').reverse 50 | lazy val pad = s.length - cleanS.length 51 | 52 | def toByteArray(implicit scheme: B64Scheme = base64): Array[Byte] = { 53 | def threeBytes(s: String): Array[Byte] = { 54 | val r = s.map(scheme.decodeTable(_)).foldLeft(0)((a, b) => (a << 6) | b) 55 | Array((r >> 16).toByte, (r >> 8).toByte, r.toByte) 56 | } 57 | if (pad > 2 || s.length % 4 != 0) throw new java.lang.IllegalArgumentException("Invalid Base64 String:" + s) 58 | try { 59 | (cleanS + "A" * pad) 60 | .grouped(4) 61 | .map(threeBytes) 62 | .flatten 63 | .toArray 64 | .dropRight(pad) 65 | } catch {case e:NoSuchElementException => throw new java.lang.IllegalArgumentException("Invalid Base64 String:" + s) } 66 | } 67 | } 68 | 69 | } -------------------------------------------------------------------------------- /protocol/src/main/scala/jupyter/kernel/protocol/ShellRequest.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.protocol 2 | 3 | sealed abstract class ShellRequest extends Product with Serializable 4 | 5 | object ShellRequest { 6 | 7 | final case class Execute( 8 | code: String, 9 | user_expressions: Map[String, String], 10 | silent: Option[Boolean] = None, 11 | store_history: Option[Boolean] = None, 12 | allow_stdin: Option[Boolean] = None, 13 | stop_on_error: Option[Boolean] = None 14 | ) extends ShellRequest 15 | 16 | final case class Inspect( 17 | code: String, 18 | cursor_pos: Int, 19 | detail_level: Int // 0 or 1 - enforce with refined, or a custom ADT? 20 | ) extends ShellRequest 21 | 22 | final case class Complete( 23 | code: String, 24 | cursor_pos: Int 25 | ) extends ShellRequest 26 | 27 | sealed abstract class History extends Product with Serializable { 28 | def output: Boolean 29 | def raw: Boolean 30 | def hist_access_type: History.AccessType 31 | } 32 | 33 | object History { 34 | 35 | sealed abstract class AccessType extends Product with Serializable 36 | object AccessType { 37 | case object Range extends AccessType 38 | case object Tail extends AccessType 39 | case object Search extends AccessType 40 | 41 | // required for the type class derivation to be fine in 2.10 42 | type Range = Range.type 43 | type Tail = Tail.type 44 | type Search = Search.type 45 | } 46 | 47 | final case class Range( 48 | output: Boolean, 49 | raw: Boolean, 50 | session: Int, // range specific 51 | start: Int, // range specific 52 | stop: Int, // range specific 53 | hist_access_type: AccessType.Range 54 | ) extends History 55 | 56 | object Range { 57 | def apply( 58 | output: Boolean, 59 | raw: Boolean, 60 | session: Int, // range specific 61 | start: Int, // range specific 62 | stop: Int 63 | ): Range = 64 | Range( 65 | output, 66 | raw, 67 | session, 68 | start, 69 | stop, 70 | AccessType.Range 71 | ) 72 | } 73 | 74 | final case class Tail( 75 | output: Boolean, 76 | raw: Boolean, 77 | n: Int, // tail and search specific 78 | hist_access_type: AccessType.Tail 79 | ) extends History 80 | 81 | object Tail { 82 | def apply( 83 | output: Boolean, 84 | raw: Boolean, 85 | n: Int 86 | ): Tail = 87 | Tail( 88 | output, 89 | raw, 90 | n, 91 | AccessType.Tail 92 | ) 93 | } 94 | 95 | final case class Search( 96 | output: Boolean, 97 | raw: Boolean, 98 | n: Int, // search specific 99 | pattern: String, // search specific 100 | unique: Option[Boolean] = None, // search specific 101 | hist_access_type: AccessType.Search 102 | ) 103 | 104 | object Search { 105 | def apply( 106 | output: Boolean, 107 | raw: Boolean, 108 | n: Int, 109 | pattern: String, 110 | unique: Option[Boolean] 111 | ): Search = 112 | Search( 113 | output, 114 | raw, 115 | n, 116 | pattern, 117 | unique, 118 | AccessType.Search 119 | ) 120 | 121 | def apply( 122 | output: Boolean, 123 | raw: Boolean, 124 | n: Int, 125 | pattern: String 126 | ): Search = 127 | Search( 128 | output, 129 | raw, 130 | n, 131 | pattern, 132 | None 133 | ) 134 | } 135 | 136 | } 137 | 138 | final case class IsComplete( 139 | code: String 140 | ) extends ShellRequest 141 | 142 | case object Connect extends ShellRequest 143 | 144 | final case class CommInfo( 145 | target_name: Option[String] = None 146 | ) extends ShellRequest 147 | 148 | case object KernelInfo extends ShellRequest 149 | 150 | final case class Shutdown( 151 | restart: Boolean 152 | ) extends ShellRequest 153 | 154 | } 155 | -------------------------------------------------------------------------------- /core/src/main/scala/jupyter/kernel/server/Server.scala: -------------------------------------------------------------------------------- 1 | package jupyter 2 | package kernel 3 | package server 4 | 5 | import java.io.File 6 | import java.nio.file.Files 7 | import java.util.UUID 8 | import java.lang.management.ManagementFactory 9 | import java.net.{InetAddress, ServerSocket} 10 | import java.util.concurrent.ExecutorService 11 | 12 | import argonaut._ 13 | import Argonaut._ 14 | import com.typesafe.scalalogging.LazyLogging 15 | import jupyter.kernel.stream.Streams 16 | import jupyter.kernel.stream.ZMQStreams 17 | import jupyter.kernel.protocol.{Connection, Formats, ShellReply} 18 | import jupyter.kernel.interpreter.InterpreterKernel 19 | import Formats.{ encodeConnection, decodeConnection } 20 | 21 | import scalaz.concurrent.Task 22 | 23 | object Server extends LazyLogging { 24 | 25 | final case class Options( 26 | connectionFile: String = "", 27 | eraseConnectionFile: Boolean = false, 28 | quiet: Boolean = false 29 | ) 30 | 31 | def newConnectionFile(connFile: File): Connection = { 32 | def randomPort(): Int = { 33 | val s = new ServerSocket(0) 34 | try s.getLocalPort 35 | finally s.close() 36 | } 37 | 38 | val ip = { 39 | val s = InetAddress.getLocalHost.toString 40 | val idx = s.lastIndexOf('/') 41 | if (idx < 0) 42 | s 43 | else 44 | s.substring(idx + 1) 45 | } 46 | 47 | val c = Connection( 48 | ip = ip, 49 | transport = "tcp", 50 | stdin_port = randomPort(), 51 | control_port = randomPort(), 52 | hb_port = randomPort(), 53 | shell_port = randomPort(), 54 | iopub_port = randomPort(), 55 | key = UUID.randomUUID().toString, 56 | signature_scheme = Some("hmac-sha256") 57 | ) 58 | 59 | Files.write(connFile.toPath, c.asJson.spaces2.getBytes) // default charset 60 | 61 | c 62 | } 63 | 64 | private def pid() = ManagementFactory.getRuntimeMXBean.getName.takeWhile(_ != '@').toInt 65 | 66 | def launch( 67 | kernel: InterpreterKernel, 68 | streams: Streams, 69 | connection: Connection, 70 | classLoader: Option[ClassLoader] 71 | )(implicit es: ExecutorService): Task[Unit] = 72 | InterpreterServer( 73 | streams, 74 | ShellReply.Connect( 75 | shell_port = connection.shell_port, 76 | iopub_port = connection.iopub_port, 77 | stdin_port = connection.stdin_port, 78 | hb_port = connection.hb_port 79 | ), 80 | kernel() 81 | ) 82 | 83 | def apply( 84 | kernel: InterpreterKernel, 85 | kernelId: String, 86 | options: Server.Options = Server.Options(), 87 | classLoaderOption: Option[ClassLoader] = None 88 | )(implicit es: ExecutorService): (File, Task[Unit]) = { 89 | 90 | val homeDir = KernelSpecs.homeDirOption.getOrElse { 91 | throw new Exception("Cannot get user home dir, set one in the HOME environment variable") 92 | } 93 | 94 | val connFile = 95 | Some(options.connectionFile).filter(_.nonEmpty).getOrElse(s"jupyter-kernel_${pid()}.json") match { 96 | case path if path.contains(File.separatorChar) => 97 | new File(path) 98 | case secure => 99 | new File(homeDir, s".ipython/profile_default/secure/$secure") 100 | } 101 | 102 | logger.info(s"Connection file: ${connFile.getAbsolutePath}") 103 | 104 | val connection = 105 | if (options.eraseConnectionFile || !connFile.exists()) { 106 | logger.info(s"Creating ipython connection file ${connFile.getAbsolutePath}") 107 | connFile.getParentFile.mkdirs() 108 | newConnectionFile(connFile) 109 | } else 110 | new String(Files.readAllBytes(connFile.toPath), "UTF-8").decodeEither[Connection] match { 111 | case Left(err) => 112 | logger.error(s"Loading connection file: $err") 113 | throw new Exception(s"Error while loading connection file: $err") 114 | case Right(c) => 115 | c 116 | } 117 | 118 | val streams = ZMQStreams(connection, identity = Some(kernelId)) 119 | 120 | if (!options.quiet) 121 | Console.err.println("Launching kernel") 122 | 123 | val t = launch(kernel, streams, connection, classLoaderOption) 124 | 125 | (connFile, t) 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /core/src/test/scala/jupyter/kernel/interpreter/Helpers.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.interpreter 2 | 3 | import java.util.concurrent.Executors 4 | 5 | import jupyter.kernel.Message 6 | import jupyter.kernel.protocol.{Publish => _, _} 7 | import java.util.UUID 8 | 9 | import jupyter.kernel.interpreter.Interpreter._ 10 | 11 | import scala.util.Random.{nextInt => randomInt} 12 | import argonaut._, Argonaut._ 13 | import utest._ 14 | 15 | import scalaz.concurrent.Strategy 16 | 17 | object Helpers { 18 | 19 | implicit val es = Executors.newCachedThreadPool( 20 | Strategy.DefaultDaemonThreadFactory 21 | ) 22 | 23 | def echoInterpreter(): Interpreter = new Interpreter { 24 | def interpret(line: String, output: Option[((String) => Unit, (String) => Unit)], storeHistory: Boolean, current: Option[ParsedMessage[_]]) = 25 | if (line.isEmpty) Error("incomplete") 26 | else { 27 | if (storeHistory) executionCount += 1 28 | if (line startsWith "error:") Error(line stripPrefix "error:") else Value(Seq(DisplayData.text(line))) 29 | } 30 | var executionCount = 0 31 | val languageInfo = ShellReply.KernelInfo.LanguageInfo("echo", "0.1", "text/x-echo", ".echo", "", Some("x-echo"), None) 32 | } 33 | 34 | def randomConnectReply() = ShellReply.Connect(randomInt(), randomInt(), randomInt(), randomInt()) 35 | 36 | def assertCmp[T](resp: Seq[T], exp: Seq[T], pos: Int = 0): Unit = { 37 | (resp.toList, exp.toList) match { 38 | case (Nil, Nil) => 39 | case (r :: rt, e :: et) => 40 | try assert(r == e) 41 | catch { case ex: utest.AssertionError => 42 | println(s"At pos $pos:") 43 | println(s"Response: $r") 44 | println(s"Expected: $e") 45 | throw ex 46 | } 47 | 48 | assertCmp(rt, et, pos + 1) 49 | 50 | case (Nil, e :: et) => 51 | println(s"At pos $pos, missing $e") 52 | assert(false) 53 | 54 | case (l @ (r :: rt), Nil) => 55 | println(s"At pos $pos, extra item(s) $l") 56 | assert(false) 57 | } 58 | } 59 | 60 | final case class Req[T: EncodeJson](msgType: String, t: T) { 61 | def apply(idents: List[Seq[Byte]], userName: String, sessionId: String, version: Option[String]): (Message, Header) = { 62 | val msg = ParsedMessage( 63 | idents, Header(UUID.randomUUID().toString, userName, sessionId, msgType, version), None, Map.empty, 64 | t 65 | ) 66 | 67 | (msg.toMessage, msg.header) 68 | } 69 | } 70 | 71 | sealed abstract class Reply extends Product with Serializable { 72 | def apply(defaultIdents: List[Seq[Byte]], userName: String, sessionId: String, replyId: String, parHdr: Header, version: Option[String]): (Channel, ParsedMessage[Json]) 73 | } 74 | final case class ReqReply[T: EncodeJson](msgType: String, t: T) extends Reply { 75 | def apply(defaultIdents: List[Seq[Byte]], userName: String, sessionId: String, replyId: String, parHdr: Header, version: Option[String]) = 76 | Channel.Requests -> ParsedMessage(defaultIdents, Header(replyId, userName, sessionId, msgType, version), Some(parHdr), Map.empty, t.asJson) 77 | } 78 | final case class PubReply[T: EncodeJson](idents: List[String], msgType: String, t: T) extends Reply { 79 | def apply(defaultIdents: List[Seq[Byte]], userName: String, sessionId: String, replyId: String, parHdr: Header, version: Option[String]) = 80 | Channel.Publish -> ParsedMessage(idents.map(_.getBytes("UTF-8").toSeq), Header(replyId, userName, sessionId, msgType, version), Some(parHdr), Map.empty, t.asJson) 81 | } 82 | 83 | implicit class ParsedMessageOps[T](m: ParsedMessage[T]) { 84 | def eraseMsgId: ParsedMessage[T] = 85 | m.copy( 86 | header = m.header.copy( 87 | msg_id = "" 88 | ) 89 | ) 90 | } 91 | 92 | def session(intp: Interpreter, version: Option[String], connectReply: ShellReply.Connect)(msgs: (Req[_], Seq[Reply])*) = { 93 | val idents = List.empty[Seq[Byte]] 94 | val userName = "user" 95 | val sessionId = UUID.randomUUID().toString 96 | val commonId = UUID.randomUUID().toString 97 | for (((req, replies), idx) <- msgs.zipWithIndex) { 98 | val (msg, msgHdr) = req(idents, userName, sessionId, version) 99 | val expected = replies.map(_(idents, userName, sessionId, commonId, msgHdr, version)).map { case (c, m) => c -> Right(m.eraseMsgId) } 100 | val response = InterpreterHandler(intp, connectReply, (_, _) => (), msg).right.map(_.runLog.unsafePerformSync.map { case (c, m) => c -> m.decodeAs[Json].right.map(_.eraseMsgId) }) 101 | assert(response.isRight) 102 | assertCmp(response.right.get, expected) 103 | } 104 | } 105 | 106 | } 107 | -------------------------------------------------------------------------------- /core/src/main/scala/jupyter/kernel/stream/ZMQStreams.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel 2 | package stream 3 | 4 | import java.util.concurrent.ExecutorService 5 | import java.util.concurrent.atomic.AtomicBoolean 6 | 7 | import com.typesafe.scalalogging.LazyLogging 8 | 9 | import jupyter.kernel.protocol.{ Channel, Connection, HMAC } 10 | 11 | import org.zeromq.ZMQ 12 | import org.zeromq.ZMQ.{ Poller, PollItem } 13 | 14 | import scalaz.concurrent.Task 15 | import scalaz.stream.{ Process, Sink } 16 | 17 | object ZMQStreams extends LazyLogging { 18 | 19 | private val delimiter = "" 20 | private val delimiterBytes: Seq[Byte] = delimiter.getBytes("UTF-8") 21 | private val pollingDelay = 1000L 22 | 23 | def apply( 24 | connection: Connection, 25 | identity: Option[String] 26 | )(implicit 27 | pool: ExecutorService 28 | ): Streams = { 29 | val ctx = ZMQ.context(1) 30 | 31 | val requests = ctx.socket(ZMQ.ROUTER) 32 | val control = ctx.socket(ZMQ.ROUTER) 33 | val publish = ctx.socket(ZMQ.PUB) 34 | val stdin = ctx.socket(ZMQ.ROUTER) 35 | val heartbeat = ctx.socket(ZMQ.REP) 36 | 37 | def uri(port: Int) = s"${connection.transport}://${connection.ip}:$port" 38 | 39 | for (id <- identity) { 40 | val b = id.getBytes("UTF-8") 41 | requests.setIdentity(b) 42 | control.setIdentity(b) 43 | stdin.setIdentity(b) 44 | } 45 | 46 | requests.setLinger(1000L) 47 | control.setLinger(1000L) 48 | publish.setLinger(1000L) 49 | stdin.setLinger(1000L) 50 | heartbeat.setLinger(1000L) 51 | 52 | requests.bind(uri(connection.shell_port)) 53 | control.bind(uri(connection.control_port)) 54 | publish.bind(uri(connection.iopub_port)) 55 | stdin.bind(uri(connection.stdin_port)) 56 | heartbeat.bind(uri(connection.hb_port)) 57 | 58 | 59 | val hmac = HMAC(connection.key, connection.signature_scheme) 60 | 61 | val closed = new AtomicBoolean() 62 | 63 | val heartBeatThread = new Thread("HeartBeat") { 64 | override def run() = ZMQ.proxy(heartbeat, heartbeat, null) 65 | } 66 | 67 | heartBeatThread.setDaemon(true) 68 | heartBeatThread.start() 69 | 70 | def socket(channel: Channel) = channel match { 71 | case Channel.Requests => requests 72 | case Channel.Control => control 73 | case Channel.Publish => publish 74 | case Channel.Input => stdin 75 | } 76 | 77 | def sink(channel: Channel): Sink[Task, Message] = { 78 | val s = socket(channel) 79 | 80 | val emitOne = Process.emit { msg: Message => 81 | Task[Unit] { 82 | logger.debug(s"Sending $msg on $channel") 83 | 84 | msg.idents.map(_.toArray) foreach { s.send(_, ZMQ.SNDMORE) } 85 | s.send(delimiterBytes.toArray, ZMQ.SNDMORE) 86 | s.send(if (connection.key.isEmpty) "" else hmac(msg.header, msg.parentHeader, msg.metaData, msg.content), ZMQ.SNDMORE) 87 | s.send(msg.header, ZMQ.SNDMORE) 88 | s.send(msg.parentHeader, ZMQ.SNDMORE) 89 | s.send(msg.metaData, ZMQ.SNDMORE) 90 | s.send(msg.content) 91 | } 92 | } 93 | 94 | lazy val helper: Sink[Task, Message] = 95 | if (closed.get()) 96 | Process.halt 97 | else 98 | emitOne ++ helper 99 | 100 | helper 101 | } 102 | 103 | def process(channel: Channel) = { 104 | val s = socket(channel) 105 | 106 | val poll = Task { 107 | if (closed.get()) 108 | None 109 | else { 110 | logger.debug(s"Polling on $channel... ($this)") 111 | 112 | val pi = new PollItem(s, Poller.POLLIN) 113 | ZMQ.poll(Array(pi), pollingDelay) 114 | 115 | Some(pi.isReadable) 116 | } 117 | } 118 | 119 | val read = Task { 120 | logger.debug(s"Reading message on $channel... ($connection)") 121 | 122 | def recvIdent(): Seq[Byte] = { 123 | val m = s.recv() 124 | logger.debug(s"Received message chunk '$m'") 125 | m 126 | } 127 | 128 | def recv(): String = { 129 | val m = s.recvStr() 130 | logger.debug(s"Received message chunk '$m'") 131 | m 132 | } 133 | 134 | val idents = 135 | if (connection.key.isEmpty) 136 | Nil 137 | else 138 | Stream.continually(recvIdent()) 139 | .takeWhile(_ != delimiterBytes) 140 | .toList 141 | 142 | val signature = 143 | if (connection.key.isEmpty) 144 | Nil 145 | else 146 | recv() 147 | 148 | val header = recv() 149 | val parentHeader = recv() 150 | val metaData = recv() 151 | val content = recv() 152 | 153 | logger.debug(s"Read message ${(idents, signature, header, parentHeader, metaData, content)} on $channel") 154 | 155 | lazy val expectedSignatureOpt = hmac(header, parentHeader, metaData, content) 156 | 157 | if (connection.key.nonEmpty && expectedSignatureOpt != signature) 158 | Left(s"Invalid HMAC signature, got $signature, expected $expectedSignatureOpt") 159 | else 160 | Right(Message(idents, header, parentHeader, metaData, content)) 161 | } 162 | 163 | lazy val helper: Process[Task, Either[String, Message]] = 164 | Process.await(poll) { 165 | case None => 166 | Process.halt 167 | case Some(true) => 168 | Process.eval(read) ++ helper 169 | case Some(false) => 170 | helper 171 | } 172 | 173 | helper 174 | } 175 | 176 | def close() = { 177 | closed.set(true) 178 | requests.close() 179 | control.close() 180 | publish.close() 181 | stdin.close() 182 | heartbeat.close() 183 | ctx.close() 184 | } 185 | 186 | val processes = Channel.channels.map { channel => 187 | channel -> ((process(channel), sink(channel))) 188 | }.toMap 189 | 190 | Streams(processes.apply, () => close()) 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /core/src/main/scala/jupyter/kernel/server/InterpreterServer.scala: -------------------------------------------------------------------------------- 1 | package jupyter 2 | package kernel 3 | package server 4 | 5 | import java.util.UUID 6 | import java.util.concurrent.{ConcurrentHashMap, ExecutorService} 7 | 8 | import argonaut.{Json, Parse} 9 | 10 | import scala.collection.mutable 11 | import com.typesafe.scalalogging.LazyLogging 12 | import interpreter.{DisplayData, Interpreter, InterpreterHandler} 13 | import jupyter.api._ 14 | import jupyter.kernel.stream.Streams 15 | import jupyter.kernel.protocol.{ Publish => PublishMsg, Comm => ProtocolComm, _ } 16 | import Formats._ 17 | 18 | import scalaz.concurrent.{Strategy, Task} 19 | import scalaz.stream.async 20 | import scalaz.stream.async.mutable.Queue 21 | 22 | object InterpreterServer extends LazyLogging { 23 | 24 | private class CommImpl(pubQueue: Queue[Message], id: String, handler: String => Option[CommChannelMessage => Unit]) { self => 25 | 26 | def received(msg: CommChannelMessage) = 27 | messageHandlers.foreach(_(msg)) 28 | 29 | var sentMessageHandlers = Seq.empty[CommChannelMessage => Unit] 30 | var messageHandlers = Seq.empty[CommChannelMessage => Unit] 31 | 32 | def onMessage(f: CommChannelMessage => Unit) = 33 | messageHandlers = messageHandlers :+ f 34 | 35 | def comm(t: ParsedMessage[_]): Comm = new Comm { 36 | def id = self.id 37 | def send(msg: CommChannelMessage) = { 38 | def parse(s: String): Json = 39 | Parse.parse(s).left.map(err => throw new IllegalArgumentException(s"Malformed JSON: $s ($err)")).merge 40 | 41 | pubQueue.enqueueOne(msg match { 42 | case CommOpen(target, data) => 43 | t.publish("comm_open", ProtocolComm.Open(id, target, parse(data))) 44 | case CommMessage(data) => 45 | t.publish("comm_msg", ProtocolComm.Message(id, parse(data))) 46 | case CommClose(data) => 47 | t.publish("comm_close", ProtocolComm.Close(id, parse(data))) 48 | }).unsafePerformSync 49 | 50 | sentMessageHandlers.foreach(_(msg)) 51 | } 52 | def onMessage(f: CommChannelMessage => Unit) = self.onMessage(f) 53 | def onSentMessage(f: CommChannelMessage => Unit) = 54 | sentMessageHandlers = sentMessageHandlers :+ f 55 | } 56 | } 57 | 58 | def apply( 59 | streams: Streams, 60 | connectReply: ShellReply.Connect, 61 | interpreter: Interpreter 62 | )(implicit 63 | es: ExecutorService 64 | ): Task[Unit] = { 65 | 66 | implicit val strategy = Strategy.Executor 67 | 68 | val queues = Channel.channels.map { channel => 69 | channel -> async.boundedQueue[Message](100) 70 | }.toMap 71 | 72 | val pubQueue = queues(Channel.Publish) 73 | 74 | val targetHandlers = new ConcurrentHashMap[String, CommChannelMessage => Unit] 75 | 76 | val comms = new mutable.HashMap[String, CommImpl] 77 | 78 | def comm0(id: String) = comms.getOrElseUpdate( 79 | id, 80 | new CommImpl(pubQueue, id, t => Option(targetHandlers.get(t))) 81 | ) 82 | 83 | interpreter.publish(t => 84 | new Publish { 85 | def stdout(text: String) = 86 | pubQueue.enqueueOne(t.publish("stream", PublishMsg.Stream(name = "stdout", text = text), ident = "stdout")).unsafePerformSync 87 | def stderr(text: String) = 88 | pubQueue.enqueueOne(t.publish("stream", PublishMsg.Stream(name = "stderr", text = text), ident = "stderr")).unsafePerformSync 89 | def display(items: (String, String)*) = 90 | pubQueue.enqueueOne( 91 | t.publish( 92 | "display_data", 93 | PublishMsg.DisplayData( 94 | items.map { case (tpe, data) => DisplayData(tpe, data).jsonField }.toMap, 95 | Map.empty 96 | ) 97 | ) 98 | ).unsafePerformSync 99 | 100 | def comm(id: String) = comm0(id).comm(t) 101 | 102 | def commHandler(target: String)(handler: CommChannelMessage => Unit) = 103 | targetHandlers.put(target, handler) 104 | } 105 | ) 106 | 107 | def commReceived(id: String, msg: CommChannelMessage) = { 108 | 109 | msg match { 110 | case CommOpen(target, _) => 111 | val target0 = Some(target).filter(_.nonEmpty) 112 | 113 | for { 114 | t <- target0 115 | h <- Option(targetHandlers.get(t)) 116 | } 117 | comm0(id).onMessage(h) 118 | 119 | case _ => 120 | } 121 | 122 | comm0(id).received(msg) 123 | } 124 | 125 | val process: Either[String, Message] => Task[Unit] = { 126 | case Left(err) => 127 | logger.debug(s"Error while decoding message: $err") 128 | Task.now(()) 129 | case Right(msg) => 130 | InterpreterHandler(interpreter, connectReply, commReceived, msg) match { 131 | case Left(err) => 132 | logger.error(s"Error while handling message: $err\n$msg") 133 | Task.now(()) 134 | case Right(proc) => 135 | proc.evalMap { 136 | case (channel, m) => 137 | queues(channel).enqueueOne(m) 138 | }.run 139 | } 140 | } 141 | 142 | val sendTasks = Channel.channels.map { channel => 143 | queues(channel).dequeue.to(streams.processes(channel)._2).run 144 | } 145 | 146 | val sendInitialStatus = pubQueue.enqueueOne( 147 | ParsedMessage( 148 | List("status".getBytes("UTF-8")), 149 | Header( 150 | msg_id = UUID.randomUUID().toString, 151 | username = "scala_kernel", 152 | session = UUID.randomUUID().toString, 153 | msg_type = "status", 154 | version = Protocol.versionStrOpt 155 | ), 156 | None, 157 | Map.empty, 158 | PublishMsg.Status(PublishMsg.ExecutionState0.Idle) 159 | ).toMessage 160 | ) 161 | 162 | val processRequestMessages = streams.processes(Channel.Requests)._1.evalMap(process).run 163 | val processControlMessages = streams.processes(Channel.Control)._1.evalMap(process).run 164 | 165 | Task.gatherUnordered( 166 | Seq( 167 | sendInitialStatus, 168 | processRequestMessages, 169 | processControlMessages 170 | ) ++ 171 | sendTasks 172 | ).map(_ => ()) 173 | } 174 | 175 | } 176 | -------------------------------------------------------------------------------- /protocol/src/main/scala/jupyter/kernel/protocol/ShellReply.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.protocol 2 | 3 | sealed abstract class ShellReply extends Product with Serializable 4 | 5 | object ShellReply { 6 | 7 | sealed abstract class Status extends Product with Serializable 8 | 9 | object Status { 10 | case object Ok extends Status 11 | case object Abort extends Status 12 | case object Error extends Status 13 | 14 | // required for the type class derivation to be fine in 2.10 15 | type Ok = Ok.type 16 | type Abort = Abort.type 17 | type Error = Error.type 18 | } 19 | 20 | 21 | final case class Error( 22 | ename: String, 23 | evalue: String, 24 | traceback: List[String], 25 | status: Status.Error, // no default value here for the value not to be swallowed by the JSON encoder 26 | execution_count: Int = -1 // required in some context (e.g. errored execute_reply from jupyter console) 27 | ) extends ShellReply 28 | 29 | object Error { 30 | def apply( 31 | ename: String, 32 | evalue: String, 33 | traceback: List[String] 34 | ): Error = 35 | Error( 36 | ename, 37 | evalue, 38 | traceback, 39 | Status.Error 40 | ) 41 | 42 | def apply( 43 | ename: String, 44 | evalue: String, 45 | traceback: List[String], 46 | execution_count: Int 47 | ): Error = 48 | Error( 49 | ename, 50 | evalue, 51 | traceback, 52 | Status.Error, 53 | execution_count 54 | ) 55 | } 56 | 57 | final case class Abort( 58 | status: Status.Abort // no default value here for the value not to be swallowed by the JSON encoder 59 | ) extends ShellReply 60 | 61 | object Abort { 62 | def apply(): Abort = 63 | Abort(Status.Abort) 64 | } 65 | 66 | 67 | // payloads not supported here 68 | final case class Execute( 69 | execution_count: Int, 70 | user_expressions: Map[String, String], 71 | status: Status.Ok // no default value here for the value not to be swallowed by the JSON encoder 72 | ) extends ShellReply 73 | 74 | object Execute { 75 | def apply( 76 | execution_count: Int, 77 | user_expressions: Map[String, String] 78 | ): Execute = 79 | Execute( 80 | execution_count, 81 | user_expressions, 82 | Status.Ok 83 | ) 84 | } 85 | 86 | final case class Inspect( 87 | found: Boolean, 88 | data: Map[String, String], 89 | metadata: Map[String, String], 90 | status: Status.Ok // no default value here for the value not to be swallowed by the JSON encoder 91 | ) extends ShellReply 92 | 93 | object Inspect { 94 | def apply( 95 | found: Boolean, 96 | data: Map[String, String], 97 | metadata: Map[String, String] 98 | ): Inspect = 99 | Inspect( 100 | found, 101 | data, 102 | metadata, 103 | Status.Ok 104 | ) 105 | } 106 | 107 | final case class Complete( 108 | matches: List[String], 109 | cursor_start: Int, 110 | cursor_end: Int, 111 | metadata: Map[String, String], 112 | status: Status.Ok 113 | ) extends ShellReply 114 | 115 | object Complete { 116 | def apply( 117 | matches: List[String], 118 | cursor_start: Int, 119 | cursor_end: Int, 120 | metadata: Map[String, String] 121 | ): Complete = 122 | Complete( 123 | matches, 124 | cursor_start, 125 | cursor_end, 126 | metadata, 127 | Status.Ok 128 | ) 129 | } 130 | 131 | sealed abstract class History extends ShellReply 132 | 133 | object History { 134 | 135 | final case class Default( 136 | history: List[(Int, Int, String)], 137 | status: Status.Ok 138 | ) extends History 139 | 140 | object Default { 141 | def apply( 142 | history: List[(Int, Int, String)] 143 | ): Default = 144 | Default( 145 | history, 146 | Status.Ok 147 | ) 148 | } 149 | 150 | final case class WithOutput( 151 | history: List[(Int, Int, (String, String))], // FIXME Not sure about the type of ._3._2 of the elements 152 | status: Status.Ok 153 | ) extends History 154 | 155 | object WithOutput { 156 | def apply( 157 | history: List[(Int, Int, (String, String))] 158 | ): WithOutput = 159 | WithOutput( 160 | history, 161 | Status.Ok 162 | ) 163 | } 164 | 165 | } 166 | 167 | sealed abstract class IsComplete extends ShellReply { 168 | def status: String 169 | } 170 | 171 | object IsComplete { 172 | case object Complete extends IsComplete { 173 | def status = "complete" 174 | } 175 | final case class Incomplete(indent: String) extends IsComplete { 176 | def status = "incomplete" 177 | } 178 | case object Invalid extends IsComplete { 179 | def status = "invalid" 180 | } 181 | case object Unknown extends IsComplete { 182 | def status = "unknown" 183 | } 184 | } 185 | 186 | final case class Connect( 187 | shell_port: Int, 188 | iopub_port: Int, 189 | stdin_port: Int, 190 | hb_port: Int 191 | ) extends ShellReply 192 | 193 | final case class CommInfo( 194 | comms: Map[String, CommInfo.Info] 195 | ) extends ShellReply 196 | 197 | object CommInfo { 198 | final case class Info(target_name: String) 199 | } 200 | 201 | final case class KernelInfo( 202 | protocol_version: String, // X.Y.Z 203 | implementation: String, 204 | implementation_version: String, // X.Y.Z 205 | language_info: KernelInfo.LanguageInfo, 206 | banner: String, 207 | help_links: Option[List[KernelInfo.Link]] = None 208 | ) extends ShellReply 209 | 210 | object KernelInfo { 211 | final case class LanguageInfo( 212 | name: String, 213 | version: String, // X.Y.Z 214 | mimetype: String, 215 | file_extension: String, // including the dot 216 | nbconvert_exporter: String, 217 | pygments_lexer: Option[String] = None, // only needed if it differs from name 218 | codemirror_mode: Option[String] = None // only needed if it differs from name - FIXME could be a dict too? 219 | ) 220 | 221 | final case class Link(text: String, url: String) 222 | } 223 | 224 | final case class Shutdown( 225 | restart: Boolean 226 | ) extends ShellReply 227 | 228 | } 229 | -------------------------------------------------------------------------------- /core/src/main/scala/jupyter/kernel/server/ServerApp.scala: -------------------------------------------------------------------------------- 1 | package jupyter 2 | package kernel 3 | package server 4 | 5 | import java.io.File 6 | import java.nio.file.Files 7 | import java.util.concurrent.Executors 8 | 9 | import com.typesafe.scalalogging.LazyLogging 10 | import jupyter.kernel.interpreter.InterpreterKernel 11 | import jupyter.kernel.protocol.{ Kernel => KernelDesc } 12 | 13 | import scala.compat.Platform._ 14 | import scala.util.{ Failure, Success, Try } 15 | import scalaz.concurrent.Strategy 16 | 17 | final case class ServerAppOptions( 18 | connectionFile: String = "", 19 | eraseConnectionFile: Boolean = false, 20 | quiet: Boolean = false, 21 | exitOnKeyPress: Boolean = false, 22 | force: Boolean = false, 23 | noCopy: Boolean = false, 24 | jupyterPath: String = "", 25 | global: Boolean = false 26 | ) { 27 | lazy val options = Server.Options( 28 | connectionFile, 29 | eraseConnectionFile, 30 | quiet 31 | ) 32 | } 33 | 34 | object ServerApp extends LazyLogging { 35 | 36 | def generateKernelSpec( 37 | id: String, 38 | name: String, 39 | language: String, 40 | progPath: String, 41 | isJar: Boolean, 42 | options: ServerAppOptions = ServerAppOptions(), 43 | extraProgArgs: Seq[String] = Nil, 44 | logos: => Seq[((Int, Int), Array[Byte])] = Nil 45 | ): Unit = { 46 | 47 | if (options.options.copy(quiet = false) != Server.Options()) 48 | Console.err.println(s"Warning: ignoring kernel launching options when kernel spec option specified") 49 | 50 | val kernelsDir = 51 | if (options.jupyterPath.isEmpty) { 52 | 53 | // error message suck... can it fail anyway? 54 | 55 | if (options.global) 56 | KernelSpecs.systemKernelSpecDirectories.headOption.getOrElse { 57 | Console.err.println("No global kernel directory found") 58 | sys.exit(1) 59 | } 60 | else 61 | KernelSpecs.userKernelSpecDirectory.getOrElse { 62 | Console.err.println("No user kernel directory found") 63 | sys.exit(1) 64 | } 65 | } else 66 | new File(options.jupyterPath + "/kernels") 67 | 68 | val kernelDir = new File(kernelsDir, id) 69 | 70 | val kernelJsonFile = new File(kernelDir, "kernel.json") 71 | val launcherFile = new File(kernelDir, "launcher.jar") 72 | 73 | if (!options.force) { 74 | if (kernelJsonFile.exists()) { 75 | Console.err.println(s"Error: $kernelJsonFile already exists, force erasing it with --force") 76 | sys.exit(1) 77 | } 78 | 79 | if (!options.noCopy && launcherFile.exists()) { 80 | Console.err.println(s"Error: $launcherFile already exists, force erasing it with --force") 81 | sys.exit(1) 82 | } 83 | } 84 | 85 | val parentDir = kernelJsonFile.getParentFile 86 | 87 | if (!parentDir.exists() && !parentDir.mkdirs() && !options.options.quiet) 88 | Console.err.println( 89 | s"Warning: cannot create directory $parentDir, attempting to generate kernel spec anyway." 90 | ) 91 | 92 | val progPath0 = 93 | if (options.noCopy) 94 | progPath 95 | else { 96 | if (options.force && launcherFile.exists()) 97 | launcherFile.delete() 98 | 99 | launcherFile.getParentFile.mkdirs() 100 | Files.copy(new File(progPath).toPath, launcherFile.toPath) 101 | launcherFile.getAbsolutePath 102 | } 103 | 104 | val launch = 105 | if (isJar) 106 | List( 107 | // FIXME What if java is not in PATH? 108 | "java", 109 | // needed by the recent proguarded coursier launchers 110 | "-noverify", 111 | "-jar", 112 | progPath0 113 | ) 114 | else 115 | List(progPath0) 116 | 117 | val conn = KernelDesc( 118 | launch ++ extraProgArgs ++ List("--quiet", "--connection-file", "{connection_file}"), 119 | name, 120 | language 121 | ) 122 | 123 | val connStr = { 124 | import argonaut._, Argonaut._, ArgonautShapeless._ 125 | conn.asJson.spaces2 126 | } 127 | 128 | Files.write(kernelJsonFile.toPath, connStr.getBytes()) // using the default charset here 129 | 130 | for (((w, h), b) <- logos) { 131 | val f = new File(kernelDir, s"logo-${w}x$h.png") 132 | if (options.force || !f.exists()) 133 | Files.write(f.toPath, b) 134 | } 135 | 136 | if (!options.options.quiet) { 137 | val kernelId0 = 138 | if (id.exists(_.isSpaceChar)) 139 | "\"" + id + "\"" 140 | else 141 | id 142 | 143 | println( 144 | s"""Generated $kernelJsonFile 145 | | 146 | |Run jupyter console with this kernel with 147 | | jupyter console --kernel $kernelId0 148 | | 149 | |Use this kernel from Jupyter notebook, running 150 | | jupyter notebook 151 | |and selecting the "$name" kernel. 152 | """.stripMargin 153 | ) 154 | } 155 | } 156 | 157 | def apply( 158 | id: String, 159 | name: String, 160 | language: String, 161 | kernel: InterpreterKernel, 162 | progPath: => String, 163 | isJar: Boolean, 164 | options: ServerAppOptions = ServerAppOptions(), 165 | extraProgArgs: Seq[String] = Nil, 166 | logos: => Seq[((Int, Int), Array[Byte])] = Nil 167 | ): Unit = 168 | if (options.options.connectionFile.isEmpty) 169 | generateKernelSpec(id, name, language, progPath, isJar, options, extraProgArgs, logos) 170 | else 171 | Try(Server(kernel, id, options.options)(Executors.newCachedThreadPool(Strategy.DefaultDaemonThreadFactory))) match { 172 | case Failure(err) => 173 | logger.error(s"Launching kernel", err) 174 | throw new RuntimeException(err) 175 | 176 | case Success((connFile, task)) => 177 | if (!options.options.quiet) 178 | Console.err.println( 179 | s"""Connect jupyter to this kernel with 180 | | jupyter console --existing "${connFile.getAbsolutePath}" 181 | """.stripMargin 182 | ) 183 | 184 | if (options.exitOnKeyPress) { 185 | if (!options.options.quiet) 186 | Console.err.println("Press enter to exit.") 187 | Console.in.readLine() 188 | sys.exit(0) 189 | } else 190 | task.unsafePerformSync 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /core/src/main/scala/jupyter/kernel/protocol/OptimizedPrinter.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.protocol 2 | 3 | import argonaut._ 4 | 5 | import scalaz.Memo 6 | 7 | object OptimizedPrinter { 8 | 9 | import PrettyParams.nospace._ 10 | 11 | import Memo._ 12 | 13 | private def vectorMemo() = { 14 | var vector: Vector[String] = Vector.empty 15 | 16 | val memoFunction: (Int => String) => Int => String = f => k => { 17 | val localVector = vector 18 | val adjustedK = if (k < 0) 0 else k 19 | if (localVector.size > adjustedK) { 20 | localVector(adjustedK) 21 | } else { 22 | val newVector = Vector.tabulate(adjustedK + 1)(f) 23 | vector = newVector 24 | newVector.last 25 | } 26 | } 27 | memo(memoFunction) 28 | } 29 | 30 | private def addIndentation(s: String): Int => String = { 31 | val lastNewLineIndex = s.lastIndexOf("\n") 32 | if (lastNewLineIndex < 0) { 33 | _ => s 34 | } else { 35 | val afterLastNewLineIndex = lastNewLineIndex + 1 36 | val start = s.substring(0, afterLastNewLineIndex) 37 | val end = s.substring(afterLastNewLineIndex) 38 | n => start + indent * n + end 39 | } 40 | } 41 | 42 | private val openBraceText = "{" 43 | private val closeBraceText = "}" 44 | private val openArrayText = "[" 45 | private val closeArrayText = "]" 46 | private val commaText = "," 47 | private val colonText = ":" 48 | private val nullText = "null" 49 | private val trueText = "true" 50 | private val falseText = "false" 51 | private val stringEnclosureText = "\"" 52 | 53 | private val _lbraceLeft = addIndentation(lbraceLeft) 54 | private val _lbraceRight = addIndentation(lbraceRight) 55 | private val _rbraceLeft = addIndentation(rbraceLeft) 56 | private val _rbraceRight = addIndentation(rbraceRight) 57 | private val _lbracketLeft = addIndentation(lbracketLeft) 58 | private val _lbracketRight = addIndentation(lbracketRight) 59 | private val _rbracketLeft = addIndentation(rbracketLeft) 60 | private val _rbracketRight = addIndentation(rbracketRight) 61 | private val _lrbracketsEmpty = addIndentation(lrbracketsEmpty) 62 | private val _arrayCommaLeft = addIndentation(arrayCommaLeft) 63 | private val _arrayCommaRight = addIndentation(arrayCommaRight) 64 | private val _objectCommaLeft = addIndentation(objectCommaLeft) 65 | private val _objectCommaRight = addIndentation(objectCommaRight) 66 | private val _colonLeft = addIndentation(colonLeft) 67 | private val _colonRight = addIndentation(colonRight) 68 | 69 | private val lbraceMemo = vectorMemo(){depth: Int => "%s%s%s".format(_lbraceLeft(depth), openBraceText, _lbraceRight(depth + 1))} 70 | private val rbraceMemo = vectorMemo(){depth: Int => "%s%s%s".format(_rbraceLeft(depth), closeBraceText, _rbraceRight(depth + 1))} 71 | 72 | private val lbracketMemo = vectorMemo(){depth: Int => "%s%s%s".format(_lbracketLeft(depth), openArrayText, _lbracketRight(depth + 1))} 73 | private val rbracketMemo = vectorMemo(){depth: Int => "%s%s%s".format(_rbracketLeft(depth), closeArrayText, _rbracketRight(depth))} 74 | private val lrbracketsEmptyMemo = vectorMemo(){depth: Int => "%s%s%s".format(openArrayText, _lrbracketsEmpty(depth), closeArrayText)} 75 | private val arrayCommaMemo = vectorMemo(){depth: Int => "%s%s%s".format(_arrayCommaLeft(depth + 1), commaText, _arrayCommaRight(depth + 1))} 76 | private val objectCommaMemo = vectorMemo(){depth: Int => "%s%s%s".format(_objectCommaLeft(depth + 1), commaText, _objectCommaRight(depth + 1))} 77 | private val colonMemo = vectorMemo(){depth: Int => "%s%s%s".format(_colonLeft(depth + 1), colonText, _colonRight(depth + 1))} 78 | 79 | /** 80 | * Returns a string representation of a pretty-printed JSON value. 81 | */ 82 | final def noSpaces(j: Json): String = { 83 | 84 | import Json._ 85 | import StringEscaping._ 86 | 87 | def appendJsonString(builder: StringBuilder, jsonString: String): StringBuilder = { 88 | for (c <- jsonString) { 89 | if (isNormalChar(c)) 90 | builder += c 91 | else 92 | builder.append(escape(c)) 93 | } 94 | 95 | builder 96 | } 97 | 98 | def encloseJsonString(builder: StringBuilder, jsonString: JsonString): StringBuilder = { 99 | appendJsonString(builder.append(stringEnclosureText), jsonString).append(stringEnclosureText) 100 | } 101 | 102 | def trav(builder: StringBuilder, depth: Int, k: Json): StringBuilder = { 103 | 104 | def lbrace(builder: StringBuilder): StringBuilder = { 105 | builder.append(lbraceMemo(depth)) 106 | } 107 | def rbrace(builder: StringBuilder): StringBuilder = { 108 | builder.append(rbraceMemo(depth)) 109 | } 110 | def lbracket(builder: StringBuilder): StringBuilder = { 111 | builder.append(lbracketMemo(depth)) 112 | } 113 | def rbracket(builder: StringBuilder): StringBuilder = { 114 | builder.append(rbracketMemo(depth)) 115 | } 116 | def lrbracketsEmpty(builder: StringBuilder): StringBuilder = { 117 | builder.append(lrbracketsEmptyMemo(depth)) 118 | } 119 | def arrayComma(builder: StringBuilder): StringBuilder = { 120 | builder.append(arrayCommaMemo(depth)) 121 | } 122 | def objectComma(builder: StringBuilder): StringBuilder = { 123 | builder.append(objectCommaMemo(depth)) 124 | } 125 | def colon(builder: StringBuilder): StringBuilder = { 126 | builder.append(colonMemo(depth)) 127 | } 128 | 129 | k.fold[StringBuilder]( 130 | builder.append(nullText) 131 | , bool => builder.append(if (bool) trueText else falseText) 132 | , n => n match { 133 | case JsonLong(x) => builder append x.toString 134 | case JsonDecimal(x) => builder append x 135 | case JsonBigDecimal(x) => builder append x.toString 136 | } 137 | , s => encloseJsonString(builder, s) 138 | , e => if (e.isEmpty) { 139 | lrbracketsEmpty(builder) 140 | } else { 141 | rbracket(e.foldLeft((true, lbracket(builder))){case ((firstElement, builder), subElement) => 142 | val withComma = if (firstElement) builder else arrayComma(builder) 143 | val updatedBuilder = trav(withComma, depth + 1, subElement) 144 | (false, updatedBuilder) 145 | }._2) 146 | } 147 | , o => { 148 | rbrace((if (preserveOrder) o.toList else o.toMap).foldLeft((true, lbrace(builder))){case ((firstElement, builder), (key, value)) => 149 | val ignoreKey = dropNullKeys && value.isNull 150 | if (ignoreKey) { 151 | (firstElement, builder) 152 | } else { 153 | val withComma = if (firstElement) builder else objectComma(builder) 154 | (false, trav(colon(encloseJsonString(withComma, key)), depth + 1, value)) 155 | } 156 | }._2) 157 | } 158 | ) 159 | } 160 | 161 | trav(new StringBuilder(), 0, j).toString() 162 | } 163 | 164 | } 165 | -------------------------------------------------------------------------------- /core/src/main/scala/jupyter/kernel/protocol/Formats.scala: -------------------------------------------------------------------------------- 1 | package jupyter.kernel.protocol 2 | 3 | import argonaut._, Argonaut._, ArgonautShapeless._ 4 | import argonaut.derive.{ JsonSumCodec, JsonSumCodecFor } 5 | 6 | import shapeless.Typeable 7 | 8 | object FormatHelpers { 9 | 10 | // TODO Move these to argonaut-shapeless? 11 | 12 | def jsonSumDirectCodecFor(name: String): JsonSumCodec = new JsonSumCodec { 13 | def encodeEmpty: Nothing = 14 | throw new IllegalArgumentException(s"empty $name") 15 | def encodeField(fieldOrObj: Either[Json, (String, Json)]): Json = 16 | fieldOrObj match { 17 | case Left(other) => other 18 | case Right((_, content)) => content 19 | } 20 | 21 | def decodeEmpty(cursor: HCursor): DecodeResult[Nothing] = 22 | // FIXME Sometimes reports the wrong error (in case of two nested sum types) 23 | DecodeResult.fail(s"unrecognized $name", cursor.history) 24 | def decodeField[A](name: String, cursor: HCursor, decode: DecodeJson[A]): DecodeResult[Either[ACursor, A]] = 25 | DecodeResult.ok { 26 | val o = decode 27 | .decode(cursor) 28 | o.toOption 29 | .toRight(ACursor.ok(cursor)) 30 | } 31 | } 32 | 33 | def constantStringSingletonDecode[T](label: String, value: T): DecodeJson[T] = 34 | DecodeJson { c => 35 | c.as[String].flatMap { s => 36 | if (s == value) 37 | DecodeResult.ok(value) 38 | else 39 | DecodeResult.fail(s"Expected $label, got $s", c.history) 40 | } 41 | } 42 | 43 | def constantStringSingletonEncode[T](label: String): EncodeJson[T] = { 44 | val json = label.asJson 45 | EncodeJson(_ => json) 46 | } 47 | 48 | trait IsEnum[-T] { 49 | def label(t: T): String 50 | } 51 | 52 | object IsEnum { 53 | def apply[T](implicit isEnum: IsEnum[T]): IsEnum[T] = isEnum 54 | 55 | def instance[T](f: T => String): IsEnum[T] = 56 | new IsEnum[T] { 57 | def label(t: T) = f(t) 58 | } 59 | } 60 | 61 | implicit def isEnumEncoder[T: IsEnum]: EncodeJson[T] = 62 | EncodeJson.of[String].contramap(IsEnum[T].label) 63 | implicit def isEnumDecoder[T] 64 | (implicit 65 | isEnum: IsEnum[T], 66 | enum: Enumerate[T], 67 | typeable: Typeable[T] 68 | ): DecodeJson[T] = 69 | DecodeJson { 70 | val underlying = DecodeJson.of[String] 71 | val map = enum().map(e => isEnum.label(e) -> e).toMap 72 | val name = typeable.describe // TODO split in words 73 | 74 | c => 75 | underlying(c).flatMap { s => 76 | map.get(s) match { 77 | case None => DecodeResult.fail(s"Unrecognized $name: '$s'", c.history) 78 | case Some(m) => DecodeResult.ok(m) 79 | } 80 | } 81 | } 82 | 83 | } 84 | 85 | trait ShellRequestDecodeJsons { 86 | 87 | import ShellRequest._ 88 | import FormatHelpers._ 89 | 90 | private implicit val shellRequestHistoryJsonCodec = 91 | JsonSumCodecFor[History](jsonSumDirectCodecFor("history request")) 92 | 93 | private implicit val decodeShellRequestHistoryAccessTypeRange = 94 | constantStringSingletonDecode("range", History.AccessType.Range) 95 | private implicit val decodeShellRequestHistoryAccessTypeTail = 96 | constantStringSingletonDecode("tail", History.AccessType.Tail) 97 | private implicit val decodeShellRequestHistoryAccessTypeSearch = 98 | constantStringSingletonDecode("search", History.AccessType.Search) 99 | 100 | implicit val decodeShellRequestExecute = DecodeJson.of[Execute] 101 | implicit val decodeShellRequestInspect = DecodeJson.of[Inspect] 102 | implicit val decodeShellRequestComplete = DecodeJson.of[Complete] 103 | implicit val decodeShellRequestHistory = DecodeJson.of[History] 104 | implicit val decodeShellRequestIsComplete = DecodeJson.of[IsComplete] 105 | implicit val decodeShellRequestConnect = DecodeJson.of[Connect.type] 106 | implicit val decodeShellRequestCommInfo = DecodeJson.of[CommInfo] 107 | implicit val decodeShellRequestKernelInfo = DecodeJson.of[KernelInfo.type] 108 | implicit val decodeShellRequestShutdown = DecodeJson.of[Shutdown] 109 | 110 | } 111 | 112 | trait ShellRequestEncodeJsons { 113 | 114 | import ShellRequest._ 115 | import FormatHelpers._ 116 | 117 | private implicit val shellRequestHistoryJsonCodec = 118 | JsonSumCodecFor[History](jsonSumDirectCodecFor("history request")) 119 | 120 | private implicit val encodeShellRequestHistoryAccessTypeRange = 121 | constantStringSingletonEncode[History.AccessType.Range]("range") 122 | private implicit val encodeShellRequestHistoryAccessTypeTail = 123 | constantStringSingletonEncode[History.AccessType.Tail]("tail") 124 | private implicit val encodeShellRequestHistoryAccessTypeSearch = 125 | constantStringSingletonEncode[History.AccessType.Search]("search") 126 | 127 | implicit val encodeShellRequestExecute = EncodeJson.of[Execute] 128 | implicit val encodeShellRequestInspect = EncodeJson.of[Inspect] 129 | implicit val encodeShellRequestComplete = EncodeJson.of[Complete] 130 | implicit val encodeShellRequestHistory = EncodeJson.of[History] 131 | implicit val encodeShellRequestIsComplete = EncodeJson.of[IsComplete] 132 | implicit val encodeShellRequestConnect = EncodeJson.of[Connect.type] 133 | implicit val encodeShellRequestCommInfo = EncodeJson.of[CommInfo] 134 | implicit val encodeShellRequestKernelInfo = EncodeJson.of[KernelInfo.type] 135 | implicit val encodeShellRequestShutdown = EncodeJson.of[Shutdown] 136 | 137 | } 138 | 139 | trait ShellReplyEncodeJsons { 140 | 141 | import ShellReply._ 142 | import FormatHelpers._ 143 | 144 | private implicit val encodeShellReplyStatusOk = 145 | constantStringSingletonEncode[Status.Ok]("ok") 146 | private implicit val encodeShellReplyStatusAbort = 147 | constantStringSingletonEncode[Status.Abort]("abort") 148 | private implicit val encodeShellReplyStatusError = 149 | constantStringSingletonEncode[Status.Error]("error") 150 | 151 | private implicit val shellReplyHistoryJsonCodec = 152 | JsonSumCodecFor[History](jsonSumDirectCodecFor("history reply")) 153 | 154 | implicit val encodeShellReplyIsComplete: EncodeJson[IsComplete] = { 155 | 156 | final case class Resp(status: String, indent: String = "") 157 | 158 | EncodeJson { isComplete => 159 | 160 | val resp0 = Resp(isComplete.status) 161 | 162 | val resp = isComplete match { 163 | case ic @ IsComplete.Incomplete(indent) => 164 | resp0.copy(indent = indent) 165 | case _ => 166 | resp0 167 | } 168 | 169 | resp.asJson 170 | } 171 | } 172 | 173 | implicit val encodeShellReplyError = EncodeJson.of[Error] 174 | implicit val encodeShellReplyAbort = EncodeJson.of[Abort] 175 | implicit val encodeShellReplyExecute = EncodeJson.of[Execute] 176 | implicit val encodeShellReplyInspect = EncodeJson.of[Inspect] 177 | implicit val encodeShellReplyComplete = EncodeJson.of[Complete] 178 | implicit val encodeShellReplyHistory = EncodeJson.of[History] 179 | implicit val encodeShellReplyHistoryDefault = EncodeJson.of[History.Default] 180 | implicit val encodeShellReplyHistoryWithOutput = EncodeJson.of[History.WithOutput] 181 | implicit val encodeShellReplyConnect = EncodeJson.of[Connect] 182 | implicit val encodeShellReplyCommInfo = EncodeJson.of[CommInfo] 183 | implicit val encodeShellReplyKernelInfo = EncodeJson.of[KernelInfo] 184 | implicit val encodeShellReplyShutdown = EncodeJson.of[Shutdown] 185 | 186 | } 187 | 188 | trait PublishJsonCodecs { 189 | 190 | import Publish._ 191 | import FormatHelpers._ 192 | 193 | private implicit val publishExecutionStateIsEnum = IsEnum.instance[ExecutionState0](_.label) 194 | 195 | // add back support for records in argonaut-shapeless!!! 196 | implicit val encodePublishClearOutput: EncodeJson[ClearOutput] = 197 | EncodeJson { 198 | msg => 199 | // Record(wait = msg.wait0).asJson 200 | Json.obj( 201 | "wait" -> msg.wait0.asJson 202 | ) 203 | } 204 | implicit val decodePublishClearOutput: DecodeJson[ClearOutput] = 205 | DecodeJson { c => 206 | // c.as[Record.`'wait -> Boolean`.T].map { rec => 207 | // ClearOutput(rec.wait) 208 | // } 209 | c.--\("wait").as[Boolean].map { w => 210 | ClearOutput(w) 211 | } 212 | } 213 | 214 | implicit val encodePublishStream = EncodeJson.of[Stream] 215 | implicit val decodePublishStream = DecodeJson.of[Stream] 216 | implicit val encodePublishDisplayData = EncodeJson.of[DisplayData] 217 | implicit val decodePublishDisplayData = DecodeJson.of[DisplayData] 218 | implicit val encodePublishExecuteInput = EncodeJson.of[ExecuteInput] 219 | implicit val decodePublishExecuteInput = DecodeJson.of[ExecuteInput] 220 | implicit val encodePublishExecuteResult = EncodeJson.of[ExecuteResult] 221 | implicit val decodePublishExecuteResult = DecodeJson.of[ExecuteResult] 222 | implicit val encodePublishError = EncodeJson.of[Error] 223 | implicit val decodePublishError = DecodeJson.of[Error] 224 | implicit val encodePublishStatus = EncodeJson.of[Status] 225 | implicit val decodePublishStatus = DecodeJson.of[Status] 226 | 227 | } 228 | 229 | trait StdinRequestDecodeJsons { 230 | 231 | import StdinRequest._ 232 | 233 | implicit val decodeStdinRequestInput = DecodeJson.of[Input] 234 | 235 | } 236 | 237 | trait StdinReplyEncodeJsons { 238 | 239 | import StdinReply._ 240 | 241 | implicit val encodeStdinReplyInput = EncodeJson.of[Input] 242 | 243 | } 244 | 245 | trait CommJsonCodecs { 246 | 247 | import Comm._ 248 | 249 | implicit val encodeCommOpen = EncodeJson.of[Open] 250 | implicit val decodeCommOpen = DecodeJson.of[Open] 251 | implicit val encodeCommMessage = EncodeJson.of[Message] 252 | implicit val decodeCommMessage = DecodeJson.of[Message] 253 | implicit val encodeCommClose = EncodeJson.of[Close] 254 | implicit val decodeCommClose = DecodeJson.of[Close] 255 | 256 | } 257 | 258 | trait HeaderJsonCodecs { 259 | 260 | implicit lazy val decodeHeader = DecodeJson.of[Header] 261 | implicit lazy val encodeHeader = EncodeJson.of[Header] 262 | 263 | } 264 | 265 | trait ConnectionJsonCodecs { 266 | 267 | implicit lazy val decodeConnection = DecodeJson.of[Connection] 268 | implicit lazy val encodeConnection = EncodeJson.of[Connection] 269 | } 270 | 271 | object Formats 272 | extends ShellRequestDecodeJsons 273 | with ShellRequestEncodeJsons 274 | with ShellReplyEncodeJsons 275 | with PublishJsonCodecs 276 | with StdinRequestDecodeJsons 277 | with StdinReplyEncodeJsons 278 | with CommJsonCodecs 279 | with HeaderJsonCodecs 280 | with ConnectionJsonCodecs -------------------------------------------------------------------------------- /core/src/main/scala/jupyter/kernel/interpreter/InterpreterHandler.scala: -------------------------------------------------------------------------------- 1 | package jupyter 2 | package kernel 3 | package interpreter 4 | 5 | import java.util.UUID 6 | import java.util.concurrent.ExecutorService 7 | 8 | import jupyter.api._ 9 | import jupyter.kernel.protocol._, Formats._ 10 | 11 | import argonaut._, Argonaut.{ EitherDecodeJson => _, EitherEncodeJson => _, _ } 12 | import com.typesafe.scalalogging.LazyLogging 13 | 14 | import scala.util.control.NonFatal 15 | import scalaz.concurrent.{ Strategy, Task } 16 | import scalaz.stream.Process 17 | 18 | object InterpreterHandler extends LazyLogging { 19 | 20 | private def busy(msg: ParsedMessage[_])(f: => Process[Task, (Channel, Message)]): Process[Task, (Channel, Message)] = { 21 | 22 | def status(state: Publish.ExecutionState0) = { 23 | val statusMsg = ParsedMessage( 24 | List("status".getBytes("UTF-8")), 25 | Header( 26 | msg_id = UUID.randomUUID().toString, 27 | username = msg.header.username, 28 | session = msg.header.session, 29 | msg_type = "status", 30 | version = Protocol.versionStrOpt 31 | ), 32 | Some(msg.header), 33 | Map.empty, 34 | Publish.Status(execution_state = state) 35 | ).toMessage 36 | 37 | Process.emit( 38 | Channel.Publish -> statusMsg 39 | ) 40 | } 41 | 42 | status(Publish.ExecutionState0.Busy) ++ f ++ status(Publish.ExecutionState0.Idle) 43 | } 44 | 45 | private def publishing( 46 | msg: ParsedMessage[_] 47 | )( 48 | f: (Message => Unit) => Seq[Message] 49 | )(implicit 50 | pool: ExecutorService 51 | ): Process[Task, (Channel, Message)] = { 52 | 53 | implicit val strategy = Strategy.Executor 54 | 55 | busy(msg) { 56 | val q = scalaz.stream.async.boundedQueue[Message](1000) 57 | 58 | val res = Task.unsafeStart { 59 | try f(q.enqueueOne(_).unsafePerformSync) 60 | finally q.close.unsafePerformSync 61 | } 62 | 63 | q.dequeue.map(Channel.Publish.->) ++ Process.eval(res).flatMap(l => Process.emitAll(l.map(Channel.Requests.->))) 64 | } 65 | } 66 | 67 | private def execute( 68 | interpreter: Interpreter, 69 | msg: ParsedMessage[ShellRequest.Execute] 70 | )(implicit 71 | pool: ExecutorService 72 | ): Process[Task, (Channel, Message)] = { 73 | 74 | def ok(msg: ParsedMessage[_], executionCount: Int): Message = 75 | msg.reply("execute_reply", ShellReply.Execute(executionCount, Map.empty)) 76 | 77 | val content = msg.content 78 | val code = content.code 79 | val silent = content.silent.exists(x => x) 80 | 81 | if (code.trim.isEmpty) 82 | Process.emit(Channel.Requests -> ok(msg, interpreter.executionCount)) 83 | else { 84 | val start = Process.emitAll(Seq( 85 | Channel.Publish -> msg.publish( 86 | "execute_input", 87 | Publish.ExecuteInput( 88 | execution_count = interpreter.executionCount + 1, 89 | code = code 90 | ) 91 | ) 92 | )) 93 | 94 | start ++ publishing(msg) { pub => 95 | def error(msg: ParsedMessage[_], executionCount: Int, err: ShellReply.Error): Message = { 96 | pub(msg.publish("error", err)) 97 | 98 | msg.reply( 99 | "execute_reply", 100 | ShellReply.Error( 101 | err.ename, 102 | err.evalue, 103 | err.traceback, 104 | executionCount 105 | ) 106 | ) 107 | } 108 | 109 | def _error(msg: ParsedMessage[_], executionCount: Int, err: String): Message = 110 | error(msg, executionCount, ShellReply.Error("", "", err.split("\n").toList, executionCount)) 111 | 112 | Seq(interpreter.interpret( 113 | code, 114 | if (silent) 115 | Some(_ => (), _ => ()) 116 | else 117 | Some( 118 | s => pub(msg.publish("stream", Publish.Stream(name = "stdout", text = s), ident = "stdout")), 119 | s => pub(msg.publish("stream", Publish.Stream(name = "stderr", text = s), ident = "stderr")) 120 | ), 121 | content.store_history getOrElse !silent, 122 | Some(msg) 123 | ) match { 124 | case value: Interpreter.Value if !silent => 125 | pub( 126 | if (interpreter.resultDisplay) 127 | msg.publish( 128 | "display_data", 129 | Publish.DisplayData( 130 | value.jsonMap, 131 | Map.empty 132 | ) 133 | ) 134 | else 135 | msg.publish( 136 | "execute_result", 137 | Publish.ExecuteResult( 138 | interpreter.executionCount, 139 | value.jsonMap, 140 | Map.empty 141 | ) 142 | ) 143 | ) 144 | 145 | ok(msg, interpreter.executionCount) 146 | 147 | case _: Interpreter.Value if silent => 148 | ok(msg, interpreter.executionCount) 149 | 150 | case Interpreter.NoValue => 151 | ok(msg, interpreter.executionCount) 152 | 153 | case exc @ Interpreter.Exception(name, message, _) => 154 | error(msg, interpreter.executionCount, ShellReply.Error(name, message, exc.traceBack)) 155 | 156 | case Interpreter.Error(errorMsg) => 157 | _error(msg, interpreter.executionCount, errorMsg) 158 | 159 | case Interpreter.Cancelled => 160 | msg.reply("execute_reply", ShellReply.Abort()) 161 | }) 162 | } 163 | } 164 | } 165 | 166 | private def isComplete( 167 | interpreter: Interpreter, 168 | msg: ParsedMessage[ShellRequest.IsComplete] 169 | ): Message = { 170 | 171 | val resp = interpreter.isComplete(msg.content.code) match { 172 | case None => 173 | ShellReply.IsComplete.Unknown 174 | case Some(Interpreter.IsComplete.Complete) => 175 | ShellReply.IsComplete.Complete 176 | case Some(Interpreter.IsComplete.Incomplete(indent)) => 177 | ShellReply.IsComplete.Incomplete(indent) 178 | case Some(Interpreter.IsComplete.Invalid) => 179 | ShellReply.IsComplete.Invalid 180 | } 181 | 182 | msg.reply( 183 | "is_complete_reply", 184 | resp 185 | ) 186 | } 187 | 188 | private def complete( 189 | interpreter: Interpreter, 190 | msg: ParsedMessage[ShellRequest.Complete] 191 | ): Message = { 192 | 193 | val pos = 194 | if (msg.content.cursor_pos >= 0) 195 | msg.content.cursor_pos 196 | else 197 | msg.content.code.length 198 | 199 | val (start, end, matches) = interpreter.complete(msg.content.code, pos) 200 | 201 | msg.reply( 202 | "complete_reply", 203 | ShellReply.Complete( 204 | matches.toList, 205 | start, 206 | end, 207 | Map.empty 208 | ) 209 | ) 210 | } 211 | 212 | private def kernelInfo( 213 | implementation: (String, String), 214 | banner: String, 215 | languageInfo: ShellReply.KernelInfo.LanguageInfo, 216 | helpLinks: Seq[(String, String)], 217 | msg: ParsedMessage[ShellRequest.KernelInfo.type] 218 | ): Message = 219 | msg.reply( 220 | "kernel_info_reply", 221 | ShellReply.KernelInfo( 222 | s"${Protocol.versionMajor}.${Protocol.versionMinor}", 223 | implementation._1, 224 | implementation._2, 225 | languageInfo, 226 | banner, 227 | if (helpLinks.isEmpty) None 228 | else Some { 229 | helpLinks.map { 230 | case (text, url) => 231 | ShellReply.KernelInfo.Link(text, url) 232 | }.toList 233 | } 234 | ) 235 | ) 236 | 237 | private def connect(connectReply: ShellReply.Connect, msg: ParsedMessage[ShellRequest.Connect.type]): Message = 238 | msg.reply( 239 | "connect_reply", 240 | connectReply 241 | ) 242 | 243 | private def shutdown(msg: ParsedMessage[ShellRequest.Shutdown]): Message = 244 | msg.reply( 245 | "shutdown_reply", 246 | ShellReply.Shutdown(restart = msg.content.restart) 247 | ) 248 | 249 | private def inspect(msg: ParsedMessage[ShellRequest.Inspect]): Message = 250 | msg.reply( 251 | "object_info_reply", 252 | ShellReply.Inspect(found = false, Map.empty, Map.empty) 253 | ) 254 | 255 | private def history(msg: ParsedMessage[ShellRequest.History]): Message = 256 | msg.reply( 257 | "history_reply", 258 | ShellReply.History.Default(Nil) 259 | ) 260 | 261 | private def single(m: Message) = Process.emit(Channel.Requests -> m) 262 | 263 | 264 | def apply( 265 | interpreter: Interpreter, 266 | connectReply: ShellReply.Connect, 267 | commHandler: (String, CommChannelMessage) => Unit, 268 | msg: Message 269 | )(implicit 270 | pool: ExecutorService 271 | ): Either[String, Process[Task, (Channel, Message)]] = try { 272 | 273 | msg.msgType.right.flatMap { 274 | case "connect_request" => 275 | msg.as[ShellRequest.Connect.type] { parsedMessage => 276 | single(connect(connectReply, parsedMessage)) 277 | } 278 | 279 | case "kernel_info_request" => 280 | msg.as[ShellRequest.KernelInfo.type] { parsedMessage => 281 | single(kernelInfo( 282 | interpreter.implementation, 283 | interpreter.banner, 284 | interpreter.languageInfo, 285 | interpreter.helpLinks, 286 | parsedMessage 287 | )) ++ { 288 | if (interpreter.initialized) 289 | Process.empty 290 | else 291 | busy(parsedMessage) { interpreter.init(); Process.empty } 292 | } 293 | } 294 | 295 | case "execute_request" => 296 | msg.as[ShellRequest.Execute] { parsedMessage => 297 | execute(interpreter, parsedMessage) 298 | } 299 | 300 | case "complete_request" => 301 | msg.as[ShellRequest.Complete] { parsedMessage => 302 | single(complete(interpreter, parsedMessage)) 303 | } 304 | 305 | case "is_complete_request" => 306 | msg.as[ShellRequest.IsComplete] { parsedMessage => 307 | single(isComplete(interpreter, parsedMessage)) 308 | } 309 | 310 | case "object_info_request" => 311 | msg.as[ShellRequest.Inspect] { parsedMessage => 312 | single(inspect(parsedMessage)) 313 | } 314 | 315 | case "shutdown_request" => 316 | msg.as[ShellRequest.Shutdown] { parsedMessage => 317 | single(shutdown(parsedMessage)) 318 | } 319 | 320 | case "history_request" => 321 | msg.as[ShellRequest.History] { parsedMessage => 322 | single(history(parsedMessage)) 323 | } 324 | 325 | case "comm_open" => 326 | msg.as[Comm.Open] { parsedMessage => 327 | val r = parsedMessage.content 328 | commHandler(r.comm_id, CommOpen(r.target_name, r.data.spaces2)) 329 | Process.halt 330 | } 331 | 332 | case "comm_msg" => 333 | msg.as[Comm.Message] { parsedMessage => 334 | val r = parsedMessage.content 335 | commHandler(r.comm_id, CommMessage(r.data.spaces2)) 336 | Process.halt 337 | } 338 | 339 | case "comm_close" => 340 | msg.as[Comm.Close] { parsedMessage => 341 | val r = parsedMessage.content 342 | commHandler(r.comm_id, CommClose(r.data.spaces2)) 343 | Process.halt 344 | } 345 | } 346 | } catch { 347 | case NonFatal(e) => 348 | logger.error(s"Exception while handling message\n$msg", e) 349 | Left(e.toString) 350 | } 351 | } 352 | --------------------------------------------------------------------------------