├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── build.sbt ├── project ├── build.properties └── plugins.sbt ├── src └── main │ └── scala │ └── eu │ └── inn │ └── fluentd │ └── FluentdLogger.scala └── travis ├── build.sh ├── gpg-private.asc.gpg └── gpg-public.asc.gpg /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | *.idea 4 | *.idea_modules 5 | *.iml 6 | 7 | # mac os specific 8 | .DS_Store 9 | 10 | # sbt specific 11 | dist/* 12 | target/ 13 | lib_managed/ 14 | src_managed/ 15 | project/boot/ 16 | project/plugins/project/ 17 | 18 | # Scala-IDE specific 19 | .scala_dependencies 20 | .classpath 21 | .project 22 | .settings 23 | .cache 24 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | 3 | scala: 4 | - 2.11.8 5 | 6 | jdk: 7 | - oraclejdk8 8 | 9 | script: $TRAVIS_BUILD_DIR/travis/build.sh 10 | 11 | cache: 12 | directories: 13 | - $HOME/.m2 14 | - $HOME/.ivy2 15 | - $HOME/.sbt 16 | 17 | env: 18 | global: 19 | - secure: o0NAGYmhpsKBDknZ0ltZE+sJZMmasJb6H0jcogFG1kan9Q1Hu/5uq/87u+Iv8eMfdQbFyKmGuaokbiCxCusD3ugaeHkCY8Rxvdi9BycH2wgk+E4AA0N6yesJp4DhLfutHB2P2scP3hnEcRECBJxe1pvo7NWV0Hqx8jEOs1IIKWtDO0p9cHYK2QkN33RdWUDnw8EYthezmb60Lmanxt00qWIkuONnbrY/4ES8Fca3mzFeBSsS24d688tyNzP8JF5KRpSbaXxgCZ/QGdxwQ6C+mr8qV/d6otLOR5TvnRr8rzt5eGP+jxOlYN5V3gnpj+OtzdMBrv7f2i9uEjdMXneQIbIrYurwP2VkdNTXeehkjtgBKR/DLvrN/esb5EcVMNkVVw1LZwiXwCj4HSOtI+IznG/UKljHwLx7Vsi/nbY3Xxc8ys5UMrdfBC5ZpoHfJ9EVbNOGlnRbop9gthClFV2/b22SPcouU6r3oloH81T8xdKT3Yqfvy0eYnqDDvjV49yIVFm5u8fLz1DpZ3pO4t3UWSug5/vW9lGOnKGKXnGqbYylfZWXnH7Cir70Nb4aplDpzOARzw8quE61hXR6s/qCe72Qo9k64dWrOmMyyp47wcoYYDmb9EcK0x3W39dZ87b/Pkapevr4HCGlssClzKd0uWzSqZOYVO3KFqMpTixwLXE= 20 | - secure: mfBaXns+qDFXVOQ+LVw1bohERBn4t54DfIzIsiogkHsBoic3C2SoB6L3RFoVOp7fDHrdYpvWs5jYDvp6maVMfSHSR2Jrcq7QAGY+G3D44CG7LlhuJj8sNMtWH3KtgALcLQKM96k6FWRKo4aSWlk9lI6WE2OW40V4uxwVUIHeZcMIseJwdRiTCoDui7FhHRQUij4blRbiJfDkfAmK80ODPT5cLqof34zpNBdihJBDErZJPI+a2SXSxJLouuDmiwfKlgrDMlOhA2ydugBv8JD1xo4RB+lfp/EHxeqRruhPjk3NHFy1EOZKAcz/ulkzzgr4Ic23I+0ybJVA6TYP2qC/KtgsiGgaZA7CyTt7dciUw44otKJ6SkCQQ9u/zjv2Wewh3LWUCZD1dARQQBQ69jhfi2bHxFnzm/eNx+Pfka5BbnyGa2XIs+BYk4UCNKi+3qZk63gJQjAWc/IZ2BNAlcMzIkwWHZp+rG3s8JUEkXae+esjlmbTSajb+pR/PiJRr3jGS3cacdzwutyAokCJVGfuGl4SHryLSvAiaELmCflXByuKY1c9NIt9Qk5Vai8spvTkdaZnIwbeYu+J/b490vSYMzWV5SohXX3GdhwcoBIQHFrXTtFT498/KkPWbPCFwkSGU1NAUcSSnMCrUs/PVsrFO6eBeFC5BS8ku4uOIs4IJ1Q= 21 | - secure: kYYzI2ymR1+a6q503WEzKPV4hZVzWzOfvUjK9GWHGGSMRCV7LvQVy8EtB02pF0SNn7lXeDfSvT6hQxRl5RcQWYeFx3WBPx8YVVBIfHJZ55Kc4VrWKh1VvrDznbLR19kvuiYHfT7QyPZz75VBXaDTGkjI8nUpmwU76ncyB504aAGtFrMUISiJBHs8KjmDj4WTIGBYL4a7VODufB5VSo6fi8L9QCHu0pgBgke4ILBGpRHZMArAxjfEpFHZ3MwiPMzUGvocLD8nbI0umLMc4EKiN+/61PlZdrFJf/IDzcbVD6JGgPEFuNq9pD/MupvL3jdN53oLvbpuxANliYXA07d7+ub28ARPqs+tUmFhJNTQqz/+MFA5wJ33meHJTm9vEoQrcGslfV1Y9kzQNqklQTEefbH2y0bDdCG14CfwmwhTgRvw99zCSWVUikCqdiFrpT6jY9QEWODFS4+ODjd3KwidwK/N1IVD6SLSmF6nwrVme5Hl90ss/HTFcLw7wE9myyovHyfDtoYZgXDLE55/swEVC33RzXAczTfqde1K5Y19w5VULIUe6cnvxv7OCf4fG3svxIPvXV12b3wfQnoiE9ulEJoxE3OeLkRI9uuh6NEkK51K4OvUXmdKCHvRwCr+X1Eyi4yX9QN6J6qNf8s8FlLit9WxROs0F75NbdlAYLmzT20= 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, Innova Co s.a r.l 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, 5 | are permitted provided that the following conditions are met: 6 | 7 | Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | Redistributions in binary form must reproduce the above copyright notice, this 11 | list of conditions and the following disclaimer in the documentation and/or 12 | other materials provided with the distribution. 13 | 14 | Neither the name of the Innova Co s.a r.l nor the names of its 15 | contributors may be used to endorse or promote products derived from 16 | this software without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | fluentd-scala 2 | ============= 3 | 4 | [![Travis CI](https://travis-ci.org/kulikov/fluentd-scala.svg?branch=master)](https://travis-ci.org/kulikov/fluentd-scala) 5 | 6 | [Logback](http://logback.qos.ch) appender for fluentd http://fluentd.org/ based on akka IO 7 | 8 | 9 | ```xml 10 | 11 | 12 | my.project 13 | localhost 14 | 24224 15 | 16 | 17 | 18 | 19 | 20 | 21 | ``` 22 | 23 | Available on Maven Central. Settings for SBT: 24 | 25 | ```scala 26 | libraryDependencies += "eu.inn" %% "fluentd-scala" % "0.2.4" 27 | ``` 28 | 29 | 30 | #### Release new version 31 | 32 | ``` 33 | sbt +publishSigned sonatypeRelease 34 | ``` 35 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | organization := "eu.inn" 2 | 3 | name := "fluentd-scala" 4 | 5 | version := "0.2" 6 | 7 | scalaVersion := "2.11.11" 8 | 9 | crossScalaVersions := Seq("2.11.11", "2.12.2") 10 | 11 | scalacOptions ++= Seq( 12 | "-language:postfixOps", 13 | "-language:implicitConversions", 14 | "-feature", 15 | "-deprecation", 16 | "-unchecked", 17 | "-optimise", 18 | "-target:jvm-1.8", 19 | "-encoding", "UTF-8" 20 | ) 21 | 22 | javacOptions ++= Seq( 23 | "-source", "1.8", 24 | "-target", "1.8", 25 | "-encoding", "UTF-8", 26 | "-Xlint:unchecked", 27 | "-Xlint:deprecation" 28 | ) 29 | 30 | libraryDependencies ++= Seq( 31 | "com.typesafe.akka" %% "akka-actor" % "2.4.17", 32 | "org.msgpack" % "msgpack-scala_2.11" % "0.6.11", 33 | "ch.qos.logback" % "logback-classic" % "1.2.2" 34 | ) 35 | 36 | pomExtra := { 37 | https://github.com/InnovaCo/fluentd-scala 38 | 39 | 40 | BSD-style 41 | http://opensource.org/licenses/BSD-3-Clause 42 | repo 43 | 44 | 45 | 46 | git@github.com:InnovaCo/fluentd-scala.git 47 | scm:git:git@github.com:InnovaCo/fluentd-scala.git 48 | 49 | 50 | 51 | InnovaCo 52 | Innova Co S.a r.l 53 | https://github.com/InnovaCo 54 | 55 | 56 | kulikov 57 | Dmitry Kulikov 58 | https://github.com/kulikov 59 | 60 | 61 | } 62 | 63 | pgpSecretRing := file("./travis/gpg-private.asc.gpg") 64 | 65 | pgpPublicRing := file("./travis/gpg-public.asc.gpg") 66 | 67 | usePgpKeyHex("1FC91057C33D1A33") 68 | 69 | pgpPassphrase := Option(System.getenv().get("oss_gpg_passphrase")).map(_.toCharArray) 70 | 71 | credentials += Credentials("Sonatype Nexus Repository Manager", 72 | "oss.sonatype.org", 73 | System.getenv().get("sonatype_username"), 74 | System.getenv().get("sonatype_password")) 75 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.8 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") 2 | 3 | addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1") 4 | 5 | -------------------------------------------------------------------------------- /src/main/scala/eu/inn/fluentd/FluentdLogger.scala: -------------------------------------------------------------------------------- 1 | package eu.inn.fluentd 2 | 3 | import java.net.{InetAddress, InetSocketAddress} 4 | import scala.collection.JavaConverters._ 5 | import scala.collection.mutable.ListBuffer 6 | import scala.concurrent.duration._ 7 | import scala.util.Try 8 | 9 | import akka.actor._ 10 | import akka.io.{IO, Tcp} 11 | import akka.util.ByteString 12 | import ch.qos.logback.classic.pattern.CallerDataConverter 13 | import ch.qos.logback.classic.spi.{ILoggingEvent, ThrowableProxyUtil} 14 | import ch.qos.logback.core.UnsynchronizedAppenderBase 15 | import com.typesafe.config.ConfigFactory 16 | import org.msgpack.ScalaMessagePack 17 | 18 | 19 | class FluentdAppender extends UnsynchronizedAppenderBase[ILoggingEvent] { 20 | import eu.inn.fluentd.FluentdAppender._ 21 | 22 | private var appender: ActorRef = null 23 | 24 | private var tag = "default" 25 | private var remoteHost = "127.0.0.1" 26 | private var version = "" 27 | private var port = 24224 28 | 29 | override def start() { 30 | super.start() 31 | appender = actorSystem.actorOf(Props(classOf[FluentdLoggerActor], tag, remoteHost, port, version)) 32 | } 33 | 34 | override def append(eventObject: ILoggingEvent) { 35 | if (isStarted && appender != null) { 36 | appender ! eventObject 37 | } 38 | } 39 | 40 | override def stop() { 41 | try { 42 | super.stop() 43 | } finally { 44 | actorSystem.stop(appender) 45 | } 46 | } 47 | 48 | def setTag(tag: String) { 49 | this.tag = tag 50 | } 51 | 52 | def setRemoteHost(remoteHost: String) { 53 | this.remoteHost = remoteHost 54 | } 55 | 56 | def setVersion(version: String) { 57 | this.version = version 58 | } 59 | 60 | def setPort(port: Int) { 61 | this.port = port 62 | } 63 | } 64 | 65 | 66 | object FluentdAppender { 67 | lazy val actorSystem = ActorSystem( 68 | "fluentd-logger", 69 | ConfigFactory.parseString(""" 70 | akka.loggers = [] 71 | akka.log-dead-letters = off 72 | akka.daemonic = on 73 | akka.actor.default-mailbox.stash-capacity = 8192 74 | """) 75 | ) 76 | } 77 | 78 | 79 | class FluentdLoggerActor(tag: String, remoteHost: String, port: Int, version: String) extends Actor with Stash with ActorLogging { 80 | import context.dispatcher 81 | 82 | case object FlushBuffer 83 | 84 | private val messagePack = new ScalaMessagePack 85 | 86 | var writeErrors = 0 87 | 88 | val BufferSize = 100 89 | val MaxWriteError = 100 90 | 91 | val LocalHostName = Try(InetAddress.getLocalHost.getHostName).getOrElse("unknown-host") 92 | 93 | val buffer = ListBuffer.empty[List[Any]] 94 | buffer.sizeHint(BufferSize) 95 | 96 | context.system.scheduler.schedule(10 seconds, 10 seconds, self, FlushBuffer) 97 | 98 | override def preStart() { 99 | connect() 100 | } 101 | 102 | private def connect(delay: FiniteDuration = 0.second) { 103 | log.debug("Try connect to fluentd after {}", delay) 104 | context.system.scheduler.scheduleOnce(delay, IO(Tcp)(context.system), Tcp.Connect(new InetSocketAddress(remoteHost, port))) 105 | } 106 | 107 | def receive = { 108 | case _: Tcp.Connected ⇒ 109 | log.info("Connected to fluentd") 110 | 111 | sender ! Tcp.Register(self) 112 | context.become(connected(sender())) 113 | unstashAll() 114 | 115 | case e @ Tcp.CommandFailed(_: Tcp.Connect) ⇒ 116 | log.warning("Error connect to fluentd agent: {}", e) 117 | connect(delay = 5 seconds) 118 | 119 | case e: Tcp.Event ⇒ 120 | log.warning("Unexpected TCP Event {}", e.getClass) 121 | 122 | case _: ILoggingEvent ⇒ 123 | try stash() catch { 124 | case e: StashOverflowException ⇒ // skip 125 | } 126 | } 127 | 128 | def connected(conn: ActorRef): Receive = { 129 | case event: ILoggingEvent ⇒ 130 | val data = event.getMDCPropertyMap.asScala ++ Map( 131 | "message" → event.getFormattedMessage, 132 | "level" → event.getLevel.toString, 133 | "logger" → event.getLoggerName, 134 | "thread" → event.getThreadName, 135 | "timemillis" → event.getTimeStamp.toString, 136 | "host" → LocalHostName 137 | ) 138 | 139 | if (event.getMarker != null) { 140 | data("marker") = event.getMarker.getName 141 | } 142 | 143 | if (event.hasCallerData) { 144 | data("caller") = new CallerDataConverter().convert(event) 145 | } 146 | 147 | if (event.getThrowableProxy != null) { 148 | data("throwable") = ThrowableProxyUtil.asString(event.getThrowableProxy).take(2048) 149 | } 150 | 151 | if (version != "") { 152 | data("version") = version 153 | } 154 | 155 | buffer += List(event.getTimeStamp / 1000, data) 156 | 157 | if (buffer.size >= BufferSize) { 158 | flushBuffer(conn) 159 | } 160 | 161 | case FlushBuffer ⇒ 162 | if (buffer.nonEmpty) { 163 | flushBuffer(conn) 164 | } 165 | 166 | case Tcp.CommandFailed(write: Tcp.Write) ⇒ 167 | log.info("Error write message to fluentd, retry") 168 | 169 | writeErrors += 1 170 | if (writeErrors > MaxWriteError) { 171 | log.warning("Too many writer errors, close current connection and reconnect") 172 | writeErrors = 0 173 | conn ! Tcp.Close 174 | } else { 175 | conn ! write // retry 176 | } 177 | 178 | case e: Tcp.ConnectionClosed ⇒ 179 | log.warning("Error write to fluentd: {}", e) 180 | context.unbecome() 181 | connect(delay = 5 seconds) 182 | } 183 | 184 | private def flushBuffer(conn: ActorRef) { 185 | conn ! Tcp.Write(ByteString(messagePack.write(List(tag, buffer.toList)))) 186 | buffer.clear() 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /travis/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_BRANCH" == "master" ]]; then 4 | echo "$oss_gpg_passphrase" | gpg --passphrase-fd 0 ./travis/gpg-public.asc.gpg 5 | echo "$oss_gpg_passphrase" | gpg --passphrase-fd 0 ./travis/gpg-private.asc.gpg 6 | 7 | sbt ';set version <<= (version)(_ + ".'${TRAVIS_BUILD_NUMBER:-0}'")' +test +publishSigned sonatypeReleaseAll 8 | fi -------------------------------------------------------------------------------- /travis/gpg-private.asc.gpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InnovaCo/fluentd-scala/b778fc69db1e117580d3b39557927e2e416b681e/travis/gpg-private.asc.gpg -------------------------------------------------------------------------------- /travis/gpg-public.asc.gpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InnovaCo/fluentd-scala/b778fc69db1e117580d3b39557927e2e416b681e/travis/gpg-public.asc.gpg --------------------------------------------------------------------------------