├── project
    ├── build.properties
    └── plugins.sbt
├── travis
    ├── setup-travis-credentials.sh
    ├── sync-to-sonatype.sh
    └── sign-released-files.sh
├── .gitignore
├── src
    ├── main
    │   └── scala
    │   │   └── io
    │   │       └── github
    │   │           └── lhotari
    │   │               └── akka
    │   │                   └── http
    │   │                       └── health
    │   │                           ├── HealthChecker.scala
    │   │                           ├── LowDiskSpaceDetector.scala
    │   │                           ├── HealthEndpoint.scala
    │   │                           └── LowMemoryDetector.scala
    └── test
    │   └── scala
    │       └── io
    │           └── github
    │               └── lhotari
    │                   └── akka
    │                       └── http
    │                           └── health
    │                               ├── LowDiskSpaceDetectorSpec.scala
    │                               ├── ProcessSpawner.scala
    │                               ├── HealthEndpointSpec.scala
    │                               └── LowMemoryDetectorSpec.scala
├── LICENSE
├── README.md
└── .travis.yml


/project/build.properties:
--------------------------------------------------------------------------------
1 | sbt.version = 0.13.18
2 | 


--------------------------------------------------------------------------------
/project/plugins.sbt:
--------------------------------------------------------------------------------
1 | logLevel := Level.Warn
2 | 
3 | addSbtPlugin("me.lessis" % "bintray-sbt" % "0.3.0")
4 | 


--------------------------------------------------------------------------------
/travis/setup-travis-credentials.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | 
 3 | # setup bintray credentials
 4 | mkdir $HOME/.bintray/
 5 | FILE=$HOME/.bintray/.credentials
 6 | cat <<EOF >$FILE
 7 | realm = Bintray API Realm
 8 | host = api.bintray.com
 9 | user = $BINTRAY_USER
10 | password = $BINTRAY_API_KEY
11 | EOF
12 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | *.class
 2 | *.log
 3 | *.conf
 4 | scm-source.json
 5 | 
 6 | # sbt specific
 7 | .cache
 8 | .history
 9 | .lib/
10 | dist/*
11 | target/
12 | lib_managed/
13 | src_managed/
14 | project/boot/
15 | project/plugins/project/
16 | 
17 | # Scala-IDE specific
18 | .scala_dependencies
19 | .worksheet
20 | 
21 | # Intellij specific
22 | .idea
23 | 
24 | scm-source.json
25 | 


--------------------------------------------------------------------------------
/src/main/scala/io/github/lhotari/akka/http/health/HealthChecker.scala:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2017 the original author or authors.
 3 |  *
 4 |  * This software may be modified and distributed under the terms
 5 |  * of the MIT license.  See the LICENSE file for details.
 6 |  */
 7 | 
 8 | package io.github.lhotari.akka.http.health
 9 | 
10 | trait HealthChecker {
11 |   def start(): Unit = {}
12 | 
13 |   def stop(): Unit = {}
14 | 
15 |   def isHealthy(): Boolean
16 | }
17 | 


--------------------------------------------------------------------------------
/travis/sync-to-sonatype.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | NAME=${1:-$(cat build.sbt | egrep '^name :=' | awk -F\" '{ print $2 }')}
3 | VERSION=${2:-$(cat build.sbt | egrep '^version :=' | awk -F\" '{ print $2 }')}
4 | echo "Syncing files to Sonatype OSS..."
5 | curl "-u$BINTRAY_USER:$BINTRAY_API_KEY" -H "Content-Type: application/json" \
6 | -d "{\"username\": \"$SONATYPE_USER\", \"password\": \"$SONATYPE_PASS\", \"close\": \"1\"}" \
7 | -X POST "https://api.bintray.com/maven_central_sync/$BINTRAY_USER/releases/$NAME/versions/$VERSION"
8 | echo -e "\nDone."
9 | 


--------------------------------------------------------------------------------
/travis/sign-released-files.sh:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env bash
 2 | NAME=${1:-$(cat build.sbt | egrep '^name :=' | awk -F\" '{ print $2 }')}
 3 | VERSION=${2:-$(cat build.sbt | egrep '^version :=' | awk -F\" '{ print $2 }')}
 4 | echo "Signing released files..."
 5 | curl -H "X-GPG-PASSPHRASE: $GPG_PASSPHRASE" "-u$BINTRAY_USER:$BINTRAY_API_KEY" -X POST "https://api.bintray.com/gpg/$BINTRAY_USER/releases/$NAME/versions/$VERSION"
 6 | echo -e "\nDone."
 7 | echo "Publishing signature files..."
 8 | curl "-u$BINTRAY_USER:$BINTRAY_API_KEY" -X POST "https://api.bintray.com/content/$BINTRAY_USER/releases/$NAME/$VERSION/publish"
 9 | echo -e "\nDone."
10 | 


--------------------------------------------------------------------------------
/src/main/scala/io/github/lhotari/akka/http/health/LowDiskSpaceDetector.scala:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2017 the original author or authors.
 3 |  *
 4 |  * This software may be modified and distributed under the terms
 5 |  * of the MIT license.  See the LICENSE file for details.
 6 |  */
 7 | 
 8 | package io.github.lhotari.akka.http.health
 9 | 
10 | import java.io.File
11 | 
12 | class LowDiskSpaceDetector(val thresholdMB: Int = 50, val path: File = new File(".")) extends HealthChecker {
13 |   val threholdInBytes = thresholdMB * 1024 * 1024
14 | 
15 |   override def isHealthy(): Boolean = {
16 |     path.getFreeSpace > threholdInBytes
17 |   }
18 | }
19 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | Copyright (c) 2017 Lari Hotari
 3 | 
 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 5 | 
 6 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
 7 | 
 8 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 9 | 
10 | 
11 | 


--------------------------------------------------------------------------------
/src/test/scala/io/github/lhotari/akka/http/health/LowDiskSpaceDetectorSpec.scala:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2017 the original author or authors.
 3 |  *
 4 |  * This software may be modified and distributed under the terms
 5 |  * of the MIT license.  See the LICENSE file for details.
 6 |  */
 7 | 
 8 | package io.github.lhotari.akka.http.health
 9 | 
10 | import java.io.File
11 | 
12 | import org.mockito.Mockito._
13 | import org.scalatest.{FunSpec, Matchers}
14 | import org.scalatestplus.mockito.MockitoSugar
15 | 
16 | class LowDiskSpaceDetectorSpec extends FunSpec with MockitoSugar with Matchers with ProcessSpawner {
17 |   describe("low diskspace detector") {
18 |     val path = mock[File]
19 |     val thresholdMB = 10
20 |     val lowDiskSpaceDetector = new LowDiskSpaceDetector(thresholdMB = thresholdMB, path = path)
21 | 
22 |     it("should detect low diskspace") {
23 |       when(path.getFreeSpace).thenReturn(thresholdMB * 1024 * 1024 - 1)
24 |       lowDiskSpaceDetector.isHealthy() should equal(false)
25 |     }
26 | 
27 |     it("should detect when diskspace is healthy") {
28 |       when(path.getFreeSpace).thenReturn(thresholdMB * 1024 * 1024 + 1)
29 |       lowDiskSpaceDetector.isHealthy() should equal(true)
30 |     }
31 |   }
32 | }
33 | 


--------------------------------------------------------------------------------
/src/test/scala/io/github/lhotari/akka/http/health/ProcessSpawner.scala:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2017 the original author or authors.
 3 |  *
 4 |  * This software may be modified and distributed under the terms
 5 |  * of the MIT license.  See the LICENSE file for details.
 6 |  */
 7 | 
 8 | package io.github.lhotari.akka.http.health
 9 | 
10 | import java.io.ByteArrayOutputStream
11 | import java.lang.System.getProperty
12 | import java.net.{URL, URLClassLoader}
13 | 
14 | import org.apache.commons.io.IOUtils
15 | 
16 | import scala.collection.JavaConverters._
17 | import scala.reflect.runtime.universe._
18 | 
19 | case class ProcessResult(retval: Integer, output: String)
20 | 
21 | trait ProcessSpawner {
22 |   lazy val classpath = resolveClassPath()
23 |   val sep = getProperty("file.separator")
24 |   val javaExecutablePath = getProperty("java.home") + sep + "bin" + sep + "java"
25 | 
26 |   private def resolveClassPath() = {
27 |     getClass.getClassLoader match {
28 |       case urlClassLoader: URLClassLoader =>
29 |         urlClassLoader.getURLs.collect {
30 |           case url: URL => url.getFile
31 |         }.mkString(getProperty("path.separator"))
32 |       case _ =>
33 |         getProperty("java.class.path")
34 |     }
35 |   }
36 | 
37 |   def executeInSeparateProcess[T](mainClassType: T, maxMemoryMB: Integer = 100, extraJvmOpts: Seq[String] = Nil, args: Seq[String] = Nil)(implicit tag: WeakTypeTag[T]): ProcessResult = {
38 |     val className = tag.tpe.termSymbol.fullName
39 |     val processBuilder = new ProcessBuilder(javaExecutablePath).redirectErrorStream(true)
40 |     val commands = processBuilder.command()
41 |     commands.add(s"-Xmx${maxMemoryMB}m")
42 |     commands.addAll(extraJvmOpts.asJava)
43 |     commands.add("-cp")
44 |     commands.add(classpath)
45 |     commands.add(className)
46 |     commands.addAll(args.asJava)
47 |     println(String.join(" ", commands))
48 |     val process = processBuilder.start()
49 |     val output = new ByteArrayOutputStream()
50 |     IOUtils.copy(process.getInputStream, output)
51 |     ProcessResult(process.waitFor(), output.toString())
52 |   }
53 | }
54 | 


--------------------------------------------------------------------------------
/src/test/scala/io/github/lhotari/akka/http/health/HealthEndpointSpec.scala:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2017 the original author or authors.
 3 |  *
 4 |  * This software may be modified and distributed under the terms
 5 |  * of the MIT license.  See the LICENSE file for details.
 6 |  */
 7 | 
 8 | package io.github.lhotari.akka.http.health
 9 | 
10 | import akka.http.scaladsl.model.StatusCodes
11 | import akka.http.scaladsl.server.Route
12 | import akka.http.scaladsl.testkit.ScalatestRouteTest
13 | import org.mockito.Mockito._
14 | import org.scalatest.{FunSpec, Matchers}
15 | import org.scalatestplus.mockito.MockitoSugar
16 | 
17 | class HealthEndpointSpec extends FunSpec with Matchers with ScalatestRouteTest with MockitoSugar with HealthEndpoint {
18 |   val mockChecker1 = mock[HealthChecker]
19 |   val mockChecker2 = mock[HealthChecker]
20 | 
21 |   override protected def createCheckers(): Seq[HealthChecker] = Seq(mockChecker1, mockChecker2)
22 | 
23 |   describe("health endpoint") {
24 |     it("should complete successfully when all checks are ok") {
25 |       checkers.foreach(checker => when(checker.isHealthy()).thenReturn(true))
26 |       Get("/health") ~> Route.seal(createHealthRoute()) ~> check {
27 |         status shouldEqual StatusCodes.OK
28 |       }
29 |     }
30 | 
31 |     it("should complete successfully when a different endpoint is specified") {
32 |       checkers.foreach(checker => when(checker.isHealthy()).thenReturn(true))
33 |       Get("/another-endpoint") ~> Route.seal(createHealthRoute("another-endpoint")) ~> check {
34 |         status shouldEqual StatusCodes.OK
35 |       }
36 |     }
37 | 
38 |     it("should return error when the wrong endpoint is specified") {
39 |       checkers.foreach(checker => when(checker.isHealthy()).thenReturn(true))
40 |       Get("/health") ~> Route.seal(createHealthRoute("another-endpoint")) ~> check {
41 |         status shouldEqual StatusCodes.NotFound
42 |       }
43 |     }
44 | 
45 |     it("should return error when a check fails") {
46 |       when(mockChecker2.isHealthy()).thenReturn(false)
47 |       Get("/health") ~> Route.seal(createHealthRoute()) ~> check {
48 |         status shouldEqual StatusCodes.ServiceUnavailable
49 |       }
50 |     }
51 | 
52 |     it("should have started each checker exactly once") {
53 |       checkers.foreach(checker => verify(checker).start())
54 |     }
55 |   }
56 | 
57 | }
58 | 


--------------------------------------------------------------------------------
/src/main/scala/io/github/lhotari/akka/http/health/HealthEndpoint.scala:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Copyright 2017 the original author or authors.
 3 |  *
 4 |  * This software may be modified and distributed under the terms
 5 |  * of the MIT license.  See the LICENSE file for details.
 6 |  */
 7 | 
 8 | package io.github.lhotari.akka.http.health
 9 | 
10 | import java.util.concurrent.atomic.AtomicBoolean
11 | 
12 | import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpResponse, StatusCodes}
13 | import akka.http.scaladsl.server.Directives._
14 | import akka.http.scaladsl.server.Route
15 | 
16 | import scala.concurrent.{ExecutionContext, Future}
17 | 
18 | trait HealthEndpoint {
19 | 
20 |   protected lazy val checkers = createCheckers
21 | 
22 |   protected def createCheckers = {
23 |     Seq(new LowDiskSpaceDetector(), new LowMemoryDetector())
24 |   }
25 | 
26 |   private lazy val successResponse: HttpResponse = createSuccessResponse
27 | 
28 |   protected def decorateResponse(response: HttpResponse) = response
29 | 
30 |   protected def createSuccessResponse = {
31 |     decorateResponse(HttpResponse(entity = HttpEntity(ContentTypes.`text/plain(UTF-8)`, "OK")))
32 |   }
33 | 
34 |   private lazy val errorResponse: HttpResponse = createErrorResponse
35 | 
36 |   protected def createErrorResponse = {
37 |     decorateResponse(HttpResponse(status = StatusCodes.ServiceUnavailable))
38 |   }
39 | 
40 |   private val started = new AtomicBoolean(false)
41 | 
42 |   def createHealthRoute(endpoint: String = HealthEndpoint.DefaultEndpoint)(implicit executor: ExecutionContext): Route =
43 |     get {
44 |       path(endpoint) {
45 |         completeHealthCheck
46 |       }
47 |     }
48 | 
49 |   def completeHealthCheck(implicit executor: ExecutionContext) = {
50 |     complete {
51 |       Future {
52 |         if (isHealthy()) successResponse else errorResponse
53 |       }
54 |     }
55 |   }
56 | 
57 |   def start(): Unit = {
58 |     if (started.compareAndSet(false, true)) {
59 |       checkers.foreach(_.start())
60 |     }
61 |   }
62 | 
63 |   def stop(): Unit = {
64 |     if (started.compareAndSet(true, false)) {
65 |       checkers.foreach(_.stop())
66 |     }
67 |   }
68 | 
69 |   def isHealthy() = {
70 |     start()
71 |     checkers.forall(_.isHealthy())
72 |   }
73 | }
74 | 
75 | object HealthEndpoint extends HealthEndpoint {
76 |   lazy val defaultHealthChecker = new HealthEndpoint {}
77 | 
78 |   val DefaultEndpoint: String = "health"
79 | 
80 |   def createDefaultHealthRoute(endpoint: String = DefaultEndpoint)(implicit executor: ExecutionContext): Route = {
81 |     defaultHealthChecker.createHealthRoute(endpoint)
82 |   }
83 | }
84 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | [![Build Status](https://travis-ci.org/lhotari/akka-http-health.svg?branch=master)](https://travis-ci.org/lhotari/akka-http-health)
 2 | 
 3 | # akka-http-health
 4 | 
 5 | Library for adding `/health` endpoint checks for Akka Http applications.
 6 | 
 7 | Provides a default `/health` endpoint that checks:
 8 | * that the memory state is healthy
 9 | * that there is enough available disk space
10 | 
11 | A load balancer like AWS ELB can be configured to call the health endpoint and
12 | it could decide to destroy any unhealthy instances.
13 | 
14 | This is not a metrics library. Please take a look at [Kamon](http://kamon.io) if you are looking for a metrics and tracing library that's well suited for akka-http applications. 
15 | 
16 | ### Getting Started
17 | 
18 | #### Adding the dependency
19 | 
20 | sbt 0.13.6+
21 | ```
22 | resolvers += Resolver.jcenterRepo
23 | libraryDependencies += "io.github.lhotari" %% "akka-http-health" % "1.0.9"
24 | ```
25 | The library gets published on bintray/jcenter and synchronized from there to Sonatype OSS for staging to Maven central (starting since version `1.0.7`).
26 | It's recommended to use the jcenter repository since new releases will be available there instantly.
27 | 
28 | #### Simple use
29 | 
30 | The route instance created by calling `io.github.lhotari.akka.http.health.HealthEndpoint.createDefaultHealthRoute()` will handle `/health` with default settings.
31 | Append that route instance to the desired route binding.
32 | 
33 | Because of security concerns, it is generally adviced to serve the health endpoint on a separate port that isn't exposed to the public.
34 | 
35 | #### Advanced use
36 | 
37 | For customization, use the trait `io.github.lhotari.akka.http.health.HealthEndpoint` and override protected methods. Calling the `createHealthRoute()` method creates the route instance.
38 | 
39 | ### Contributing
40 | 
41 | Pull requests are welcome.
42 | 
43 | Some basic guidelines:
44 | * A few unit tests would help a lot as well - someone has to do that before the PR gets merged.
45 | * Please rebase the pull request branch against the current master.
46 | * When writing a commit message please follow [these conventions](http://chris.beams.io/posts/git-commit/#seven-rules).
47 | * If you are fixing an existing issue please add `Fixes gh-XXXX` at the end of the commit message (where `XXXX` is the issue number).
48 | * Add the license header to each source code file (see existing source code files for an example)
49 | 
50 | ### Contact
51 | 
52 | [Lari Hotari](mailto:lari@hotari.net)
53 | 
54 | ### License
55 | 
56 | The library is Open Source Software released under the MIT license. See the [LICENSE file](LICENSE) for details.
57 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | # Use container-based infrastructure
 2 | sudo: false
 3 | 
 4 | language: scala
 5 | 
 6 | scala:
 7 |   - 2.11.12
 8 |   - 2.12.10
 9 |   - 2.13.1
10 | 
11 | jdk:
12 |   - openjdk8
13 | 
14 | env:
15 |   global:
16 |     - BINTRAY_USER=lhotari
17 |     - secure: aWkIXPu5vps1Iafpb0R8Cfm/cP+8aZrphZnTPdYckQMi+mCELt6GR0VDQ3UgMyvEi32ACBdW+qRpf4D9rM6/2B2XWQci/HUMRqpUK5XrteBjddH1kTtLuI/PHlU4Mv4RYyeLS8lUD4z67jMS/WjL0dTnVvtnGn2PpIHeqzj1Ns4ekOO+JnxDEUcB4V7fDVSwitye73YcEPkU/b1Fsvlk7ADzqUHaGCSVI9jnUugmTHztD4fneR3f/IXVnWaiMRtL9DwyTEb1rzoZdEnYbDzafo1C4L/YgF5uM9FqZV83fQG/v8Xbu0j/u+nv/U4BopHZIRpC1vSb7wWWG3tC/6tOFHBdtY5KwQIorXf9LIeiWEqChN32wC/d7h5k21cavNsY+AyEGi+rkQ/Q7kh7XMiodP+lP4zvpWvjt08Ga9z/hRkYIkEL5I8xn7W/oWJrOTMKEZdzhLkSFK2fMvTIKD00Hher6ZTBN7hD2i69hrplVSyFrjDojOA3tuVgk1c0Ras0bDZ/ebc8U9CErGNmC9Q/FSvOoQMCLzfSTyGb+y/rqiU4ghNf5M3YRWA4joI+IyqZ33MdAVYxCPdUHMqk9mG4I1r/JlBcaAoSKc0Baff0pRua9ftJF94KwaYMEkaWsC8CSODVQPTJespLRTdaDqxCYwVea3chqNHIAm2e95EO8Xc=
18 |     - secure: iqvgj6GEBnlFRwJYO2/fJ8BwM/4A377Qs7WQBjFbMSE/2D6x4jfG+7guDMS7VEK161nZhqNC1Qer6x7YESwiww2FH7GlKd4rrRQDeR37xTjPMW07r/ZYgaoz3BvndFHNki3yB3rh7vZ9srGtmnl+wsq2GPgkE+vPSQ5SFLkEBm/Y4AaApXi68uYyxRufyqMZUKxrlMmpu0eo7LEsLn4+QUmA4aF3BOkgXoLCBrufpLJKZIG/IYonNYYouj7X39q4NnjkAF+jQypZc3VhaZ31GLnC/oy2KRuxbCvJtS9hSOGeMbbgjvT19r5Vy14G+iFZiC0XLfStW8nj0lTBrf8As/rVCB94xkCJnHDNzKdPOHtD9bVRvM27gTxqXAf9hOAofHJynDeVqHXwTopEzSVxY7LdjD3x9zlzW51HdHA+kwe7DeaADpPlR/yuutXjssJ6dE91XWmX0YppzaP+Unf/BbGz7ow7sWdv2qaYHegAoHX/Y6VtXUp84i5eB4Cr4oXiJ6kFDnSsjlaYDkg0e6Oke5Z7M+zIfAWq2cX7idByeGdUvAPyMknLzB5OzvMn2HXmJ6xJjFRQpnDnMg/eR/Y9WDBxurcow0wTPmqTHN8+Wr9PUPHo8sbv59AWI29sXta9UJ+lo2FKIDtJ0ydmIsKTrxKxe6YjVMDs3maGWUdKu3s=
19 |     - SONATYPE_USER=lhotari
20 |     - secure: LBSCL6kIkQMbL6oPazmzmezYhhhGS/HC8djIxNbKC/+UcxdeVAGTcCfU3gqOzJA1JPnVSNTfjS0MXhPYkYKeCc6Z8dmy/3WTJvcP3gw+Erb9dVu/oB9tkAXdmtTXGS6vVdMOyJJoPVzkody/Qy2SJV9RFfNI+IekcO53Ynt+vqjJb6tKH8Bc+x3IgubRA8dp8cq0JcV7E84BG/bkaGCmS8pCDDtjdoS7xN3gFSO9pqQIQmzv2RcSqlpbdtcTz7ismFzReXFSepP8wgon2WL3JCqGyJyUwFWoxgtcah4movWZPOF7LDoMqxfVqvfCZ+b+CdiJ5P3NlM2zYtO1Tn5lUpYBbXBPn8Se9QncbcrJuxvJtkhT6mreA1ebmWAfWZjpjcritvhW11uaSjqrTewC0+voZhKKVRaohbg2U5u9e6uGtDx+gUZ5qqLngpPOXh2lib51LlxeV7u24LPcY0wDaI+a+If3gG8Kcu/PFErXspDLeTup5CH1+WJD2Ti6mgf4eVW+OR7QTAsSw5WGO2XgISBeTJ/X14XsPT4yYmkA9DBFiXK3/gjsrbDnqGo4k7MfaUORdhfNGjS/Di0bRd6jtrJa3bVKvRUnVXq9if47tQciFNIyGW2iAt+EZfC0L4NN4DNSl5JW9zk1lezC1DMvW6HFsFHYqsFALA2WAY4YNZw=
21 | 
22 | # These directories are cached to S3 at the end of the build
23 | cache:
24 |   directories:
25 |     - $HOME/.ivy2/cache
26 |     - $HOME/.sbt/boot/
27 | 
28 | before_cache:
29 |   # Tricks to avoid unnecessary cache updates
30 |   - find $HOME/.ivy2 -name "ivydata-*.properties" -delete
31 |   - find $HOME/.sbt -name "*.lock" -delete
32 | 
33 | script:
34 |   - sbt ++$TRAVIS_SCALA_VERSION test
35 | 
36 | after_success:
37 |   - if [[ $TRAVIS_TAG =~ v[0-9]+\.[0-9]+\.[0-9]+ ]]; then
38 |       ./travis/setup-travis-credentials.sh;
39 |       sbt ++$TRAVIS_SCALA_VERSION publish && ./travis/sign-released-files.sh && ./travis/sync-to-sonatype.sh;
40 |     fi
41 | 


--------------------------------------------------------------------------------
/src/main/scala/io/github/lhotari/akka/http/health/LowMemoryDetector.scala:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright 2017 the original author or authors.
  3 |  *
  4 |  * This software may be modified and distributed under the terms
  5 |  * of the MIT license.  See the LICENSE file for details.
  6 |  */
  7 | 
  8 | package io.github.lhotari.akka.http.health
  9 | 
 10 | import java.lang.management._
 11 | import java.util.concurrent.atomic.AtomicBoolean
 12 | import java.util.logging.{Level, Logger}
 13 | import javax.management.openmbean.CompositeData
 14 | import javax.management.{Notification, NotificationBroadcaster, NotificationListener}
 15 | 
 16 | import com.sun.management.GarbageCollectionNotificationInfo
 17 | 
 18 | import scala.jdk.CollectionConverters._
 19 | 
 20 | /**
 21 |   * Detects low memory condition by using JMX API for consuming memory usage and garbage collection notification events.
 22 |   *
 23 |   * Uses <a href="http://docs.oracle.com/javase/8/docs/api/java/lang/management/MemoryPoolMXBean.html#CollectionThreshold">MemoryPoolMXBean's JMX API</a> to set
 24 |   * collection usage threshold. The listener is registered to <a href="http://docs.oracle.com/javase/8/docs/api/java/lang/management/MemoryMXBean.html">MemoryMXBean</a>.
 25 |   * <a href="https://docs.oracle.com/javase/8/docs/jre/api/management/extension/com/sun/management/GarbageCollectionNotificationInfo.html">Garbage collection notification</a> API
 26 |   * is also used. This is used to detect if memory usage drops below the threshold after crossing the threshold.
 27 |   *
 28 |   */
 29 | class LowMemoryDetector(val occupiedHeapPercentageThreshold: Int = 90, val gcBeans: Seq[GarbageCollectorMXBean] = ManagementFactory.getGarbageCollectorMXBeans.asScala.toSeq, val memoryPoolBeans: Seq[MemoryPoolMXBean] = ManagementFactory.getMemoryPoolMXBeans.asScala.toSeq, val memoryBean: MemoryMXBean = ManagementFactory.getMemoryMXBean) extends HealthChecker {
 30 |   private val LOG: Logger = Logger.getLogger(classOf[LowMemoryDetector].getName)
 31 | 
 32 |   private val lowMemoryDetectedFlag: AtomicBoolean = new AtomicBoolean(false)
 33 |   private val started: AtomicBoolean = new AtomicBoolean(false)
 34 | 
 35 |   private lazy val tenuredSpaceMemoryPoolBean = findTenuredSpaceMemoryPoolBean()
 36 | 
 37 |   private lazy val tenuredSpaceGcBeans: Seq[GarbageCollectorMXBean] = findTenuredSpaceGcBeans()
 38 | 
 39 |   val gcListener: NotificationListener = new NotificationListener {
 40 |     override def handleNotification(notification: Notification, handback: Any) = {
 41 |       if (GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION == notification.getType) {
 42 |         handleGcNotification(GarbageCollectionNotificationInfo.from(notification.getUserData.asInstanceOf[CompositeData]))
 43 |       }
 44 |     }
 45 |   }
 46 | 
 47 |   val memoryListener: NotificationListener = new NotificationListener {
 48 |     override def handleNotification(notification: Notification, handback: scala.Any) = {
 49 |       if (MemoryNotificationInfo.MEMORY_COLLECTION_THRESHOLD_EXCEEDED == notification.getType) {
 50 |         handleMemoryCollectionThresholdExceeded(MemoryNotificationInfo.from(notification.getUserData.asInstanceOf[CompositeData]))
 51 |       }
 52 |     }
 53 |   }
 54 | 
 55 |   def findTenuredSpaceMemoryPoolBean(): MemoryPoolMXBean = {
 56 |     val filtered = memoryPoolBeans.filter(memoryPoolBean => memoryPoolBean.isCollectionUsageThresholdSupported && memoryPoolBean.getType == MemoryType.HEAP && isTenuredSpace(memoryPoolBean.getName))
 57 |     assert(filtered.length == 1, "Expecting a single tenured space memory pool bean.")
 58 |     filtered.head
 59 |   }
 60 | 
 61 |   def findTenuredSpaceGcBeans(): Seq[GarbageCollectorMXBean] = {
 62 |     gcBeans.filter(_.getMemoryPoolNames.exists(isTenuredSpace))
 63 |   }
 64 | 
 65 |   private def isTenuredSpace(name: String): Boolean = name.endsWith("Old Gen") || name.endsWith("Tenured Gen")
 66 | 
 67 |   override def start(): Unit = {
 68 |     if (started.compareAndSet(false, true)) {
 69 |       registerGcListeners()
 70 |       applyTenuredSpaceUsageThreshold()
 71 |       registerMemoryBeanListener()
 72 |     }
 73 |   }
 74 | 
 75 |   override def stop(): Unit = {
 76 |     if (started.compareAndSet(true, false)) {
 77 |       unregisterGcListeners()
 78 |       unregisterMemoryBeanListener()
 79 |     }
 80 |   }
 81 | 
 82 |   private def registerGcListeners(): Unit = {
 83 |     var listenerAdded = false
 84 |     for (gcBean <- tenuredSpaceGcBeans) {
 85 |       gcBean.asInstanceOf[NotificationBroadcaster].addNotificationListener(gcListener, null, null)
 86 |       listenerAdded = true
 87 |     }
 88 |     if (!listenerAdded) {
 89 |       LOG.warning("Cannot find GarbageCollectorMXBean for tenured space.")
 90 |     }
 91 |   }
 92 | 
 93 |   private def unregisterGcListeners(): Unit = {
 94 |     for (gcBean <- tenuredSpaceGcBeans) {
 95 |       gcBean.asInstanceOf[NotificationBroadcaster].removeNotificationListener(gcListener)
 96 |     }
 97 |   }
 98 | 
 99 |   private def applyTenuredSpaceUsageThreshold() = {
100 |     val usageThreshold = occupiedHeapPercentageThreshold.toLong * tenuredSpaceMemoryPoolBean.getUsage.getMax / 100L
101 |     if (LOG.isLoggable(Level.INFO)) {
102 |       LOG.info(s"Setting threshold for ${tenuredSpaceMemoryPoolBean.getName} to ${usageThreshold} of ${tenuredSpaceMemoryPoolBean.getUsage.getMax} (${occupiedHeapPercentageThreshold}%)")
103 |     }
104 |     tenuredSpaceMemoryPoolBean.setCollectionUsageThreshold(usageThreshold)
105 |   }
106 | 
107 |   private def registerMemoryBeanListener() = {
108 |     memoryBean.asInstanceOf[NotificationBroadcaster].addNotificationListener(memoryListener, null, null)
109 |   }
110 | 
111 |   private def unregisterMemoryBeanListener() = {
112 |     memoryBean.asInstanceOf[NotificationBroadcaster].removeNotificationListener(memoryListener)
113 |   }
114 | 
115 |   protected def handleGcNotification(info: GarbageCollectionNotificationInfo): Unit = {
116 |     val spaces = info.getGcInfo.getMemoryUsageAfterGc.asScala
117 |     for ((spaceName, space) <- spaces; if (isTenuredSpace(spaceName) && space.getMax > 0)) {
118 |       val percentUsed: Long = 100L * space.getUsed / space.getMax
119 |       if (percentUsed < occupiedHeapPercentageThreshold && lowMemoryDetectedFlag.compareAndSet(true, false)) {
120 |         exitedLowMemoryState(space)
121 |       }
122 |     }
123 |   }
124 | 
125 |   protected def handleMemoryCollectionThresholdExceeded(info: MemoryNotificationInfo): Unit = {
126 |     if (lowMemoryDetectedFlag.compareAndSet(false, true)) {
127 |       enteredLowMemoryState(info.getUsage)
128 |     }
129 |   }
130 | 
131 |   protected def enteredLowMemoryState(space: MemoryUsage) = {
132 |     logMemoryStateChange("Low memory state detected.", space)
133 |   }
134 | 
135 |   protected def exitedLowMemoryState(space: MemoryUsage) = {
136 |     logMemoryStateChange("Memory state back to healthy.", space)
137 |   }
138 | 
139 |   private def logMemoryStateChange(description: String, space: MemoryUsage) = {
140 |     val msg = s"${description} tenured space usage ${space.getUsed} / ${space.getMax}"
141 |     logErrorMessage(msg)
142 |   }
143 | 
144 |   protected def logErrorMessage(message: String) = {
145 |     System.err.println(message)
146 |     LOG.warning(message)
147 |   }
148 | 
149 |   def lowMemoryDetected = lowMemoryDetectedFlag.get()
150 | 
151 |   override def isHealthy(): Boolean = !lowMemoryDetected
152 | }
153 | 


--------------------------------------------------------------------------------
/src/test/scala/io/github/lhotari/akka/http/health/LowMemoryDetectorSpec.scala:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright 2017 the original author or authors.
  3 |  *
  4 |  * This software may be modified and distributed under the terms
  5 |  * of the MIT license.  See the LICENSE file for details.
  6 |  */
  7 | 
  8 | package io.github.lhotari.akka.http.health
  9 | 
 10 | import java.lang.management._
 11 | import java.util.concurrent.ConcurrentMap
 12 | import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger}
 13 | import javax.management.{Notification, NotificationBroadcaster, NotificationFilter, NotificationListener}
 14 | 
 15 | import com.google.common.collect.MapMaker
 16 | import com.sun.management.{GarbageCollectionNotificationInfo, GcInfo}
 17 | import org.mockito.ArgumentCaptor
 18 | import org.mockito.Matchers._
 19 | import org.mockito.Mockito._
 20 | import org.mockito.internal.stubbing.answers.Returns
 21 | import org.scalatest.{FunSpec, Matchers}
 22 | import org.scalatestplus.mockito.MockitoSugar
 23 | import sun.management.MemoryNotifInfoCompositeData
 24 | 
 25 | import scala.collection.JavaConverters._
 26 | 
 27 | class LowMemoryDetectorSpec extends FunSpec with MockitoSugar with Matchers with ProcessSpawner {
 28 |   describe("low memory detector") {
 29 |     val gcBean: GarbageCollectorMXBean = createGcBean("PS Old Gen")
 30 |     val newGenGcBean: GarbageCollectorMXBean = createGcBean("New Gen")
 31 |     val memoryBean: MemoryMXBean = createMemoryBean()
 32 |     val usageMax = 10L * 1000L * 1000L * 1000L // about 10GB using 10-base
 33 |     val memoryPoolBean: MemoryPoolMXBean = createMemoryPoolBean("PS Old Gen", usageMax)
 34 | 
 35 |     val lowMemoryDetector = new LowMemoryDetector(occupiedHeapPercentageThreshold = 90, gcBeans = Seq(gcBean, newGenGcBean), memoryPoolBeans = Seq(memoryPoolBean), memoryBean = memoryBean)
 36 |     lowMemoryDetector.start()
 37 | 
 38 |     val gcListenerCapturer = ArgumentCaptor.forClass(classOf[NotificationListener])
 39 |     var gcNotificationListener: Option[NotificationListener] = None
 40 | 
 41 |     it("should register listener to tenure collector") {
 42 |       verify(gcBean.asInstanceOf[NotificationBroadcaster]).addNotificationListener(gcListenerCapturer.capture(), any(classOf[NotificationFilter]), any)
 43 |       gcNotificationListener = Some(gcListenerCapturer.getValue)
 44 |     }
 45 | 
 46 |     it("should not register listener to new collector") {
 47 |       verify(newGenGcBean.asInstanceOf[NotificationBroadcaster], never()).addNotificationListener(any(classOf[NotificationListener]), any(classOf[NotificationFilter]), any)
 48 |     }
 49 | 
 50 |     it("should set the threshold to the memory pool bean") {
 51 |       verify(memoryPoolBean).setCollectionUsageThreshold(9000000000L)
 52 |     }
 53 | 
 54 |     val memoryListenerCapturer = ArgumentCaptor.forClass(classOf[NotificationListener])
 55 |     var memoryNotificationListener: Option[NotificationListener] = None
 56 | 
 57 |     it("should register listener to memory bean") {
 58 |       verify(memoryBean.asInstanceOf[NotificationBroadcaster]).addNotificationListener(memoryListenerCapturer.capture(), any(classOf[NotificationFilter]), any)
 59 |       memoryNotificationListener = Some(memoryListenerCapturer.getValue)
 60 |     }
 61 | 
 62 |     it("should set low memory detected flag when the limit is exceeded") {
 63 |       memoryNotificationListener.get.handleNotification(createMemoryNotification(9100000000L, usageMax), null)
 64 |       assert(lowMemoryDetector.lowMemoryDetected, "Low memory should be now detected")
 65 |     }
 66 | 
 67 |     it("should unset low memory detected flag when memory usage goes under threshold") {
 68 |       gcNotificationListener.get.handleNotification(createGcNotification(8900000000L, usageMax), null)
 69 |       assert(!lowMemoryDetector.lowMemoryDetected, "Low memory not be flagged after healthy state")
 70 |     }
 71 | 
 72 |     it("should unregister listeners when it is stopped") {
 73 |       lowMemoryDetector.stop()
 74 |       verify(memoryBean.asInstanceOf[NotificationBroadcaster]).removeNotificationListener(memoryNotificationListener.get)
 75 |       verify(gcBean.asInstanceOf[NotificationBroadcaster]).removeNotificationListener(gcNotificationListener.get)
 76 |     }
 77 |   }
 78 | 
 79 |   private def createGcBean(memPoolName: String) = {
 80 |     val gcBean = mock[GarbageCollectorMXBean](withSettings().extraInterfaces(classOf[NotificationBroadcaster]))
 81 |     when(gcBean.getMemoryPoolNames).thenReturn(Array(memPoolName))
 82 |     gcBean
 83 |   }
 84 | 
 85 |   private def createMemoryPoolBean(memPoolName: String, usageMax: Long) = {
 86 |     val memoryPoolBean = mock[MemoryPoolMXBean]
 87 |     when(memoryPoolBean.getName).thenReturn(memPoolName)
 88 |     when(memoryPoolBean.isCollectionUsageThresholdSupported).thenReturn(true)
 89 |     when(memoryPoolBean.getType).thenReturn(MemoryType.HEAP)
 90 |     val memoryUsage = mock[MemoryUsage]
 91 |     when(memoryUsage.getMax).thenReturn(usageMax)
 92 |     when(memoryPoolBean.getUsage).thenReturn(memoryUsage)
 93 |     memoryPoolBean
 94 |   }
 95 | 
 96 |   private def createMemoryBean() = {
 97 |     mock[MemoryMXBean](withSettings().extraInterfaces(classOf[NotificationBroadcaster]))
 98 |   }
 99 | 
100 |   private def createGcNotification(usage: Long, usageMax: Long, gcCause: String = ""): Notification = {
101 |     val notification = mock[Notification]
102 |     when(notification.getType).thenReturn(GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION)
103 |     val gcInfo = mock[GcInfo]
104 |     val gcNotificationInfo = new GarbageCollectionNotificationInfo("gcName", "end of major GC", gcCause, gcInfo)
105 |     when(notification.getUserData).thenAnswer(new Returns(gcNotificationInfo.toCompositeData(null).asInstanceOf[Object]))
106 |     val memoryUsage = mock[MemoryUsage]
107 |     when(gcInfo.getMemoryUsageAfterGc).thenReturn(Map("PS Old Gen" -> memoryUsage).asJava)
108 |     when(memoryUsage.getUsed).thenReturn(usage)
109 |     when(memoryUsage.getMax).thenReturn(usageMax)
110 |     notification
111 |   }
112 | 
113 |   private def createMemoryNotification(usage: Long, usageMax: Long): Notification = {
114 |     val notification = mock[Notification]
115 |     when(notification.getType).thenReturn(MemoryNotificationInfo.MEMORY_COLLECTION_THRESHOLD_EXCEEDED)
116 |     val memoryUsage = mock[MemoryUsage]
117 |     when(memoryUsage.getUsed).thenReturn(usage)
118 |     when(memoryUsage.getMax).thenReturn(usageMax)
119 |     val memoryNotificationInfo = new MemoryNotificationInfo("PS Old Gen", memoryUsage, 1)
120 |     val userData: Object = MemoryNotifInfoCompositeData.toCompositeData(memoryNotificationInfo)
121 |     when(notification.getUserData).thenAnswer(new Returns(userData))
122 |     notification
123 |   }
124 | 
125 |   for ((extraDescription: String, extraArgs: Seq[String], retval: Int) <- Seq(("", Nil, 100), (" that becomes healthy after exceeding limit", Seq("dropcache"), 101))) {
126 |     describe(s"low memory detector in leaky application${extraDescription}") {
127 |       for ((collectorName, jvmArgs) <- Seq(("default", Nil), ("CMS", Seq("-XX:+UseConcMarkSweepGC")), ("G1GC", Seq("-XX:+UseG1GC")))) {
128 |         it(s"should detect a memory leak with ${collectorName} collector") {
129 |           val result = executeInSeparateProcess(mainClassType = LeakyApplication, extraJvmOpts = jvmArgs, maxMemoryMB = 200, args = extraArgs)
130 |           println(result.output)
131 |           result.retval should equal(retval)
132 |         }
133 |       }
134 |     }
135 |   }
136 | }
137 | 
138 | object LeakyApplication {
139 |   def main(args: Array[String]): Unit = {
140 |     val cachedropped = new AtomicBoolean(false)
141 |     val dropcache = args.length > 0 && args(0) == "dropcache"
142 | 
143 |     val strongMap: ConcurrentMap[Integer, Array[Byte]] = new MapMaker().makeMap()
144 | 
145 |     var retval = 100
146 | 
147 |     val lowMemoryDetector = new LowMemoryDetector(occupiedHeapPercentageThreshold = 70) {
148 |       override def handleGcNotification(info: GarbageCollectionNotificationInfo): Unit = {
149 |         println(s"GC notification action '${info.getGcAction}' cause '${info.getGcCause}' name '${info.getGcName}' duration ${info.getGcInfo.getDuration}")
150 |         super.handleGcNotification(info)
151 |       }
152 | 
153 |       override protected def enteredLowMemoryState(space: MemoryUsage): Unit = {
154 |         println("Entering low memory state")
155 |         super.enteredLowMemoryState(space)
156 |         if (dropcache) {
157 |           if (cachedropped.compareAndSet(false, true)) {
158 |             strongMap.clear()
159 |           } else {
160 |             System.exit(retval)
161 |           }
162 |         } else {
163 |           System.exit(retval)
164 |         }
165 |       }
166 | 
167 |       override protected def exitedLowMemoryState(space: MemoryUsage): Unit = {
168 |         println("Exited low memory state")
169 |         retval = 101
170 |         super.exitedLowMemoryState(space)
171 |       }
172 |     }
173 |     lowMemoryDetector.start()
174 | 
175 |     val weakMap: ConcurrentMap[Integer, Array[Byte]] = new MapMaker().weakValues().makeMap()
176 |     val counter = new AtomicInteger(0)
177 |     var delay = 10
178 |     var allocationSizeMB = 11
179 |     while (true) {
180 |       val bytes = Array.fill[Byte](1024 * 1024 * allocationSizeMB)(0)
181 |       weakMap.put(counter.incrementAndGet(), bytes)
182 |       if (counter.get() % 3 == 0) {
183 |         strongMap.put(counter.get(), bytes)
184 |       }
185 |       Thread.sleep(delay)
186 |     }
187 |   }
188 | }
189 | 


--------------------------------------------------------------------------------