├── .gitignore ├── LICENSE.md ├── README.md ├── build.sbt ├── project └── build.properties ├── src └── ooyala │ └── common │ └── metrics_storm │ ├── MetricsStorm.scala │ └── MetricsStormHooks.scala └── test └── ooyala └── common └── metrics_storm └── MetricsStormSpec.scala /.gitignore: -------------------------------------------------------------------------------- 1 | # Standard sbt ignores 2 | target/ 3 | src_managed/ 4 | project/boot/ 5 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright 2013 Ooyala, Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Intro 2 | ===== 3 | The metrics_storm project adds easy performance metrics collection infrastructure, as well as an 4 | embedded web console, to any Storm (http://github.com/nathanmarz/storm) topology, based on Coda Hale's 5 | excellent metrics library (http://metrics.codahale.com). At a minimum, the following is supported: 6 | 7 | * All metrics (gauges, meters, etc.) will be exported via JMX 8 | * A web console that exposes these routes: 9 | * /metricz - all metrics as JSON blobs, including JVM uptime & other info 10 | * /healthz 11 | * /threadz - JVM thread information 12 | 13 | In addition, if you enable MetricsStormHooks, then you get automatic emit and ack rate meters for every bolt 14 | and spout in your topology. To enable, place the following in your storm.yaml: 15 | 16 | topology.auto.task.hooks: 17 | - "ooyala.common.metrics_storm.MetricsStormHooks" 18 | 19 | You can also programmatically inject the above configuration, like this: 20 | 21 | config.put(Config.TOPOLOGY_AUTO_TASK_HOOKS, List(classOf[MetricsStormHooks].getName).asJava) 22 | 23 | Refer to https://github.com/nathanmarz/storm/wiki/Hooks for more information. 24 | 25 | Web Console Configuration 26 | ========================= 27 | 28 | Set the following parameter in storm.yaml, listing one port per supervisor.slot.ports, like this: 29 | 30 | worker.webconsole.ports: 31 | - 7000 32 | - 7001 33 | 34 | If this parameter is not set, the web console port defaults to 7070. 35 | 36 | Building 37 | ======== 38 | 39 | You can build this project using SBT 0.11.2 or higher. To run the unit tests: 40 | 41 | sbt test 42 | 43 | To create a jar in target/scala-*/: 44 | 45 | sbt package 46 | 47 | To publish to ~/.ivy2/local/, including POM files: 48 | 49 | sbt publish-local 50 | 51 | Contributing 52 | ============ 53 | Contributions via pull request are very welcome. 54 | 55 | License 56 | ======= 57 | Apache 2.0, see LICENSE.md 58 | 59 | Copyright(c) 2013, Ooyala, Inc. 60 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | name := "metrics_storm" 2 | 3 | // Remove -SNAPSHOT from the version before publishing a release. Don't forget to change the version to 4 | // $(NEXT_VERSION)-SNAPSHOT afterwards! 5 | version := "0.0.7-SNAPSHOT" 6 | 7 | scalaVersion := "2.9.1" 8 | 9 | scalacOptions += "-Yresolve-term-conflict:package" 10 | 11 | unmanagedSourceDirectories in Compile <++= Seq(baseDirectory(_ / "src" )).join 12 | 13 | unmanagedSourceDirectories in Test <++= Seq(baseDirectory(_ / "test" )).join 14 | 15 | libraryDependencies ++= Seq( 16 | // slf4j >= 1.6 is needed so jetty logging won't throw an exception 17 | "org.slf4j" % "slf4j-api" % "1.6.4", 18 | "org.slf4j" % "slf4j-log4j12" % "1.6.4", 19 | "com.yammer.metrics" % "metrics-core" % "2.1.2", 20 | "com.yammer.metrics" % "metrics-servlet" % "2.1.2", 21 | "javax.servlet" % "servlet-api" % "2.5" 22 | ) 23 | 24 | // Testing deps 25 | libraryDependencies ++= Seq("org.scalatest" %% "scalatest" % "1.9.1" % "test", 26 | "org.mockito" % "mockito-all" % "1.9.0" % "test") 27 | 28 | resolvers ++= Seq("clojars" at "http://clojars.org/repo/", 29 | "clojure-releases" at "http://build.clojure.org/releases") 30 | 31 | libraryDependencies += "storm" % "storm" % "0.8.2" 32 | 33 | // Testing deps 34 | libraryDependencies ++= Seq() 35 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.11.2 2 | -------------------------------------------------------------------------------- /src/ooyala/common/metrics_storm/MetricsStorm.scala: -------------------------------------------------------------------------------- 1 | package ooyala.common.metrics_storm 2 | 3 | import collection.JavaConversions._ 4 | import collection.mutable.HashMap 5 | import java.util.concurrent.TimeUnit 6 | 7 | import backtype.storm.task.TopologyContext 8 | import com.yammer.metrics.util.DeadlockHealthCheck 9 | import com.yammer.metrics.reporting.{MetricsServlet, ThreadDumpServlet, HealthCheckServlet} 10 | import com.yammer.metrics.{Metrics, HealthChecks} 11 | import com.yammer.metrics.core.{MetricName, Meter} 12 | import org.mortbay.jetty.Server 13 | import org.mortbay.jetty.servlet.Context 14 | 15 | // TODO(ev): Move this into ScalaStorm 16 | package object config { 17 | type StormConfigMap = java.util.Map[_, _] 18 | 19 | class StormConfig(val origConfig: StormConfigMap) { 20 | val accessibleConfig = origConfig.asInstanceOf[java.util.Map[String, AnyRef]] 21 | 22 | // Returns an integer list from a configuration list of numbers, or Nil if key doesn't exist 23 | def getIntList(key: String): Seq[Int] = { 24 | if (accessibleConfig.containsKey(key)) 25 | accessibleConfig.get(key).asInstanceOf[java.util.List[Any]]. 26 | map { case l: Long => l.toInt 27 | case i: Int => i } 28 | else 29 | Nil 30 | } 31 | } 32 | 33 | implicit def toScalaStormConfig(config: StormConfigMap) = new StormConfig(config) 34 | } 35 | 36 | /** 37 | * Utilities for registering metrics for Storm spouts and bolts, as well as spinning up a metrics web 38 | * console for storm workers. 39 | * 40 | * This can be used for monitoring and debugging of storm workers. 41 | * 42 | * Metrics monitored: 43 | * - acks - the rate and count of tuples acked by this bolt 44 | * - emits - the rate and count of tuples emitted by this bolt 45 | */ 46 | object MetricsStorm { 47 | import config._ 48 | 49 | // Default port if one above cannot be found 50 | val DefaultPort = 7070 51 | 52 | // Various metrics for tasks 53 | val ackMeters = new HashMap[Int, Meter] 54 | val emitMeters = new HashMap[Int, Meter] 55 | 56 | private var consoleInitialized = false 57 | private var server: Server = _ 58 | 59 | /** 60 | * Initializes the web console using the storm configuration parameter "worker.webconsole.ports" 61 | * This should be a list of ports for the web console, and should have the same length as 62 | * "supervisor.slots.ports", where the console port corresponds to the slot in the same position. 63 | */ 64 | def initWebConsoleFromTask(conf: StormConfigMap, context: TopologyContext) { 65 | // Get the list of Storm worker ports, and find the index of current port. Then use that 66 | // to index into worker.webconsole.ports. 67 | val portIndex = conf.getIntList("supervisor.slots.ports").indexOf(context.getThisWorkerPort) 68 | val portList = conf.getIntList("worker.webconsole.ports") 69 | val port = if (portIndex >= 0 && portList.length > portIndex) portList(portIndex) else DefaultPort 70 | initWebConsole(port) 71 | } 72 | 73 | /** 74 | * Initializes an embedded Jetty web console 75 | */ 76 | def initWebConsole(port: Int) { 77 | synchronized { 78 | if (consoleInitialized) return 79 | 80 | HealthChecks.register(new DeadlockHealthCheck) 81 | 82 | server = new Server(port) 83 | val context = new Context(server, "/") 84 | context.addServlet(classOf[HealthCheckServlet], "/healthz/*") 85 | context.addServlet(classOf[ThreadDumpServlet], "/threadz/*") 86 | context.addServlet(classOf[MetricsServlet], "/metricz/*") 87 | 88 | // Starts the Jetty server on a separate thread; won't block. Don't call join. 89 | server.start() 90 | 91 | consoleInitialized = true 92 | } 93 | } 94 | 95 | /** 96 | * Returns true if the web console is initialized. 97 | * 98 | * @return Boolean True if the web console is initialized, false otherwise 99 | */ 100 | def isConsoleInitialized = consoleInitialized 101 | 102 | /** 103 | * Stops the embedded web console. It's safe to call this multiple times. 104 | */ 105 | def stopWebConsole() { 106 | synchronized { 107 | if (!consoleInitialized) return 108 | server.stop() 109 | consoleInitialized = false 110 | } 111 | } 112 | 113 | /** 114 | * A convenience function to create a new {@link com.yammer.metrics.core.MetricName} 115 | * for use with Metrics.new* metrics factory methods for Storm tasks. 116 | * 117 | * @return a new {@link com.yammer.metrics.core.MetricName} 118 | */ 119 | def getMetricName(name: String, context: TopologyContext) = { 120 | val scope = if (context == null) "UNKNOWN" else context.getThisComponentId 121 | new MetricName("storm", "taskInfo", name, scope) 122 | } 123 | 124 | /** 125 | * Register metrics for one task. Should be called only once. 126 | */ 127 | def setupTaskMetrics(context: TopologyContext) { 128 | ackMeters(context.getThisTaskId) = Metrics.newMeter( 129 | getMetricName("acks", context), "acks", TimeUnit.SECONDS) 130 | emitMeters(context.getThisTaskId) = Metrics.newMeter( 131 | getMetricName("emits", context), "emits", TimeUnit.SECONDS) 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/ooyala/common/metrics_storm/MetricsStormHooks.scala: -------------------------------------------------------------------------------- 1 | package ooyala.common.metrics_storm 2 | 3 | import backtype.storm.hooks.info._ 4 | 5 | import backtype.storm.hooks._ 6 | import backtype.storm.task.TopologyContext 7 | import backtype.storm.hooks.info._ 8 | import com.yammer.metrics.Metrics 9 | 10 | /** 11 | * Implements Storm 0.8.1 hooks interface for easy 12 | * metrics / embedded web console for Storm topologies. 13 | * 14 | * Add using the topology.auto.task.hooks config in Storm. 15 | * This class must have a zero-arg constructor. 16 | */ 17 | class MetricsStormHooks extends BaseTaskHook { 18 | override def prepare(config: java.util.Map[_, _], context: TopologyContext) { 19 | MetricsStorm.initWebConsoleFromTask(config, context) 20 | MetricsStorm.setupTaskMetrics(context) 21 | } 22 | 23 | override def cleanup() { 24 | MetricsStorm.stopWebConsole() 25 | Metrics.shutdown() 26 | } 27 | 28 | override def emit(emitData: EmitInfo) { 29 | MetricsStorm.emitMeters.get(emitData.taskId) foreach { _.mark } 30 | } 31 | 32 | override def spoutAck(ackData: SpoutAckInfo) {} 33 | 34 | override def spoutFail(failData: SpoutFailInfo) {} 35 | 36 | override def boltAck(ackData: BoltAckInfo) { 37 | val taskId = ackData.ackingTaskId 38 | MetricsStorm.ackMeters.get(taskId) foreach { meter => meter.mark() } 39 | } 40 | 41 | override def boltFail(failData: BoltFailInfo) {} 42 | } 43 | -------------------------------------------------------------------------------- /test/ooyala/common/metrics_storm/MetricsStormSpec.scala: -------------------------------------------------------------------------------- 1 | package ooyala.common.metrics_storm 2 | 3 | import scala.sys.process._ 4 | 5 | import org.scalatest.matchers.ShouldMatchers 6 | import org.scalatest.FunSpec 7 | import backtype.storm.task.TopologyContext 8 | import org.scalatest.mock.MockitoSugar 9 | import org.mockito.Mockito._ 10 | 11 | class MetricsStormSpec extends FunSpec with ShouldMatchers with MockitoSugar { 12 | describe("MetricsStorm") { 13 | it("should initialize web console and then shut it down properly") { 14 | MetricsStorm.isConsoleInitialized should equal (false) 15 | 16 | MetricsStorm.initWebConsole(7000) 17 | MetricsStorm.isConsoleInitialized should equal (true) 18 | Thread sleep 100 19 | val healthzOutput = { "curl localhost:7000/healthz" !! } 20 | healthzOutput should include ("deadlock") 21 | 22 | MetricsStorm.stopWebConsole() 23 | MetricsStorm.isConsoleInitialized should equal (false) 24 | } 25 | 26 | it("should have a /threadz route") (pending) 27 | } 28 | 29 | describe("getMetricname") { 30 | it("should handle passing in null TopologyContext") { 31 | val metricName = MetricsStorm.getMetricName("foo", null) 32 | metricName.getScope should equal ("UNKNOWN") 33 | } 34 | 35 | it("should create metric name from context") { 36 | val context = mock[TopologyContext] 37 | when(context.getThisComponentId).thenReturn("foo") 38 | when(context.getThisTaskId).thenReturn(5) 39 | MetricsStorm.getMetricName("metric1", context).toString should equal ( 40 | "storm:type=taskInfo,scope=foo,name=metric1") 41 | } 42 | } 43 | } 44 | --------------------------------------------------------------------------------