├── project ├── build.properties ├── plugins.sbt ├── Publish.scala └── Dependencies.scala ├── version.sbt ├── .gitignore ├── docs ├── akka-actors.png └── akka-dispatchers.png ├── akka-sensors └── src │ ├── main │ ├── resources │ │ └── reference.conf │ └── scala │ │ └── akka │ │ ├── sensors │ │ ├── ClusterHealthCheck.scala │ │ ├── dispatch │ │ │ ├── ScalaRunnableWrapper.scala │ │ │ ├── DispatcherMetricsRegistration.scala │ │ │ └── InstrumentedDispatchers.scala │ │ ├── metered │ │ │ ├── MeteredDispatcherSettings.scala │ │ │ ├── MeteredDispatcherSetup.scala │ │ │ ├── SetupNotFound.scala │ │ │ ├── MeteredDispatcher.scala │ │ │ ├── MeteredDispatcherConfigurator.scala │ │ │ ├── MeteredDispatcherInstrumentation.scala │ │ │ ├── DispatcherInstrumentationWrapper.scala │ │ │ └── MeteredInstrumentedExecutor.scala │ │ ├── behavior │ │ │ ├── ReceiveTimeoutMetrics.scala │ │ │ ├── BehaviorMetrics.scala │ │ │ └── BasicActorMetrics.scala │ │ ├── PrometheusCompat.scala │ │ ├── actor │ │ │ ├── ClusterEventWatchActor.scala │ │ │ ├── ClusterEventWatchTypedActor.scala │ │ │ └── ActorMetrics.scala │ │ ├── DispatcherMetrics.scala │ │ ├── ClassNameUtil.scala │ │ ├── RunnableWatcher.scala │ │ ├── SensorMetrics.scala │ │ └── AkkaSensorsExtension.scala │ │ └── persistence │ │ └── sensors │ │ └── EventSourcedMetrics.scala │ └── test │ ├── resources │ ├── logback-test.xml │ └── application.conf │ └── scala │ └── akka │ └── sensors │ ├── DispatcherMetricsSpec.scala │ ├── SensorMetricsSpec.scala │ ├── metered │ ├── MeteredDispatcherConfiguratorSpec.scala │ ├── MeteredInstrumentedExecutorSpec.scala │ └── MeteredLogicSpec.scala │ └── AkkaSensorsSpec.scala ├── .scalafmt.conf ├── LICENSE ├── .github └── workflows │ └── release.yml └── README.md /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.11.7 2 | 3 | 4 | -------------------------------------------------------------------------------- /version.sbt: -------------------------------------------------------------------------------- 1 | ThisBuild / version := "1.0.3-SNAPSHOT" 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | /.bsp/ 4 | target 5 | .idea 6 | .sbt.ivy.lock 7 | cache -------------------------------------------------------------------------------- /docs/akka-actors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacum/akka-sensors/HEAD/docs/akka-actors.png -------------------------------------------------------------------------------- /docs/akka-dispatchers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jacum/akka-sensors/HEAD/docs/akka-dispatchers.png -------------------------------------------------------------------------------- /akka-sensors/src/main/resources/reference.conf: -------------------------------------------------------------------------------- 1 | akka.sensors { 2 | extension-class = "akka.sensors.AkkaSensorsExtension" 3 | thread-state-snapshot-period = 5s 4 | cluster-watch-enabled = false 5 | } 6 | 7 | akka.coordinated-shutdown.exit-jvm = off 8 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/ClusterHealthCheck.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors 2 | 3 | import akka.actor.ActorSystem 4 | import akka.cluster.{Cluster, MemberStatus} 5 | 6 | import scala.concurrent.Future 7 | 8 | class ClusterHealthCheck(system: ActorSystem) extends (() => Future[Boolean]) { 9 | private val cluster = Cluster(system) 10 | override def apply(): Future[Boolean] = 11 | Future.successful(cluster.selfMember.status == MemberStatus.Up) 12 | } 13 | -------------------------------------------------------------------------------- /akka-sensors/src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/dispatch/ScalaRunnableWrapper.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.dispatch 2 | 3 | import akka.dispatch.Batchable 4 | import akka.sensors.dispatch.DispatcherInstrumentationWrapper.Run 5 | 6 | import scala.PartialFunction.condOpt 7 | 8 | object ScalaRunnableWrapper { 9 | def unapply(runnable: Runnable): Option[Run => Runnable] = 10 | condOpt(runnable) { 11 | case runnable: Batchable => new OverrideBatchable(runnable, _) 12 | } 13 | 14 | class OverrideBatchable(self: Runnable, r: Run) extends Batchable with Runnable { 15 | def run(): Unit = r(() => self.run()) 16 | def isBatchable: Boolean = true 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/metered/MeteredDispatcherSettings.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.metered 2 | 3 | import akka.dispatch.MessageDispatcherConfigurator 4 | 5 | import scala.concurrent.duration._ 6 | import akka.dispatch.ExecutorServiceFactoryProvider 7 | import akka.sensors.DispatcherMetrics 8 | 9 | private[metered] case class MeteredDispatcherSettings( 10 | name: String, 11 | metrics: DispatcherMetrics, 12 | _configurator: MessageDispatcherConfigurator, 13 | id: String, 14 | throughput: Int, 15 | throughputDeadlineTime: Duration, 16 | executorServiceFactoryProvider: ExecutorServiceFactoryProvider, 17 | shutdownTimeout: FiniteDuration 18 | ) 19 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | import sbt.addSbtPlugin 2 | addSbtPlugin("com.github.sbt" % "sbt-git" % "2.1.0") 3 | addSbtPlugin("org.wartremover" % "sbt-wartremover" % "3.4.1") 4 | addSbtPlugin("net.vonbuchholtz" % "sbt-dependency-check" % "5.1.0") 5 | addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.11.4") 6 | addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.5") 7 | addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.12.2") 8 | addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.3.1") 9 | addSbtPlugin("com.github.sbt" % "sbt-release" % "1.4.0") 10 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.4.0") 11 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/metered/MeteredDispatcherSetup.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.metered 2 | 3 | import akka.actor.setup.Setup 4 | import akka.dispatch.DispatcherPrerequisites 5 | import akka.sensors.DispatcherMetrics 6 | 7 | final case class MeteredDispatcherSetup(metrics: DispatcherMetrics) extends Setup 8 | 9 | object MeteredDispatcherSetup { 10 | 11 | /** Extract LocalDispatcherSetup out from DispatcherPrerequisites or throw an exception */ 12 | def setupOrThrow(prereq: DispatcherPrerequisites): MeteredDispatcherSetup = 13 | prereq.settings.setup 14 | .get[MeteredDispatcherSetup] 15 | .getOrElse(throw SetupNotFound[MeteredDispatcherSetup]) 16 | } 17 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/metered/SetupNotFound.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.metered 2 | 3 | import scala.reflect.ClassTag 4 | import scala.util.control.NoStackTrace 5 | 6 | trait SetupNotFound extends NoStackTrace 7 | 8 | object SetupNotFound { 9 | private final case class Impl(msg: String) extends RuntimeException(msg) with SetupNotFound 10 | private def errorMsg[T: ClassTag]: String = { 11 | val className = implicitly[ClassTag[T]].runtimeClass.getName 12 | s"Can't find dispatcher setup for '$className'." + 13 | s" Please check if you have `$className` defined for your ActorSystem." + 14 | s" Check ActorSystemSetup for more info." 15 | } 16 | def apply[T: ClassTag]: SetupNotFound = Impl(errorMsg[T]) 17 | } 18 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/metered/MeteredDispatcher.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.metered 2 | 3 | import akka.dispatch.Dispatcher 4 | import akka.sensors.DispatcherMetrics 5 | 6 | private[metered] class MeteredDispatcher(settings: MeteredDispatcherSettings) 7 | extends Dispatcher( 8 | settings._configurator, 9 | settings.id, 10 | settings.throughput, 11 | settings.throughputDeadlineTime, 12 | executorServiceFactoryProvider = settings.executorServiceFactoryProvider, 13 | shutdownTimeout = settings.shutdownTimeout 14 | ) 15 | with MeteredDispatcherInstrumentation { 16 | protected override val actorSystemName: String = settings.name 17 | protected override val metrics: DispatcherMetrics = settings.metrics 18 | } 19 | -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | version = 2.6.4 2 | 3 | maxColumn = 180 4 | docstrings = JavaDoc 5 | importSelectors = singleLine 6 | style = defaultWithAlign 7 | align.openParenCallSite = false 8 | align.openParenDefnSite = false 9 | continuationIndent.callSite = 2 10 | continuationIndent.defnSite = 2 11 | //danglingParentheses = true 12 | //indentOperator = spray 13 | newlines.alwaysBeforeTopLevelStatements = false 14 | newlines.alwaysBeforeElseAfterCurlyIf = false 15 | spaces.inImportCurlyBraces = false 16 | unindentTopLevelOperators = true 17 | rewrite.rules = [ 18 | RedundantParens, 19 | RedundantBraces, 20 | PreferCurlyFors, 21 | AsciiSortImports, 22 | SortModifiers 23 | ] 24 | rewrite.sortModifiers.order = [ 25 | "private", "protected", "implicit", "final", "sealed", "abstract", 26 | "override", "lazy" 27 | ] 28 | 29 | project.excludeFilters = [] 30 | 31 | includeCurlyBraceInSelectChains = false 32 | 33 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/dispatch/DispatcherMetricsRegistration.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.dispatch 2 | 3 | import akka.sensors.{AkkaSensors, DispatcherMetrics} 4 | import io.prometheus.metrics.core.metrics.{Gauge, Histogram} 5 | import io.prometheus.metrics.model.registry.PrometheusRegistry 6 | 7 | /** Creates and registers Dispatcher metrics in the global registry */ 8 | private[dispatch] object DispatcherMetricsRegistration { 9 | val registry: PrometheusRegistry = AkkaSensors.prometheusRegistry 10 | 11 | private val metrics = DispatcherMetrics.makeAndRegister(registry) 12 | def queueTime: Histogram = metrics.queueTime 13 | def runTime: Histogram = metrics.runTime 14 | def activeThreads: Histogram = metrics.activeThreads 15 | def threadStates: Gauge = metrics.threadStates 16 | def threads: Gauge = metrics.threads 17 | def executorValue: Gauge = metrics.executorValue 18 | } 19 | -------------------------------------------------------------------------------- /akka-sensors/src/test/scala/akka/sensors/DispatcherMetricsSpec.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors 2 | 3 | import io.prometheus.metrics.model.registry.PrometheusRegistry 4 | import org.scalatest.freespec.AnyFreeSpec 5 | import org.scalatest.matchers.should.Matchers 6 | 7 | import scala.jdk.CollectionConverters._ 8 | 9 | class DispatcherMetricsSpec extends AnyFreeSpec with Matchers { 10 | "DispatcherMetrics" - { 11 | "registers all metrics" in { 12 | val cr = new PrometheusRegistry() 13 | DispatcherMetrics.makeAndRegister(cr) 14 | val samples = cr.scrape().iterator().asScala.toList 15 | val names = samples.map(_.getMetadata.getName) 16 | 17 | names should contain("akka_sensors_dispatchers_queue_time_millis") 18 | names should contain("akka_sensors_dispatchers_run_time_millis") 19 | names should contain("akka_sensors_dispatchers_active_threads") 20 | names should contain("akka_sensors_dispatchers_thread_states") 21 | names should contain("akka_sensors_dispatchers_threads") 22 | names should contain("akka_sensors_dispatchers_executor_value") 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/behavior/ReceiveTimeoutMetrics.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.behavior 2 | 3 | import akka.actor.typed.scaladsl.Behaviors 4 | import akka.actor.typed.{Behavior, BehaviorInterceptor, TypedActorContext} 5 | import akka.sensors.{AkkaSensorsExtension, SensorMetrics} 6 | import akka.sensors.PrometheusCompat._ 7 | 8 | import scala.reflect.ClassTag 9 | 10 | final case class ReceiveTimeoutMetrics[C]( 11 | actorLabel: String, 12 | metrics: SensorMetrics, 13 | timeoutCmd: C 14 | ) { 15 | 16 | private val receiveTimeouts = metrics.receiveTimeouts.labels(actorLabel) 17 | 18 | def apply(behavior: Behavior[C])(implicit ct: ClassTag[C]): Behavior[C] = { 19 | 20 | val interceptor = () => 21 | new BehaviorInterceptor[C, C] { 22 | @SuppressWarnings(Array("org.wartremover.warts.Equals")) 23 | def aroundReceive( 24 | ctx: TypedActorContext[C], 25 | msg: C, 26 | target: BehaviorInterceptor.ReceiveTarget[C] 27 | ): Behavior[C] = { 28 | if (msg == timeoutCmd) receiveTimeouts.inc() 29 | target(ctx, msg) 30 | } 31 | } 32 | 33 | Behaviors.intercept(interceptor)(behavior) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Tim Evdokimov 4 | Copyright (c) 2017 Evolution Gaming 5 | (see https://github.com/evolution-gaming/akka-tools) 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in all 15 | copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | SOFTWARE. 24 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/metered/MeteredDispatcherConfigurator.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.metered 2 | 3 | import akka.dispatch.{Dispatcher, DispatcherPrerequisites, MessageDispatcher, MessageDispatcherConfigurator} 4 | import akka.sensors.DispatcherMetrics 5 | import akka.sensors.dispatch.Helpers 6 | import com.typesafe.config.Config 7 | 8 | class MeteredDispatcherConfigurator(config: Config, prerequisites: DispatcherPrerequisites) extends MessageDispatcherConfigurator(config, prerequisites) { 9 | import Helpers._ 10 | 11 | private val instance: MessageDispatcher = { 12 | val _metrics = MeteredDispatcherSetup.setupOrThrow(prerequisites).metrics 13 | val settings = MeteredDispatcherSettings( 14 | name = prerequisites.mailboxes.settings.name, 15 | metrics = _metrics, 16 | _configurator = this, 17 | id = config.getString("id"), 18 | throughput = config.getInt("throughput"), 19 | throughputDeadlineTime = config.getNanosDuration("throughput-deadline-time"), 20 | executorServiceFactoryProvider = configureExecutor(), 21 | shutdownTimeout = config.getMillisDuration("shutdown-timeout") 22 | ) 23 | 24 | new MeteredDispatcher(settings) 25 | } 26 | 27 | def dispatcher(): MessageDispatcher = instance 28 | } 29 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/PrometheusCompat.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors 2 | 3 | import io.prometheus.metrics.core.datapoints.{CounterDataPoint, DistributionDataPoint, GaugeDataPoint} 4 | import io.prometheus.metrics.core.metrics.{Counter, Gauge, Histogram} 5 | 6 | object PrometheusCompat { 7 | implicit class CounterBuilderCompat(private val b: Counter.Builder) extends AnyVal { 8 | def create(): Counter = b.build() 9 | } 10 | implicit class GaugeBuilderCompat(private val b: Gauge.Builder) extends AnyVal { 11 | def create(): Gauge = b.build() 12 | } 13 | implicit class HistogramBuilderCompat(private val b: Histogram.Builder) extends AnyVal { 14 | def create(): Histogram = b.build() 15 | } 16 | 17 | // Compatibility aliases: old simpleclient used `.labels(...)`, new API uses `.labelValues(...)` 18 | implicit class CounterLabelsCompat(private val c: Counter) extends AnyVal { 19 | def labels(values: String*): CounterDataPoint = c.labelValues(values: _*) 20 | } 21 | implicit class GaugeLabelsCompat(private val g: Gauge) extends AnyVal { 22 | def labels(values: String*): GaugeDataPoint = g.labelValues(values: _*) 23 | } 24 | implicit class HistogramLabelsCompat(private val h: Histogram) extends AnyVal { 25 | def labels(values: String*): DistributionDataPoint = h.labelValues(values: _*) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /akka-sensors/src/test/scala/akka/sensors/SensorMetricsSpec.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors 2 | 3 | import org.scalatest.freespec.AnyFreeSpec 4 | import org.scalatest.matchers.should.Matchers._ 5 | 6 | import scala.jdk.CollectionConverters.IteratorHasAsScala 7 | import io.prometheus.metrics.model.registry.PrometheusRegistry 8 | 9 | class SensorMetricsSpec extends AnyFreeSpec { 10 | "SensorMetrics" - { 11 | "registers all metrics" in { 12 | val cr = new PrometheusRegistry() 13 | val result = SensorMetrics.makeAndRegister(cr) 14 | val samples = cr.scrape().iterator().asScala.toList 15 | val names = samples.map(_.getMetadata.getName) 16 | 17 | names should contain("akka_sensors_actor_activity_time_seconds") 18 | names should contain("akka_sensors_actor_active_actors") 19 | names should contain("akka_sensors_actor_unhandled_messages") 20 | names should contain("akka_sensors_actor_exceptions") 21 | names should contain("akka_sensors_actor_receive_time_millis") 22 | names should contain("akka_sensors_actor_receive_timeouts") 23 | names should contain("akka_sensors_actor_cluster_events") 24 | names should contain("akka_sensors_actor_cluster_members") 25 | names should contain("akka_sensors_actor_recovery_time_millis") 26 | names should contain("akka_sensors_actor_persist_time_millis") 27 | names should contain("akka_sensors_actor_recoveries") 28 | names should contain("akka_sensors_actor_recovery_events") 29 | names should contain("akka_sensors_actor_persist_failures") 30 | names should contain("akka_sensors_actor_recovery_failures") 31 | names should contain("akka_sensors_actor_persist_rejects") 32 | names should contain("akka_sensors_actor_waiting_for_recovery_permit_actors") 33 | names should contain("akka_sensors_actor_waiting_for_recovery_permit_time_millis") 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/actor/ClusterEventWatchActor.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.actor 2 | 3 | import akka.actor.{Actor, ActorLogging} 4 | import akka.cluster.ClusterEvent._ 5 | import akka.cluster.{Cluster, Member} 6 | import akka.sensors.PrometheusCompat.CounterLabelsCompat 7 | import akka.sensors.{AkkaSensorsExtension, ClassNameUtil} 8 | 9 | class ClusterEventWatchActor extends Actor with ActorLogging { 10 | 11 | private val cluster = Cluster(context.system) 12 | private val metrics: AkkaSensorsExtension = AkkaSensorsExtension(this.context.system) 13 | private val clusterEvents = metrics.clusterEvents 14 | 15 | override def preStart(): Unit = { 16 | cluster.subscribe(self, initialStateMode = InitialStateAsEvents, classOf[ClusterDomainEvent]) 17 | log.info("Starting cluster event watch") 18 | } 19 | 20 | override def postStop(): Unit = cluster.unsubscribe(self) 21 | 22 | private def registerEvent(e: ClusterDomainEvent, member: Option[Member] = None): Unit = 23 | clusterEvents 24 | .labels(ClassNameUtil.simpleName(e.getClass), member.map(_.address.toString).getOrElse("")) 25 | .inc() 26 | 27 | def receive: Receive = { 28 | case e @ MemberUp(member) => 29 | registerEvent(e, Some(member)) 30 | log.info("Member is Up: {}", member.address) 31 | case e @ UnreachableMember(member) => 32 | registerEvent(e, Some(member)) 33 | log.info("Member detected as unreachable: {}", member) 34 | case e @ MemberRemoved(member, previousStatus) => 35 | registerEvent(e, Some(member)) 36 | log.info("Member is Removed: {} after {}", member.address, previousStatus) 37 | case e @ MemberDowned(member) => 38 | registerEvent(e, Some(member)) 39 | log.info("Member is Down: {}", member.address) 40 | case e: ClusterDomainEvent => 41 | registerEvent(e) 42 | log.info(s"Cluster domain event: $e") 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/actor/ClusterEventWatchTypedActor.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.actor 2 | 3 | import akka.actor.typed.scaladsl.Behaviors 4 | import akka.actor.typed.{Behavior, PostStop, PreRestart} 5 | import akka.cluster.ClusterEvent._ 6 | import akka.cluster.Member 7 | import akka.cluster.typed.{Cluster, Subscribe, Unsubscribe} 8 | import io.prometheus.metrics.core.metrics.Counter 9 | import akka.sensors.PrometheusCompat._ 10 | object ClusterEventWatchTypedActor { 11 | def apply(clusterEvents: Counter): Behavior[ClusterDomainEvent] = 12 | Behaviors.setup { context => 13 | val cluster = Cluster(context.system) 14 | val log = context.log 15 | val meterEvent = registerEvent(clusterEvents) _ 16 | 17 | cluster.subscriptions ! Subscribe(context.self, classOf[MemberEvent]) 18 | 19 | Behaviors 20 | .receiveMessage[ClusterDomainEvent] { event => 21 | event match { 22 | case e @ MemberUp(member) => 23 | meterEvent(e, Some(member)) 24 | log.info("Member is Up: {}", member.address) 25 | case e @ UnreachableMember(member) => 26 | meterEvent(e, Some(member)) 27 | log.info("Member detected as unreachable: {}", member) 28 | case e @ MemberRemoved(member, previousStatus) => 29 | meterEvent(e, Some(member)) 30 | log.info("Member is Removed: {} after {}", member.address, previousStatus) 31 | case e @ MemberDowned(member) => 32 | meterEvent(e, Some(member)) 33 | log.info("Member is Down: {}", member.address) 34 | case e => 35 | meterEvent(e, None) 36 | log.info(s"Cluster domain event: $e") 37 | } 38 | Behaviors.same 39 | } 40 | .receiveSignal { 41 | case (_, PostStop) => 42 | cluster.subscriptions ! Unsubscribe(context.self) 43 | Behaviors.same 44 | } 45 | } 46 | 47 | private def registerEvent(clusterEvents: Counter)(e: ClusterDomainEvent, member: Option[Member]): Unit = 48 | clusterEvents 49 | .labels(e.getClass.getSimpleName, member.map(_.address.toString).getOrElse("")) 50 | .inc() 51 | } 52 | -------------------------------------------------------------------------------- /akka-sensors/src/test/scala/akka/sensors/metered/MeteredDispatcherConfiguratorSpec.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.metered 2 | 3 | import akka.ConfigurationException 4 | import akka.actor.BootstrapSetup 5 | import akka.actor.setup.ActorSystemSetup 6 | import akka.actor.typed.{ActorSystem, DispatcherSelector, SpawnProtocol} 7 | import akka.sensors.DispatcherMetrics 8 | import akka.sensors.metered.MeteredDispatcherConfiguratorSpec._ 9 | import com.typesafe.config.ConfigFactory 10 | import org.scalatest.freespec.AnyFreeSpec 11 | import org.scalatest.matchers.should.Matchers 12 | 13 | class MeteredDispatcherConfiguratorSpec extends AnyFreeSpec with Matchers { 14 | "MeteredDispatcherConfigurator" - { 15 | "is returned if configured(MeteredDispatcherSetup is defined)" in { 16 | val metrics = DispatcherMetrics.make() 17 | val withConfig = BootstrapSetup(cfg) 18 | val withMetrics = MeteredDispatcherSetup(metrics) 19 | val setup = ActorSystemSetup.create(withConfig, withMetrics) 20 | val actorSystem = ActorSystem[SpawnProtocol.Command](SpawnProtocol(), "test-system", setup) 21 | val dispatcher = actorSystem.dispatchers.lookup(DispatcherSelector.defaultDispatcher()) 22 | 23 | try dispatcher shouldBe a[MeteredDispatcher] 24 | finally actorSystem.terminate() 25 | } 26 | 27 | "throws SetupNotFound if MeteredDispatcherSetup is not defined" in { 28 | val withConfig = BootstrapSetup(cfg) 29 | val setup = ActorSystemSetup.create(withConfig) 30 | def actorSystem = ActorSystem[SpawnProtocol.Command](SpawnProtocol(), "test-system", setup) 31 | 32 | val exception = the[ConfigurationException] thrownBy actorSystem 33 | exception.getCause shouldBe a[SetupNotFound] 34 | } 35 | } 36 | } 37 | 38 | object MeteredDispatcherConfiguratorSpec { 39 | private val cfgStr = 40 | """ 41 | |akka.actor.default-dispatcher { 42 | | type = "akka.sensors.metered.MeteredDispatcherConfigurator" 43 | | instrumented-executor { 44 | | delegate = "java.util.concurrent.ForkJoinPool" 45 | | measure-runs = false 46 | | watch-long-runs = false 47 | | } 48 | |} 49 | |""".stripMargin 50 | 51 | private val cfg = ConfigFactory.parseString(cfgStr) 52 | } 53 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/DispatcherMetrics.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors 2 | 3 | import io.prometheus.metrics.core.metrics.{Gauge, Histogram} 4 | import io.prometheus.metrics.model.registry.{Collector, PrometheusRegistry} 5 | 6 | final case class DispatcherMetrics( 7 | queueTime: Histogram, 8 | runTime: Histogram, 9 | activeThreads: Histogram, 10 | threadStates: Gauge, 11 | threads: Gauge, 12 | executorValue: Gauge 13 | ) { 14 | val allCollectors: List[Collector] = List(queueTime, runTime, activeThreads, threadStates, threads, executorValue) 15 | } 16 | 17 | object DispatcherMetrics { 18 | def make(): DispatcherMetrics = 19 | DispatcherMetrics( 20 | queueTime = Histogram 21 | .builder() 22 | .classicUpperBounds(10000) 23 | .name("akka_sensors_dispatchers_queue_time_millis") 24 | .help(s"Milliseconds in queue") 25 | .labelNames("dispatcher") 26 | .build(), 27 | runTime = Histogram 28 | .builder() 29 | .classicUpperBounds(10000) 30 | .name("akka_sensors_dispatchers_run_time_millis") 31 | .help(s"Milliseconds running") 32 | .labelNames("dispatcher") 33 | .build(), 34 | activeThreads = Histogram 35 | .builder() 36 | .classicOnly() 37 | .name("akka_sensors_dispatchers_active_threads") 38 | .help(s"Active worker threads") 39 | .labelNames("dispatcher") 40 | .build(), 41 | threadStates = Gauge 42 | .builder() 43 | .name("akka_sensors_dispatchers_thread_states") 44 | .help("Threads per state and dispatcher") 45 | .labelNames("dispatcher", "state") 46 | .build(), 47 | threads = Gauge 48 | .builder() 49 | .name("akka_sensors_dispatchers_threads") 50 | .help("Threads per dispatcher") 51 | .labelNames("dispatcher") 52 | .build(), 53 | executorValue = Gauge 54 | .builder() 55 | .name("akka_sensors_dispatchers_executor_value") 56 | .help("Internal executor values per type") 57 | .labelNames("dispatcher", "value") 58 | .build() 59 | ) 60 | 61 | def makeAndRegister(cr: PrometheusRegistry): DispatcherMetrics = { 62 | val metrics = make() 63 | metrics.allCollectors.foreach(c => cr.register(c)) 64 | metrics 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /akka-sensors/src/test/scala/akka/sensors/metered/MeteredInstrumentedExecutorSpec.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.metered 2 | 3 | import akka.ConfigurationException 4 | import akka.actor.BootstrapSetup 5 | import akka.actor.setup.ActorSystemSetup 6 | import akka.actor.typed.{ActorSystem, DispatcherSelector, SpawnProtocol} 7 | import akka.sensors.DispatcherMetrics 8 | import akka.sensors.metered.MeteredInstrumentedExecutorSpec._ 9 | import io.prometheus.metrics.model.registry.PrometheusRegistry 10 | import org.scalatest.freespec.AnyFreeSpec 11 | import org.scalatest.matchers.should.Matchers 12 | 13 | class MeteredInstrumentedExecutorSpec extends AnyFreeSpec with Matchers { 14 | "MeteredInstrumentedExecutor" - { 15 | "is returned if configured(MeteredDispatcherSetup is defined)" in { 16 | val cr = new PrometheusRegistry() 17 | val metrics = DispatcherMetrics.makeAndRegister(cr) 18 | val withConfig = BootstrapSetup(cfg) 19 | val withMetrics = MeteredDispatcherSetup(metrics) 20 | val setup = ActorSystemSetup.create(withConfig, withMetrics) 21 | val actorSystem = ActorSystem[SpawnProtocol.Command](SpawnProtocol(), "test-system", setup) 22 | val dispatcher = actorSystem.dispatchers.lookup(DispatcherSelector.defaultDispatcher()) 23 | 24 | // do some execution to make sure that our executor is created 25 | dispatcher.execute(() => ()) 26 | } 27 | 28 | "throws SetupNotFound if MeteredDispatcherSetup is not defined" in { 29 | val withConfig = BootstrapSetup(cfg) 30 | val setup = ActorSystemSetup.create(withConfig) 31 | def actorSystem = ActorSystem[SpawnProtocol.Command](SpawnProtocol(), "test-system", setup) 32 | 33 | val exception = the[IllegalArgumentException] thrownBy actorSystem 34 | exception.getCause shouldBe a[SetupNotFound] 35 | } 36 | } 37 | } 38 | 39 | object MeteredInstrumentedExecutorSpec { 40 | import com.typesafe.config.ConfigFactory 41 | private val cfgStr = 42 | """ 43 | |akka.actor.default-dispatcher { 44 | | executor = "akka.sensors.metered.MeteredInstrumentedExecutor" 45 | | instrumented-executor { 46 | | delegate = "fork-join-executor" 47 | | measure-runs = false 48 | | watch-long-runs = false 49 | | } 50 | |} 51 | |""".stripMargin 52 | 53 | private val cfg = ConfigFactory.parseString(cfgStr) 54 | } 55 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/behavior/BehaviorMetrics.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.behavior 2 | 3 | import akka.actor.typed.Behavior 4 | import akka.actor.typed.scaladsl.{ActorContext, Behaviors} 5 | import akka.persistence.sensors.EventSourcedMetrics 6 | import akka.sensors.{AkkaSensorsExtension, ClassNameUtil, SensorMetrics} 7 | 8 | import scala.reflect.ClassTag 9 | 10 | object BehaviorMetrics { 11 | 12 | private type CreateBehaviorMetrics[C] = (SensorMetrics, Behavior[C]) => Behavior[C] 13 | private val defaultMessageLabel: Any => Option[String] = msg => Some(ClassNameUtil.simpleName(msg.getClass)) 14 | 15 | def apply[C: ClassTag](actorLabel: String, getLabel: C => Option[String] = defaultMessageLabel): BehaviorMetricsBuilder[C] = { 16 | val defaultMetrics = (metrics: SensorMetrics, behavior: Behavior[C]) => BasicActorMetrics[C](actorLabel, metrics, getLabel)(behavior) 17 | new BehaviorMetricsBuilder(actorLabel, defaultMetrics :: Nil) 18 | } 19 | 20 | class BehaviorMetricsBuilder[C: ClassTag]( 21 | actorLabel: String, 22 | createMetrics: List[CreateBehaviorMetrics[C]] 23 | ) { self => 24 | 25 | def setup(factory: ActorContext[C] => Behavior[C]): Behavior[C] = 26 | Behaviors.setup { actorContext => 27 | val metrics = AkkaSensorsExtension(actorContext.asScala.system).metrics 28 | setupWithMetrics(metrics)(factory) 29 | } 30 | 31 | def setupWithMetrics(metrics: SensorMetrics)(factory: ActorContext[C] => Behavior[C]): Behavior[C] = 32 | Behaviors.setup { actorContext => 33 | val behavior = factory(actorContext) 34 | createMetrics.foldLeft(behavior)((b, createMetrics) => createMetrics(metrics, b)) 35 | } 36 | 37 | def withReceiveTimeoutMetrics(timeoutCmd: C): BehaviorMetricsBuilder[C] = { 38 | val receiveTimeoutMetrics = (metrics: SensorMetrics, behavior: Behavior[C]) => ReceiveTimeoutMetrics[C](actorLabel, metrics, timeoutCmd)(behavior) 39 | new BehaviorMetricsBuilder[C](self.actorLabel, receiveTimeoutMetrics :: self.createMetrics) 40 | } 41 | 42 | def withPersistenceMetrics: BehaviorMetricsBuilder[C] = { 43 | val eventSourcedMetrics = (metrics: SensorMetrics, behaviorToObserve: Behavior[C]) => EventSourcedMetrics(actorLabel, metrics).apply(behaviorToObserve) 44 | new BehaviorMetricsBuilder[C](actorLabel, eventSourcedMetrics :: self.createMetrics) 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/ClassNameUtil.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors 2 | 3 | object ClassNameUtil { 4 | 5 | /** 6 | * Safer than Class obj's getSimpleName which may throw Malformed class name error in scala. 7 | * This method mimics scalatest's getSimpleNameOfAnObjectsClass. 8 | */ 9 | def simpleName(cls: Class[_]): String = 10 | try stripDollars(cls.getSimpleName) 11 | catch { 12 | // TODO: the value returned here isn't even quite right; it returns simple names 13 | // like UtilsSuite$MalformedClassObject$MalformedClass instead of MalformedClass 14 | // The exact value may not matter much as it's used in log statements 15 | case _: InternalError => 16 | stripDollars(stripPackages(cls.getName)) 17 | } 18 | 19 | /** 20 | * Remove the packages from full qualified class name 21 | */ 22 | private def stripPackages(fullyQualifiedName: String): String = 23 | fullyQualifiedName.split("\\.").takeRight(1)(0) 24 | 25 | /** 26 | * Remove trailing dollar signs from qualified class name, 27 | * and return the trailing part after the last dollar sign in the middle 28 | */ 29 | private def stripDollars(s: String): String = { 30 | val lastDollarIndex = s.lastIndexOf('$') 31 | if (lastDollarIndex < s.length - 1) 32 | // The last char is not a dollar sign 33 | if (lastDollarIndex == -1 || !s.contains("$iw")) 34 | // The name does not have dollar sign or is not an interpreter 35 | // generated class, so we should return the full string 36 | s 37 | else 38 | // The class name is interpreter generated, 39 | // return the part after the last dollar sign 40 | // This is the same behavior as getClass.getSimpleName 41 | s.substring(lastDollarIndex + 1) 42 | else { 43 | // The last char is a dollar sign 44 | // Find last non-dollar char 45 | val lastNonDollarChar = s.reverse.find(_ != '$') 46 | lastNonDollarChar match { 47 | case None => s 48 | case Some(c) => 49 | val lastNonDollarIndex = s.lastIndexOf(c) 50 | if (lastNonDollarIndex == -1) 51 | s 52 | else 53 | // Strip the trailing dollar signs 54 | // Invoke stripDollars again to get the simple name 55 | stripDollars(s.substring(0, lastNonDollarIndex + 1)) 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /akka-sensors/src/test/resources/application.conf: -------------------------------------------------------------------------------- 1 | 2 | akka { 3 | 4 | loggers = ["akka.event.slf4j.Slf4jLogger"] 5 | loglevel = "DEBUG" 6 | logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" 7 | jvm-exit-on-fatal-error = true 8 | log-config-on-start = off 9 | 10 | actor { 11 | provider = "akka.cluster.ClusterActorRefProvider" 12 | allow-java-serialization = true // for in-memory unit tests only! 13 | default-dispatcher { 14 | type = "akka.sensors.dispatch.InstrumentedDispatcherConfigurator" 15 | executor = "akka.sensors.dispatch.InstrumentedExecutor" 16 | 17 | instrumented-executor { 18 | delegate = "fork-join-executor" 19 | measure-runs = true 20 | watch-long-runs = true 21 | watch-check-interval = 1s 22 | watch-too-long-run = 3s 23 | } 24 | } 25 | 26 | default-blocking-io-dispatcher { 27 | type = "akka.sensors.dispatch.InstrumentedDispatcherConfigurator" 28 | executor = "akka.sensors.dispatch.InstrumentedExecutor" 29 | 30 | instrumented-executor { 31 | delegate = "thread-pool-executor" 32 | measure-runs = true 33 | watch-long-runs = false 34 | } 35 | } 36 | } 37 | 38 | remote.artery { 39 | canonical { 40 | hostname = "127.0.0.1" 41 | port = 2551 42 | } 43 | } 44 | 45 | cluster { 46 | roles = ["state-node"] 47 | min-nr-of-members = 1 48 | configuration-compatibility-check.enforce-on-join = off 49 | downing-provider-class = "akka.cluster.sbr.SplitBrainResolverProvider" 50 | 51 | sharding { 52 | least-shard-allocation-strategy.rebalance-threshold = 5 53 | remember-entities = on 54 | } 55 | shutdown-after-unsuccessful-join-seed-nodes = 5m 56 | akka.remote.use-passive-connections = off 57 | 58 | downing-provider-class = "akka.cluster.sbr.SplitBrainResolverProvider" 59 | 60 | split-brain-resolver { 61 | active-strategy = keep-majority 62 | stable-after = 20s 63 | down-all-when-unstable = on 64 | } 65 | seed-nodes = ["akka://instrumented@127.0.0.1:2551"] 66 | } 67 | 68 | persistence { 69 | 70 | max-concurrent-recoveries = 1000 71 | snapshot-store.plugin = "" 72 | 73 | journal { 74 | plugin = "inmemory-journal" 75 | auto-start-journals = ["akka.persistence.journal.inmem"] 76 | } 77 | } 78 | 79 | extensions = [ 80 | akka.persistence.Persistence, 81 | akka.sensors.AkkaSensorsExtension 82 | ] 83 | 84 | sensors { 85 | thread-state-snapshot-period = 1s 86 | cluster-watch-enabled = true 87 | } 88 | 89 | } -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/behavior/BasicActorMetrics.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.behavior 2 | 3 | import akka.actor.typed._ 4 | import akka.actor.typed.scaladsl.Behaviors 5 | import akka.sensors.MetricOps._ 6 | import akka.sensors.SensorMetrics 7 | import akka.sensors.PrometheusCompat._ 8 | 9 | import scala.reflect.ClassTag 10 | import scala.util.control.NonFatal 11 | 12 | final case class BasicActorMetrics[C]( 13 | actorLabel: String, 14 | metrics: SensorMetrics, 15 | messageLabel: C => Option[String] 16 | ) { 17 | 18 | private lazy val exceptions = metrics.exceptions.labels(actorLabel) 19 | private val activeActors = metrics.activeActors.labels(actorLabel) 20 | private val activityTimer = metrics.activityTime.labels(actorLabel).startTimer() 21 | 22 | def apply(behavior: Behavior[C])(implicit ct: ClassTag[C]): Behavior[C] = { 23 | 24 | val interceptor = () => 25 | new BehaviorInterceptor[C, C] { 26 | override def aroundSignal( 27 | ctx: TypedActorContext[C], 28 | signal: Signal, 29 | target: BehaviorInterceptor.SignalTarget[C] 30 | ): Behavior[C] = { 31 | signal match { 32 | case PostStop => 33 | activeActors.dec() 34 | activityTimer.observeDuration() 35 | 36 | case _ => 37 | } 38 | 39 | target(ctx, signal) 40 | } 41 | 42 | override def aroundStart( 43 | ctx: TypedActorContext[C], 44 | target: BehaviorInterceptor.PreStartTarget[C] 45 | ): Behavior[C] = { 46 | activeActors.inc() 47 | target.start(ctx) 48 | } 49 | 50 | @SuppressWarnings(Array("org.wartremover.warts.Throw")) 51 | override def aroundReceive( 52 | ctx: TypedActorContext[C], 53 | msg: C, 54 | target: BehaviorInterceptor.ReceiveTarget[C] 55 | ): Behavior[C] = 56 | try { 57 | val next = messageLabel(msg).map { 58 | metrics.receiveTime 59 | .labels(actorLabel, _) 60 | .observeExecution(target(ctx, msg)) 61 | } 62 | .getOrElse(target(ctx, msg)) 63 | 64 | if (Behavior.isUnhandled(next)) 65 | messageLabel(msg) 66 | .foreach(metrics.unhandledMessages.labels(actorLabel, _).inc()) 67 | next 68 | } catch { 69 | case NonFatal(e) => 70 | exceptions.inc() 71 | throw e 72 | } 73 | } 74 | 75 | Behaviors.intercept(interceptor)(behavior) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release to Sonatype 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | permissions: 7 | contents: write # sbt-release will create tags and push commits 8 | 9 | jobs: 10 | release: 11 | runs-on: ubuntu-latest 12 | timeout-minutes: 60 13 | env: 14 | # Trigger Publish.settings ReleaseToSonatype branch 15 | USERNAME: ${{ github.actor }} 16 | SONATYPE_USER: ${{ secrets.SONATYPE_USER }} 17 | SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} 18 | # Passphrase for signing (sbt-pgp reads this) 19 | PGP_PASSPHRASE: ${{ secrets.GPG_PASS }} 20 | # gpg on headless runners 21 | 22 | steps: 23 | - name: Checkout 24 | # sbt-release needs tags and full history 25 | uses: actions/checkout@v4 26 | with: 27 | fetch-depth: 0 28 | 29 | - name: Set up Java 30 | uses: actions/setup-java@v4 31 | with: 32 | distribution: temurin 33 | java-version: '21' 34 | cache: sbt 35 | 36 | - name: Setup sbt 37 | uses: sbt/setup-sbt@v1 38 | 39 | - name: Cache Ivy and Coursier 40 | uses: actions/cache@v4 41 | with: 42 | path: | 43 | ~/.ivy2/cache 44 | ~/.cache/coursier 45 | ~/.sbt 46 | key: ${{ runner.os }}-sbt-${{ hashFiles('**/build.sbt', '**/project/**.sbt', '**/project/**.scala') }} 47 | restore-keys: | 48 | ${{ runner.os }}-sbt- 49 | 50 | - name: Import GPG private key 51 | shell: bash 52 | run: | 53 | mkdir -p ~/.gnupg 54 | chmod 700 ~/.gnupg 55 | echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf 56 | echo "use-agent" >> ~/.gnupg/gpg.conf 57 | echo "allow-loopback-pinentry" >> ~/.gnupg/gpg-agent.conf 58 | printf "%s" "${GPG_SECRET}" | gpg --batch --import 59 | # reload agent to pick up allow-loopback-pinentry 60 | gpgconf --kill gpg-agent || true 61 | env: 62 | GPG_SECRET: ${{ secrets.GPG_SECRET }} 63 | 64 | - name: Show GPG keys (debug) 65 | if: always() 66 | run: | 67 | gpg --version 68 | gpg --list-keys || true 69 | gpg --list-secret-keys || true 70 | 71 | - name: Configure Git identity for release 72 | run: | 73 | git config user.email "ci@users.noreply.github.com" 74 | git config user.name "GitHub Actions Release Bot" 75 | 76 | - name: Run tests 77 | run: sbt "++2.13.16; test" 78 | 79 | - name: Release to Sonatype (bundle + close + release) 80 | run: sbt "release with-defaults" 81 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/metered/MeteredDispatcherInstrumentation.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.metered 2 | 3 | import akka.dispatch.{Dispatcher, Mailbox} 4 | import akka.event.Logging.Error 5 | import akka.sensors.PrometheusCompat.GaugeLabelsCompat 6 | import akka.sensors.{AkkaSensors, DispatcherMetrics} 7 | 8 | import java.lang.management.{ManagementFactory, ThreadMXBean} 9 | import java.util.concurrent.RejectedExecutionException 10 | 11 | private[metered] trait MeteredDispatcherInstrumentation extends Dispatcher { 12 | protected def actorSystemName: String 13 | protected def metrics: DispatcherMetrics 14 | 15 | private lazy val wrapper = new DispatcherInstrumentationWrapper(metrics, configurator.config) 16 | private val threadMXBean: ThreadMXBean = ManagementFactory.getThreadMXBean 17 | private val interestingStateNames = Set("runnable", "waiting", "timed_waiting", "blocked") 18 | private val interestingStates = Thread.State.values.filter(s => interestingStateNames.contains(s.name().toLowerCase)) 19 | 20 | AkkaSensors.schedule( 21 | s"$id-states", 22 | () => { 23 | val threads = threadMXBean 24 | .getThreadInfo(threadMXBean.getAllThreadIds, 0) 25 | .filter(t => 26 | t != null 27 | && t.getThreadName.startsWith(s"$actorSystemName-$id") 28 | ) 29 | 30 | interestingStates foreach { state => 31 | val stateLabel = state.toString.toLowerCase 32 | metrics.threadStates 33 | .labels(id, stateLabel) 34 | .set(threads.count(_.getThreadState.name().equalsIgnoreCase(stateLabel))) 35 | } 36 | metrics.threads 37 | .labels(id) 38 | .set(threads.length) 39 | } 40 | ) 41 | 42 | override def execute(runnable: Runnable): Unit = wrapper(runnable, super.execute) 43 | 44 | protected[akka] override def registerForExecution(mbox: Mailbox, hasMessageHint: Boolean, hasSystemMessageHint: Boolean): Boolean = 45 | if (mbox.canBeScheduledForExecution(hasMessageHint, hasSystemMessageHint)) 46 | if (mbox.setAsScheduled()) 47 | try { 48 | wrapper(mbox, executorService.execute) 49 | true 50 | } catch { 51 | case _: RejectedExecutionException => 52 | try { 53 | wrapper(mbox, executorService.execute) 54 | true 55 | } catch { //Retry once 56 | case e: RejectedExecutionException => 57 | mbox.setAsIdle() 58 | eventStream.publish(Error(e, getClass.getName, getClass, "registerForExecution was rejected twice!")) 59 | throw e 60 | } 61 | } 62 | else false 63 | else false 64 | } 65 | -------------------------------------------------------------------------------- /project/Publish.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import Keys._ 3 | import sbtrelease.ReleasePlugin.autoImport._ 4 | import sbtrelease.ReleaseStateTransformations._ 5 | import xerial.sbt.Sonatype.SonatypeKeys._ 6 | 7 | object Publish { 8 | 9 | val SuppressJavaDocsAndSources = Seq( 10 | doc / sources := Seq(), 11 | packageDoc / publishArtifact := false, 12 | packageSrc / publishArtifact := false 13 | ) 14 | 15 | val ReleaseToSonatype = Seq( 16 | credentials ++= Seq( 17 | Credentials( 18 | "Sonatype Central", 19 | "central.sonatype.com", 20 | sys.env.getOrElse("SONATYPE_USER", ""), 21 | sys.env.getOrElse("SONATYPE_PASSWORD", "") 22 | ), 23 | Credentials( 24 | "GnuPG Key ID", 25 | "gpg", 26 | "80639E9F764EA1049652FDBBDA743228BD43ED35", // key identifier 27 | "ignored" // sbt-pgp uses PGP_PASSPHRASE; this field is ignored 28 | ) 29 | ), 30 | sonatypeProfileName := "nl.pragmasoft.sensors", 31 | // Central Publishing Portal (OSSRH EOL) 32 | sonatypeCredentialHost := "central.sonatype.com", 33 | sonatypeRepository := "https://central.sonatype.com/api", 34 | licenses := Seq("MIT" -> url("https://opensource.org/licenses/MIT")), 35 | homepage := Some(url("https://github.com/jacum/akka-sensors")), 36 | scmInfo := Some(ScmInfo(browseUrl = url("https://github.com/jacum/akka-sensors"), connection = "scm:git@github.com:jacum/akka-sensors.git")), 37 | pomExtra := ( 38 | 39 | 40 | PragmaSoft 41 | PragmaSoft 42 | 43 | 44 | ), 45 | publishMavenStyle := true, 46 | publishTo := sonatypePublishToBundle.value, 47 | Test / publishArtifact := false, 48 | packageDoc / publishArtifact := true, 49 | packageSrc / publishArtifact := true, 50 | pomIncludeRepository := (_ => false), 51 | releaseCrossBuild := true, 52 | releaseIgnoreUntrackedFiles := true, 53 | releaseProcess := Seq[ReleaseStep]( 54 | checkSnapshotDependencies, 55 | inquireVersions, 56 | runClean, 57 | setReleaseVersion, 58 | // runTest, // can't run test w/cross-version release 59 | releaseStepCommand("sonatypeBundleClean"), 60 | releaseStepCommandAndRemaining("+publishSigned"), 61 | releaseStepCommand("sonatypeCentralUpload"), 62 | releaseStepCommand("sonatypeCentralRelease") 63 | ) 64 | ) 65 | 66 | val settings = 67 | if (sys.env.contains("SONATYPE_USER")) { 68 | println(s"Releasing to Sonatype as ${sys.env("SONATYPE_USER")}") 69 | ReleaseToSonatype 70 | } else SuppressJavaDocsAndSources 71 | 72 | } 73 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/metered/DispatcherInstrumentationWrapper.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.metered 2 | 3 | import akka.sensors.PrometheusCompat.HistogramLabelsCompat 4 | import akka.sensors.dispatch.DispatcherInstrumentationWrapper.{InstrumentedRun, Run} 5 | import akka.sensors.dispatch.RunnableWrapper 6 | import akka.sensors.{DispatcherMetrics, RunnableWatcher} 7 | import com.typesafe.config.Config 8 | 9 | import java.util.concurrent.atomic.LongAdder 10 | import scala.concurrent.duration.Duration 11 | 12 | private[metered] class DispatcherInstrumentationWrapper(metrics: DispatcherMetrics, config: Config) { 13 | import DispatcherInstrumentationWrapper._ 14 | import akka.sensors.dispatch.Helpers._ 15 | 16 | private val executorConfig = config.getConfig("instrumented-executor") 17 | 18 | private val instruments: List[InstrumentedRun] = 19 | List( 20 | if (executorConfig.getBoolean("measure-runs")) Some(meteredRun(metrics, config.getString("id"))) else None, 21 | if (executorConfig.getBoolean("watch-long-runs")) 22 | Some(watchedRun(config.getString("id"), executorConfig.getMillisDuration("watch-too-long-run"), executorConfig.getMillisDuration("watch-check-interval"))) 23 | else None 24 | ) flatten 25 | 26 | def apply(runnable: Runnable, execute: Runnable => Unit): Unit = { 27 | val beforeRuns = for { f <- instruments } yield f() 28 | val run = new Run { 29 | def apply[T](run: () => T): T = { 30 | val afterRuns = for { f <- beforeRuns } yield f() 31 | try run() 32 | finally for { f <- afterRuns } f() 33 | } 34 | } 35 | execute(RunnableWrapper(runnable, run)) 36 | } 37 | } 38 | private[metered] object DispatcherInstrumentationWrapper { 39 | def meteredRun(metrics: DispatcherMetrics, id: String): InstrumentedRun = { 40 | val currentWorkers = new LongAdder 41 | val queue = metrics.queueTime.labels(id) 42 | val run = metrics.runTime.labels(id) 43 | val active = metrics.activeThreads.labels(id) 44 | 45 | () => { 46 | val created = System.currentTimeMillis() 47 | () => { 48 | val started = System.currentTimeMillis() 49 | queue.observe((started - created).toDouble) 50 | currentWorkers.increment() 51 | active.observe(currentWorkers.intValue()) 52 | () => { 53 | val stopped = System.currentTimeMillis() 54 | run.observe((stopped - started).toDouble) 55 | currentWorkers.decrement() 56 | active.observe(currentWorkers.intValue) 57 | () 58 | } 59 | } 60 | } 61 | } 62 | 63 | private def watchedRun(id: String, tooLongThreshold: Duration, checkInterval: Duration): InstrumentedRun = { 64 | val watcher = RunnableWatcher(tooLongRunThreshold = tooLongThreshold, checkInterval = checkInterval) 65 | 66 | () => { () => 67 | val stop = watcher.start() 68 | () => { 69 | stop() 70 | () 71 | } 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /project/Dependencies.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | 3 | //noinspection TypeAnnotation 4 | object Dependencies { 5 | 6 | val akkaInmemoryJournal = ("com.github.dnvriend" %% "akka-persistence-inmemory" % "2.5.15.2") 7 | .exclude("com.typesafe.akka", "akka-actor") 8 | .exclude("com.typesafe.akka", "akka-persistence") 9 | .exclude("com.typesafe.akka", "akka-persistence-query") 10 | .exclude("com.typesafe.akka", "akka-stream") 11 | .exclude("com.typesafe.akka", "akka-protobuf") 12 | 13 | object Logging { 14 | val slf4jversion = "2.0.17" 15 | val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jversion 16 | val logback = "ch.qos.logback" % "logback-classic" % "1.5.20" 17 | val scalaLogging = "com.typesafe.scala-logging" %% "scala-logging" % "3.9.6" 18 | val deps = Seq(slf4jApi, scalaLogging, logback) 19 | } 20 | 21 | object Akka { 22 | val akkaVersion = "2.6.21" 23 | val akkaManagementVersion = "1.0.9" 24 | val akkaPersistenceCassandraVersion = "1.0.5" 25 | val akkaHttpVersion = "10.2.1" 26 | 27 | val actor = "com.typesafe.akka" %% "akka-actor" % akkaVersion 28 | val typed = "com.typesafe.akka" %% "akka-actor-typed" % akkaVersion 29 | val persistence = "com.typesafe.akka" %% "akka-persistence" % akkaVersion 30 | val persistenceTyped = "com.typesafe.akka" %% "akka-persistence-typed" % akkaVersion 31 | val persistenceQuery = "com.typesafe.akka" %% "akka-persistence-query" % akkaVersion 32 | 33 | val cluster = "com.typesafe.akka" %% "akka-cluster" % akkaVersion 34 | val clusterTyped = "com.typesafe.akka" %% "akka-cluster-typed" % akkaVersion 35 | val clusterTools = "com.typesafe.akka" %% "akka-cluster-tools" % akkaVersion 36 | val slf4j = "com.typesafe.akka" %% "akka-slf4j" % akkaVersion 37 | 38 | val deps = Seq(actor, typed, persistence, persistenceTyped, persistenceQuery, cluster, clusterTyped, clusterTools, slf4j) ++ Logging.deps 39 | } 40 | 41 | object Prometheus { 42 | val hotspot = "io.prometheus" % "prometheus-metrics-instrumentation-jvm" % "1.4.1" 43 | val common = "io.prometheus" % "prometheus-metrics-core" % "1.4.1" 44 | val exposition = "io.prometheus" % "prometheus-metrics-exposition-textformats" % "1.4.1" 45 | val exporterCommon = "io.prometheus" % "prometheus-metrics-exporter-common" % "1.4.1" 46 | 47 | val jmx = "io.prometheus.jmx" % "collector" % "1.5.0" 48 | val snakeYaml = "org.yaml" % "snakeyaml" % "2.5" 49 | 50 | val deps = Seq(hotspot, common, exporterCommon, jmx, exposition, snakeYaml) 51 | } 52 | 53 | object TestTools { 54 | val log = "ch.qos.logback" % "logback-classic" % "1.5.20" 55 | val scalaTest = "org.scalatest" %% "scalatest" % "3.2.19" 56 | val deps = Logging.deps ++ Seq(scalaTest, akkaInmemoryJournal, log) map (_ % Test) 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/RunnableWatcher.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors 2 | 3 | import java.lang.management.{ManagementFactory, ThreadMXBean} 4 | import java.util.concurrent.Executors 5 | import akka.sensors.RunnableWatcher.stackTraceToString 6 | import com.typesafe.scalalogging.LazyLogging 7 | 8 | import scala.annotation.nowarn 9 | import scala.collection.concurrent.TrieMap 10 | import scala.concurrent.BlockContext.withBlockContext 11 | import scala.concurrent.duration._ 12 | import scala.concurrent.{BlockContext, CanAwait} 13 | import scala.util.control.NonFatal 14 | 15 | trait RunnableWatcher { 16 | def apply[T](f: => T): T 17 | def start(): () => Unit 18 | } 19 | 20 | object RunnableWatcher extends LazyLogging { 21 | 22 | type ThreadId = java.lang.Long 23 | 24 | type StartTime = java.lang.Long 25 | 26 | private lazy val threads = ManagementFactory.getThreadMXBean 27 | 28 | def apply( 29 | tooLongRunThreshold: Duration, 30 | checkInterval: Duration = 1.second, 31 | maxDepth: Int = 300, 32 | threads: ThreadMXBean = threads 33 | ): RunnableWatcher = { 34 | 35 | val cache = TrieMap.empty[ThreadId, StartTime] 36 | 37 | AkkaSensors.schedule( 38 | "RunnableWatcher", 39 | () => 40 | try { 41 | val currentTime = System.nanoTime() 42 | for { 43 | (threadId, startTime) <- cache 44 | duration = (currentTime - startTime).nanos 45 | if duration >= tooLongRunThreshold 46 | _ <- cache.remove(threadId) 47 | threadInfo <- Option(threads.getThreadInfo(threadId, maxDepth)) 48 | } { 49 | val threadName = threadInfo.getThreadName 50 | val stackTrace = threadInfo.getStackTrace 51 | val formattedStackTrace = stackTraceToString(stackTrace) 52 | logger.error(s"Detected a thread that is locked for ${duration.toMillis} ms: $threadName, current state:\t$formattedStackTrace") 53 | } 54 | } catch { 55 | case NonFatal(failure) => logger.error(s"failed to check hanging threads: $failure", failure) 56 | }, 57 | checkInterval 58 | ) 59 | 60 | val startWatching = (threadId: ThreadId) => { 61 | val startTime = System.nanoTime() 62 | cache.put(threadId, startTime) 63 | () 64 | } 65 | 66 | val stopWatching = (threadId: ThreadId) => { 67 | cache.remove(threadId) 68 | () 69 | } 70 | 71 | apply(startWatching, stopWatching) 72 | } 73 | 74 | def apply( 75 | startWatching: RunnableWatcher.ThreadId => Unit, 76 | stopWatching: RunnableWatcher.ThreadId => Unit 77 | ): RunnableWatcher = 78 | new RunnableWatcher { 79 | 80 | def apply[T](f: => T): T = { 81 | val stop = start() 82 | try f 83 | finally stop() 84 | } 85 | 86 | @nowarn // we don't care about [highly theoretical] thread ID mutability 87 | def start(): () => Unit = { 88 | val threadId = Thread.currentThread().getId 89 | startWatching(threadId) 90 | () => stopWatching(threadId) 91 | } 92 | } 93 | 94 | def stackTraceToString(xs: Array[StackTraceElement]): String = xs.mkString("\tat ", "\n\tat ", "") 95 | 96 | } 97 | -------------------------------------------------------------------------------- /akka-sensors/src/test/scala/akka/sensors/metered/MeteredLogicSpec.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.metered 2 | 3 | import akka.actor.BootstrapSetup 4 | import akka.actor.typed.{ActorSystem, DispatcherSelector, SpawnProtocol} 5 | import akka.sensors.DispatcherMetrics 6 | import com.typesafe.config.ConfigFactory 7 | import org.scalatest.freespec.AnyFreeSpec 8 | import org.scalatest.matchers.should.Matchers 9 | 10 | import scala.jdk.CollectionConverters._ 11 | import MeteredLogicSpec._ 12 | import akka.actor.setup.ActorSystemSetup 13 | import io.prometheus.metrics.model.registry.PrometheusRegistry 14 | 15 | /** 16 | * This spec contains checks for metrics gathering implemented in .metered package. 17 | */ 18 | class MeteredLogicSpec extends AnyFreeSpec with Matchers { 19 | "Metered logic" - { 20 | "collects metrics for runnables" in { 21 | val cr = new PrometheusRegistry() 22 | val metrics = DispatcherMetrics.make() 23 | metrics.allCollectors.foreach(cr.register) 24 | 25 | val withConfig = BootstrapSetup(cfg) 26 | val withMetrics = MeteredDispatcherSetup(metrics) 27 | val setup = ActorSystemSetup.create(withConfig, withMetrics) 28 | val actorSystem = ActorSystem[SpawnProtocol.Command](SpawnProtocol(), "test-system", setup) 29 | 30 | try { 31 | // here we get a metered dispatcher from a custom config 32 | // Avoid using it as the default dispatcher as it is going to be used by Akka itself. 33 | // In this case that usage will affect metrics we are testing 34 | val dispatcher = actorSystem.dispatchers.lookup(DispatcherSelector.fromConfig("our-test-dispatcher")) 35 | 36 | // check that samples in the metrics are not defined before running the test task 37 | val prevSamples = cr.scrape().iterator().asScala.toList.map(in => (in.getMetadata.getName, in)).toMap 38 | prevSamples("akka_sensors_dispatchers_queue_time_millis").getDataPoints shouldBe empty 39 | prevSamples("akka_sensors_dispatchers_run_time_millis").getDataPoints shouldBe empty 40 | prevSamples("akka_sensors_dispatchers_active_threads").getDataPoints shouldBe empty 41 | prevSamples("akka_sensors_dispatchers_thread_states").getDataPoints shouldBe empty 42 | prevSamples("akka_sensors_dispatchers_threads").getDataPoints shouldBe empty 43 | prevSamples("akka_sensors_dispatchers_executor_value").getDataPoints shouldBe empty 44 | 45 | dispatcher.execute(() => Thread.sleep(3000)) 46 | 47 | //Now we can check that these metrics contain some samples after 3 secs of execution 48 | val samples = cr.scrape().iterator().asScala.toList.map(in => (in.getMetadata.getName, in)).toMap 49 | samples("akka_sensors_dispatchers_queue_time_millis").getDataPoints should not be empty 50 | samples("akka_sensors_dispatchers_run_time_millis").getDataPoints should not be empty 51 | samples("akka_sensors_dispatchers_active_threads").getDataPoints should not be empty 52 | samples("akka_sensors_dispatchers_thread_states").getDataPoints shouldBe empty 53 | samples("akka_sensors_dispatchers_threads").getDataPoints shouldBe empty 54 | samples("akka_sensors_dispatchers_executor_value").getDataPoints shouldBe empty 55 | } finally actorSystem.terminate() 56 | } 57 | } 58 | } 59 | 60 | object MeteredLogicSpec { 61 | private val cfgStr = 62 | """ 63 | |our-test-dispatcher { 64 | | type = "akka.sensors.metered.MeteredDispatcherConfigurator" 65 | | instrumented-executor { 66 | | delegate = "java.util.concurrent.ForkJoinPool" 67 | | measure-runs = true 68 | | watch-long-runs = false 69 | | } 70 | |} 71 | |""".stripMargin 72 | 73 | private val cfg = ConfigFactory.parseString(cfgStr) 74 | } 75 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/metered/MeteredInstrumentedExecutor.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.metered 2 | 3 | import akka.dispatch.{DispatcherPrerequisites, ExecutorServiceConfigurator, ExecutorServiceFactory, ForkJoinExecutorConfigurator, ThreadPoolExecutorConfigurator} 4 | import akka.sensors.AkkaSensors 5 | import akka.sensors.PrometheusCompat.GaugeLabelsCompat 6 | import com.typesafe.config.Config 7 | 8 | import java.util.concurrent.{ExecutorService, ForkJoinPool, ThreadFactory, ThreadPoolExecutor} 9 | 10 | class MeteredInstrumentedExecutor(val config: Config, val prerequisites: DispatcherPrerequisites) extends ExecutorServiceConfigurator(config, prerequisites) { 11 | private val metrics = MeteredDispatcherSetup.setupOrThrow(prerequisites).metrics 12 | 13 | private lazy val delegate: ExecutorServiceConfigurator = 14 | serviceConfigurator(config.getString("instrumented-executor.delegate")) 15 | 16 | override def createExecutorServiceFactory(id: String, threadFactory: ThreadFactory): ExecutorServiceFactory = { 17 | val esf = delegate.createExecutorServiceFactory(id, threadFactory) 18 | new ExecutorServiceFactory { 19 | def createExecutorService: ExecutorService = { 20 | val es = esf.createExecutorService 21 | 22 | val activeCount = metrics.executorValue.labels(id, "activeCount") 23 | val corePoolSize = metrics.executorValue.labels(id, "corePoolSize") 24 | val largestPoolSize = metrics.executorValue.labels(id, "largestPoolSize") 25 | val maximumPoolSize = metrics.executorValue.labels(id, "maximumPoolSize") 26 | val queueSize = metrics.executorValue.labels(id, "queueSize") 27 | val completedTasks = metrics.executorValue.labels(id, "completedTasks") 28 | val poolSize = metrics.executorValue.labels(id, "poolSize") 29 | val steals = metrics.executorValue.labels(id, "steals") 30 | val parallelism = metrics.executorValue.labels(id, "parallelism") 31 | val queuedSubmissions = metrics.executorValue.labels(id, "queuedSubmissions") 32 | val queuedTasks = metrics.executorValue.labels(id, "queuedTasks") 33 | val runningThreads = metrics.executorValue.labels(id, "runningThreads") 34 | 35 | es match { 36 | case tp: ThreadPoolExecutor => 37 | AkkaSensors.schedule( 38 | id, 39 | () => { 40 | activeCount.set(tp.getActiveCount) 41 | corePoolSize.set(tp.getCorePoolSize) 42 | largestPoolSize.set(tp.getLargestPoolSize) 43 | maximumPoolSize.set(tp.getMaximumPoolSize) 44 | queueSize.set(tp.getQueue.size()) 45 | completedTasks.set(tp.getCompletedTaskCount.toDouble) 46 | poolSize.set(tp.getPoolSize) 47 | } 48 | ) 49 | 50 | case fj: ForkJoinPool => 51 | AkkaSensors.schedule( 52 | id, 53 | () => { 54 | poolSize.set(fj.getPoolSize) 55 | steals.set(fj.getStealCount.toDouble) 56 | parallelism.set(fj.getParallelism) 57 | activeCount.set(fj.getActiveThreadCount) 58 | queuedSubmissions.set(fj.getQueuedSubmissionCount) 59 | queuedTasks.set(fj.getQueuedTaskCount.toDouble) 60 | runningThreads.set(fj.getRunningThreadCount) 61 | } 62 | ) 63 | 64 | case _ => 65 | 66 | } 67 | 68 | es 69 | } 70 | } 71 | } 72 | 73 | private def serviceConfigurator(executor: String): ExecutorServiceConfigurator = 74 | executor match { 75 | case null | "" | "fork-join-executor" => new ForkJoinExecutorConfigurator(config.getConfig("fork-join-executor"), prerequisites) 76 | case "thread-pool-executor" => new ThreadPoolExecutorConfigurator(config.getConfig("thread-pool-executor"), prerequisites) 77 | case fqcn => 78 | val args = List(classOf[Config] -> config, classOf[DispatcherPrerequisites] -> prerequisites) 79 | prerequisites.dynamicAccess 80 | .createInstanceFor[ExecutorServiceConfigurator](fqcn, args) 81 | .recover({ 82 | case exception => 83 | throw new IllegalArgumentException( 84 | """Cannot instantiate ExecutorServiceConfigurator ("executor = [%s]"), defined in [%s], 85 | make sure it has an accessible constructor with a [%s,%s] signature""" 86 | .format(fqcn, config.getString("id"), classOf[Config], classOf[DispatcherPrerequisites]), 87 | exception 88 | ) 89 | }) 90 | .get 91 | } 92 | 93 | } 94 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Minimalist Akka Observability 2 | ![Maven Central](https://img.shields.io/maven-central/v/nl.pragmasoft.sensors/sensors-core_2.13?color=%2300AA00) 3 | 4 | **Non-intrusive native Prometheus collectors for Akka internals, negligible performance overhead, suitable for production use.** 5 | 6 | - Are you running (or about to run) Akka in production, full-throttle, and want to see what happens inside? Did your load tests produce some ask timeouts? thread starvation? threads behaving non-reactively? old code doing nasty blocking I/O? 7 | 8 | - Would be nice to use Cinnamon Telemetry, but LightBend subscription is out of reach? 9 | 10 | - Overhead created by Kamon doesn't look acceptable, especially when running full-throttle? 11 | 12 | - Already familiar with Prometheus/Grafana observability stack? 13 | 14 | If you answer 'yes' to most of the questions above, Akka Sensors may be the right choice for you: 15 | 16 | - Comprehensive feature set to make internals of your Akka visible, in any environment, including high-load production. 17 | 18 | - It is OSS/free, as in MIT license, and uses explicit, very lightweight instrumentation - yet is a treasure trove for a busy observability engineer. 19 | 20 | - Won't affect CPU costs, when running in public cloud. 21 | 22 | Actor dashboard: 23 | ![Actors](./docs/akka-actors.png) 24 | 25 | Dispatcher dashboard: 26 | ![Dispatchers](./docs/akka-dispatchers.png) 27 | 28 | ## Features 29 | 30 | ### Dispatchers 31 | - time of runnable waiting in queue (histogram) 32 | - time of runnable run (histogram) 33 | - implementation-specific ForkJoinPool and ThreadPool stats (gauges) 34 | - thread states, as seen from JMX ThreadInfo (histogram, updated once in X seconds) 35 | - active worker threads (histogram, updated on each runnable) 36 | 37 | ### Thread watcher 38 | - thread watcher, keeping eye on threads running suspiciously long, and reporting their stacktraces - to help you find blocking code quickly 39 | 40 | ### Basic actor stats 41 | - number of actors (gauge) 42 | - time of actor 'receive' run (histogram) 43 | - actor activity time (histogram) 44 | - unhandled messages (count) 45 | - exceptions (count) 46 | 47 | ### Persistent actor stats 48 | - recovery time (histogram) 49 | - number of recovery events (histogram) 50 | - persist time (histogram) 51 | - recovery failures (counter) 52 | - persist failures (counter) 53 | 54 | ### Cluster 55 | - cluster events, per type/member (counter) 56 | 57 | ### Java Virtual Machine (from Prometheus default collectors) 58 | - number of instances 59 | - start since / uptime 60 | - JVM version 61 | - memory pools 62 | - garbage collector 63 | 64 | ## Usage 65 | 66 | ### SBT dependency 67 | 68 | ``` 69 | libraryDependencies ++= 70 | Seq( 71 | "nl.pragmasoft.sensors" %% "sensors-core" % "1.0.0" 72 | ) 73 | ``` 74 | 75 | ### Application configuration 76 | 77 | Override `type` and `executor` with Sensors' instrumented executors. 78 | Add `akka.sensors.AkkaSensorsExtension` to extensions. 79 | 80 | ``` 81 | akka { 82 | 83 | actor { 84 | 85 | # main/global/default dispatcher 86 | 87 | default-dispatcher { 88 | type = "akka.sensors.dispatch.InstrumentedDispatcherConfigurator" 89 | executor = "akka.sensors.dispatch.InstrumentedExecutor" 90 | 91 | instrumented-executor { 92 | delegate = "fork-join-executor" 93 | measure-runs = true 94 | watch-long-runs = true 95 | watch-check-interval = 1s 96 | watch-too-long-run = 3s 97 | } 98 | } 99 | 100 | # some other dispatcher used in your app 101 | 102 | default-blocking-io-dispatcher { 103 | type = "akka.sensors.dispatch.InstrumentedDispatcherConfigurator" 104 | executor = "akka.sensors.dispatch.InstrumentedExecutor" 105 | 106 | instrumented-executor { 107 | delegate = "thread-pool-executor" 108 | measure-runs = true 109 | watch-long-runs = false 110 | } 111 | } 112 | } 113 | 114 | extensions = [ 115 | akka.sensors.AkkaSensorsExtension 116 | ] 117 | } 118 | 119 | ``` 120 | 121 | ### Using explicit/inline executor definition 122 | 123 | ``` 124 | akka { 125 | default-dispatcher { 126 | type = "akka.sensors.dispatch.InstrumentedDispatcherConfigurator" 127 | executor = "akka.sensors.dispatch.InstrumentedExecutor" 128 | 129 | instrumented-executor { 130 | delegate = "fork-join-executor" 131 | measure-runs = true 132 | watch-long-runs = false 133 | } 134 | 135 | fork-join-executor { 136 | parallelism-min = 6 137 | parallelism-factor = 1 138 | parallelism-max = 6 139 | } 140 | } 141 | } 142 | ``` 143 | 144 | 145 | ### Actors (classic) 146 | 147 | ``` 148 | # Non-persistent actors 149 | class MyImportantActor extends Actor with ActorMetrics { 150 | 151 | # This becomes label 'actor', default is simple class name 152 | # but you may segment it further 153 | # Just make sure the cardinality is sane (<100) 154 | override protected def actorTag: String = ... 155 | 156 | ... # your implementation 157 | } 158 | 159 | # Persistent actors 160 | class MyImportantPersistentActor extends Actor with PersistentActorMetrics { 161 | ... 162 | 163 | 164 | ``` 165 | 166 | ### Actors (typed) 167 | 168 | ``` 169 | val behavior = BehaviorMetrics[Command]("ActorLabel") # basic actor metrics 170 | .withReceiveTimeoutMetrics(TimeoutCmd) # provides metric for amount of received timeout commands 171 | .withPersistenceMetrics # if inner behavior is event sourced, persistence metrics would be collected 172 | .setup { ctx: ActorContext[Command] => 173 | ... # your implementation 174 | } 175 | ``` 176 | 177 | ### Internal parameters 178 | 179 | Some parameters of the Sensors library itself, that you may want to tune: 180 | ``` 181 | akka.sensors { 182 | thread-state-snapshot-period = 5s 183 | cluster-watch-enabled = false 184 | } 185 | ``` 186 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/SensorMetrics.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors 2 | 3 | import io.prometheus.metrics.core.metrics._ 4 | import io.prometheus.metrics.model.registry.{Collector, PrometheusRegistry} 5 | 6 | final case class SensorMetrics( 7 | activityTime: Histogram, 8 | activeActors: Gauge, 9 | unhandledMessages: Counter, 10 | exceptions: Counter, 11 | receiveTime: Histogram, 12 | receiveTimeouts: Counter, 13 | clusterEvents: Counter, 14 | clusterMembers: Gauge, 15 | recoveryTime: Histogram, 16 | recoveryToFirstEventTime: Histogram, 17 | persistTime: Histogram, 18 | recoveries: Counter, 19 | recoveryEvents: Counter, 20 | persistFailures: Counter, 21 | recoveryFailures: Counter, 22 | persistRejects: Counter, 23 | waitingForRecovery: Gauge, 24 | waitingForRecoveryTime: Histogram 25 | ) { 26 | val allCollectors: List[Collector] = List( 27 | activityTime, 28 | activeActors, 29 | unhandledMessages, 30 | exceptions, 31 | receiveTime, 32 | receiveTimeouts, 33 | clusterEvents, 34 | clusterMembers, 35 | recoveryTime, 36 | recoveryToFirstEventTime, 37 | persistTime, 38 | recoveries, 39 | recoveryEvents, 40 | persistFailures, 41 | recoveryFailures, 42 | persistRejects, 43 | waitingForRecovery, 44 | waitingForRecoveryTime 45 | ) 46 | } 47 | 48 | object SensorMetrics { 49 | 50 | def make(): SensorMetrics = 51 | SensorMetrics( 52 | activityTime = Histogram 53 | .builder() 54 | .classicUpperBounds(10) 55 | .name("akka_sensors_actor_activity_time_seconds") 56 | .help(s"Seconds of activity") 57 | .labelNames("actor") 58 | .build(), 59 | activeActors = Gauge 60 | .builder() 61 | .name("akka_sensors_actor_active_actors") 62 | .help(s"Active actors") 63 | .labelNames("actor") 64 | .build(), 65 | unhandledMessages = Counter 66 | .builder() 67 | .name("akka_sensors_actor_unhandled_messages") 68 | .help(s"Unhandled messages") 69 | .labelNames("actor", "message") 70 | .build(), 71 | exceptions = Counter 72 | .builder() 73 | .name("akka_sensors_actor_exceptions") 74 | .help(s"Exceptions thrown by actors") 75 | .labelNames("actor") 76 | .build(), 77 | receiveTime = Histogram 78 | .builder() 79 | .classicUpperBounds(10000) 80 | .name("akka_sensors_actor_receive_time_millis") 81 | .help(s"Millis to process receive") 82 | .labelNames("actor", "message") 83 | .build(), 84 | receiveTimeouts = Counter 85 | .builder() 86 | .name("akka_sensors_actor_receive_timeouts") 87 | .help("Number of receive timeouts") 88 | .labelNames("actor") 89 | .build(), 90 | clusterEvents = Counter 91 | .builder() 92 | .name("akka_sensors_actor_cluster_events") 93 | .help(s"Number of cluster events, per type") 94 | .labelNames("event", "member") 95 | .build(), 96 | clusterMembers = Gauge 97 | .builder() 98 | .name("akka_sensors_actor_cluster_members") 99 | .help(s"Cluster members") 100 | .build(), 101 | recoveryTime = Histogram 102 | .builder() 103 | .classicUpperBounds(10000) 104 | .name("akka_sensors_actor_recovery_time_millis") 105 | .help(s"Millis to process recovery") 106 | .labelNames("actor") 107 | .build(), 108 | recoveryToFirstEventTime = Histogram 109 | .builder() 110 | .classicUpperBounds(10000) 111 | .name("akka_sensors_actor_recovery_to_first_event_time_millis") 112 | .help(s"Millis to process recovery before first event is applied") 113 | .labelNames("actor") 114 | .build(), 115 | persistTime = Histogram 116 | .builder() 117 | .classicUpperBounds(10000) 118 | .name("akka_sensors_actor_persist_time_millis") 119 | .help(s"Millis to process single event persist") 120 | .labelNames("actor", "event") 121 | .build(), 122 | recoveries = Counter 123 | .builder() 124 | .name("akka_sensors_actor_recoveries") 125 | .help(s"Recoveries by actors") 126 | .labelNames("actor") 127 | .build(), 128 | recoveryEvents = Counter 129 | .builder() 130 | .name("akka_sensors_actor_recovery_events") 131 | .help(s"Recovery events by actors") 132 | .labelNames("actor") 133 | .build(), 134 | persistFailures = Counter 135 | .builder() 136 | .name("akka_sensors_actor_persist_failures") 137 | .help(s"Persist failures") 138 | .labelNames("actor") 139 | .build(), 140 | recoveryFailures = Counter 141 | .builder() 142 | .name("akka_sensors_actor_recovery_failures") 143 | .help(s"Recovery failures") 144 | .labelNames("actor") 145 | .build(), 146 | persistRejects = Counter 147 | .builder() 148 | .name("akka_sensors_actor_persist_rejects") 149 | .help(s"Persist rejects") 150 | .labelNames("actor") 151 | .build(), 152 | waitingForRecovery = Gauge 153 | .builder() 154 | .name("akka_sensors_actor_waiting_for_recovery_permit_actors") 155 | .help(s"Actors waiting for recovery permit") 156 | .labelNames("actor") 157 | .build(), 158 | waitingForRecoveryTime = Histogram 159 | .builder() 160 | .classicUpperBounds(10000) 161 | .name("akka_sensors_actor_waiting_for_recovery_permit_time_millis") 162 | .help(s"Millis from actor creation to recovery permit being granted") 163 | .labelNames("actor") 164 | .build() 165 | ) 166 | 167 | def makeAndRegister(cr: PrometheusRegistry): SensorMetrics = { 168 | val metrics = make() 169 | metrics.allCollectors.foreach(c => cr.register(c)) 170 | metrics 171 | } 172 | 173 | } 174 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/actor/ActorMetrics.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.actor 2 | 3 | import akka.actor.{Actor, ActorLogging, ReceiveTimeout} 4 | import akka.persistence.PersistentActor 5 | import akka.sensors.MetricOps.HistogramExtensions 6 | import akka.sensors.PrometheusCompat.{CounterLabelsCompat, GaugeLabelsCompat, HistogramLabelsCompat} 7 | import akka.sensors.{AkkaSensorsExtension, ClassNameUtil} 8 | 9 | import scala.collection.immutable 10 | import scala.util.control.NonFatal 11 | 12 | trait ActorMetrics extends Actor with ActorLogging { 13 | self: Actor => 14 | 15 | protected def actorLabel: String = ClassNameUtil.simpleName(this.getClass) 16 | 17 | protected def messageLabel(value: Any): Option[String] = Some(ClassNameUtil.simpleName(value.getClass)) 18 | 19 | protected val metrics = AkkaSensorsExtension(this.context.system).metrics 20 | private val receiveTimeouts = metrics.receiveTimeouts.labelValues(actorLabel) 21 | private lazy val exceptions = metrics.exceptions.labelValues(actorLabel) 22 | private val activeActors = metrics.activeActors.labelValues(actorLabel) 23 | 24 | private val activityTimer = metrics.activityTime.labelValues(actorLabel).startTimer() 25 | 26 | protected[akka] override def aroundReceive(receive: Receive, msg: Any): Unit = 27 | internalAroundReceive(receive, msg) 28 | 29 | protected def internalAroundReceive(receive: Receive, msg: Any): Unit = { 30 | msg match { 31 | case ReceiveTimeout => 32 | receiveTimeouts.inc() 33 | case _ => 34 | } 35 | try messageLabel(msg) 36 | .map( 37 | metrics.receiveTime 38 | .labelValues(actorLabel, _) 39 | .observeExecution(super.aroundReceive(receive, msg)) 40 | ) 41 | .getOrElse(super.aroundReceive(receive, msg)) 42 | catch { 43 | case NonFatal(e) => 44 | exceptions.inc() 45 | throw e 46 | } 47 | } 48 | protected[akka] override def aroundPreStart(): Unit = { 49 | super.aroundPreStart() 50 | activeActors.inc() 51 | } 52 | 53 | protected[akka] override def aroundPostStop(): Unit = { 54 | activeActors.dec() 55 | activityTimer.observeDuration() 56 | super.aroundPostStop() 57 | } 58 | 59 | override def unhandled(message: Any): Unit = { 60 | messageLabel(message) 61 | .foreach(metrics.unhandledMessages.labels(actorLabel, _).inc()) 62 | super.unhandled(message) 63 | } 64 | 65 | } 66 | 67 | trait PersistentActorMetrics extends ActorMetrics with PersistentActor { 68 | 69 | // normally we don't need to watch internal akka persistence messages 70 | protected override def messageLabel(value: Any): Option[String] = 71 | if (!recoveryFinished) None 72 | else // ignore commands while doing recovery, these are auto-stashed 73 | if (value.getClass.getName.startsWith("akka.persistence")) None // ignore akka persistence internal buzz 74 | else super.messageLabel(value) 75 | 76 | protected def eventLabel(value: Any): Option[String] = messageLabel(value) 77 | 78 | private var recovered: Boolean = false 79 | private var firstEventPassed: Boolean = false 80 | private lazy val recoveries = metrics.recoveries.labels(actorLabel) 81 | private lazy val recoveryEvents = metrics.recoveryEvents.labels(actorLabel) 82 | private val recoveryTime = metrics.recoveryTime.labels(actorLabel).startTimer() 83 | private val recoveryToFirstEventTime = metrics.recoveryTime.labels(actorLabel).startTimer() 84 | private lazy val recoveryFailures = metrics.recoveryFailures.labels(actorLabel) 85 | private lazy val persistFailures = metrics.persistFailures.labels(actorLabel) 86 | private lazy val persistRejects = metrics.persistRejects.labels(actorLabel) 87 | private val waitingForRecoveryGauge = metrics.waitingForRecovery.labels(actorLabel) 88 | private val waitingForRecoveryTime = metrics.waitingForRecoveryTime.labels(actorLabel).startTimer() 89 | 90 | waitingForRecoveryGauge.inc() 91 | 92 | protected[akka] override def aroundReceive(receive: Receive, msg: Any): Unit = { 93 | if (!recoveryFinished) 94 | ClassNameUtil.simpleName(msg.getClass) match { 95 | case msg if msg.startsWith("ReplayedMessage") => 96 | if (!firstEventPassed) { 97 | recoveryToFirstEventTime.observeDuration() 98 | firstEventPassed = true 99 | } 100 | recoveryEvents.inc() 101 | 102 | case msg if msg.startsWith("RecoveryPermitGranted") => 103 | waitingForRecoveryGauge.dec() 104 | waitingForRecoveryTime.observeDuration() 105 | 106 | case _ => () 107 | } 108 | else if (!recovered) { 109 | recoveries.inc() 110 | recoveryTime.observeDuration() 111 | recovered = true 112 | } 113 | internalAroundReceive(receive, msg) 114 | } 115 | 116 | override def persist[A](event: A)(handler: A => Unit): Unit = 117 | eventLabel(event) 118 | .map(label => 119 | metrics.persistTime 120 | .labels(actorLabel, label) 121 | .observeExecution( 122 | this.internalPersist(event)(handler) 123 | ) 124 | ) 125 | .getOrElse(this.internalPersist(event)(handler)) 126 | 127 | override def persistAll[A](events: immutable.Seq[A])(handler: A => Unit): Unit = 128 | metrics.persistTime 129 | .labels(actorLabel, "_all") 130 | .observeExecution( 131 | this.internalPersistAll(events)(handler) 132 | ) 133 | 134 | override def persistAsync[A](event: A)(handler: A => Unit): Unit = 135 | eventLabel(event) 136 | .map(label => 137 | metrics.persistTime 138 | .labels(actorLabel, label) 139 | .observeExecution( 140 | this.internalPersistAsync(event)(handler) 141 | ) 142 | ) 143 | .getOrElse(this.internalPersistAsync(event)(handler)) 144 | 145 | override def persistAllAsync[A](events: immutable.Seq[A])(handler: A => Unit): Unit = 146 | metrics.persistTime 147 | .labels(actorLabel, "_all") 148 | .observeExecution( 149 | this.internalPersistAllAsync(events)(handler) 150 | ) 151 | 152 | protected override def onRecoveryFailure(cause: Throwable, event: Option[Any]): Unit = { 153 | log.error(cause, "Recovery failed") 154 | recoveryFailures.inc() 155 | } 156 | protected override def onPersistFailure(cause: Throwable, event: Any, seqNr: Long): Unit = { 157 | log.error(cause, "Persist failed") 158 | persistFailures.inc() 159 | } 160 | protected override def onPersistRejected(cause: Throwable, event: Any, seqNr: Long): Unit = { 161 | log.error(cause, "Persist rejected") 162 | persistRejects.inc() 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/persistence/sensors/EventSourcedMetrics.scala: -------------------------------------------------------------------------------- 1 | package akka.persistence.sensors 2 | 3 | import akka.actor.typed.internal.BehaviorImpl.DeferredBehavior 4 | import akka.actor.typed.internal.InterceptorImpl 5 | import akka.actor.typed.scaladsl.Behaviors 6 | import akka.actor.typed.scaladsl.Behaviors.{ReceiveImpl, ReceiveMessageImpl} 7 | import akka.actor.typed.{Behavior, BehaviorInterceptor, ExtensibleBehavior, TypedActorContext} 8 | import akka.persistence.typed.internal.{CompositeEffect, EffectImpl, EventSourcedBehaviorImpl, Persist, PersistAll} 9 | import akka.persistence.typed.scaladsl.{EffectBuilder, EventSourcedBehavior} 10 | import akka.persistence.{RecoveryPermitter, JournalProtocol => P} 11 | import akka.sensors.MetricOps._ 12 | import akka.sensors.{ClassNameUtil, SensorMetrics} 13 | import com.typesafe.scalalogging.LazyLogging 14 | import akka.persistence.typed.scaladsl.Effect 15 | 16 | import scala.annotation.tailrec 17 | 18 | final case class EventSourcedMetrics[C, E, S]( 19 | actorLabel: String, 20 | metrics: SensorMetrics 21 | ) extends LazyLogging { 22 | 23 | private val recoveries = metrics.recoveries.labelValues(actorLabel) 24 | private val recoveryEvents = metrics.recoveryEvents.labelValues(actorLabel) 25 | private var firstEventPassed: Boolean = false 26 | private val recoveryTime = metrics.recoveryTime.labelValues(actorLabel).startTimer() 27 | private val recoveryToFirstEventTime = metrics.recoveryToFirstEventTime.labelValues(actorLabel).startTimer() 28 | private val recoveryFailures = metrics.recoveryFailures.labelValues(actorLabel) 29 | private val persistFailures = metrics.persistFailures.labelValues(actorLabel) 30 | private val persistRejects = metrics.persistRejects.labelValues(actorLabel) 31 | private val waitingForRecoveryGauge = metrics.waitingForRecovery.labelValues(actorLabel) 32 | private val waitingForRecoveryTime = metrics.waitingForRecoveryTime.labelValues(actorLabel).startTimer() 33 | 34 | waitingForRecoveryGauge.inc() 35 | 36 | def messageLabel(value: Any): Option[String] = 37 | Some(ClassNameUtil.simpleName(value.getClass)) 38 | 39 | def apply(behaviorToObserve: Behavior[C]): Behavior[C] = { 40 | 41 | val interceptor = () => 42 | new BehaviorInterceptor[Any, Any] { 43 | def aroundReceive( 44 | ctx: TypedActorContext[Any], 45 | msg: Any, 46 | target: BehaviorInterceptor.ReceiveTarget[Any] 47 | ): Behavior[Any] = { 48 | msg match { 49 | case res: P.Response => 50 | res match { 51 | case _: P.ReplayedMessage => 52 | if (!firstEventPassed) { 53 | recoveryToFirstEventTime.observeDuration() 54 | firstEventPassed = true 55 | } 56 | recoveryEvents.inc() 57 | case _: P.ReplayMessagesFailure => recoveryFailures.inc() 58 | case _: P.WriteMessageRejected => persistRejects.inc() 59 | case _: P.WriteMessageFailure => persistFailures.inc() 60 | case _: P.RecoverySuccess => 61 | recoveries.inc() 62 | recoveryTime.observeDuration() 63 | 64 | case _ => 65 | } 66 | 67 | case RecoveryPermitter.RecoveryPermitGranted => 68 | waitingForRecoveryGauge.dec() 69 | waitingForRecoveryTime.observeDuration() 70 | 71 | case _ => 72 | } 73 | 74 | target(ctx, msg) 75 | } 76 | } 77 | 78 | Behaviors.intercept(interceptor)(observedBehavior(behaviorToObserve).unsafeCast[Any]).narrow 79 | } 80 | 81 | /** 82 | * recursively inspects subsequent chain of behaviors in behaviorToObserve to find a [[EventSourcedBehaviorImpl]] 83 | * then the function overrides behavior's command handler to start a [[metrics.persistTime]] timer when 84 | * [[Persist]] / [[PersistAll]] / [[CompositeEffect]] effects being produced. 85 | */ 86 | @SuppressWarnings(Array("org.wartremover.warts.AsInstanceOf", "org.wartremover.warts.Recursion")) 87 | private def observedBehavior(behaviorToObserve: Behavior[C]): Behavior[C] = 88 | behaviorToObserve match { 89 | 90 | case eventSourced: EventSourcedBehaviorImpl[C @unchecked, E @unchecked, S @unchecked] => 91 | val observedCommandHandler: EventSourcedBehavior.CommandHandler[C, E, S] = (state: S, command: C) => { 92 | eventSourced.commandHandler(state, command) match { 93 | case eff: EffectImpl[E, S] => observeEffect(eff) 94 | // This case should never happen as `EffectImpl` is a parent for all real cases of `Effect` 95 | case other => other 96 | } 97 | } 98 | 99 | eventSourced.copy(commandHandler = observedCommandHandler) 100 | 101 | case deferred: DeferredBehavior[C] => 102 | Behaviors.setup(ctx => observedBehavior(deferred(ctx))) 103 | 104 | case receive: ReceiveImpl[C] => 105 | Behaviors.receive((ctx, msg) => observedBehavior(receive.onMessage(ctx, msg))) 106 | 107 | case receive: ReceiveMessageImpl[C @unchecked] => 108 | Behaviors.receiveMessage(msg => observedBehavior(receive.onMessage(msg))) 109 | 110 | case interceptor: InterceptorImpl[_, C @unchecked] => 111 | new InterceptorImpl( 112 | interceptor = interceptor.interceptor.asInstanceOf[BehaviorInterceptor[Any, C]], 113 | nestedBehavior = observedBehavior(interceptor.nestedBehavior) 114 | ).asInstanceOf[ExtensibleBehavior[C]] 115 | 116 | case other => other 117 | } 118 | 119 | private def observeEffect(effect: EffectImpl[E, S]): Effect[E, S] = { 120 | def foldComposites[E1, S1]( 121 | e: EffectBuilder[E1, S1], 122 | composites: List[CompositeEffect[E1, S1]] 123 | ): Effect[E1, S1] = 124 | composites.foldLeft(e)((e, c) => c.copy(persistingEffect = e)) 125 | 126 | @tailrec 127 | def loop[E1, S1]( 128 | e: EffectBuilder[E1, S1], 129 | composites: List[CompositeEffect[E1, S1]] 130 | ): Effect[E1, S1] = 131 | e match { 132 | case eff @ Persist(_) => 133 | val withMetrics = messageLabel(eff.event).map { label => 134 | metrics.persistTime 135 | .labelValues(actorLabel, label) 136 | .observeEffect(eff) 137 | } 138 | .getOrElse(eff) 139 | foldComposites(withMetrics, composites) 140 | 141 | case eff @ PersistAll(_) => 142 | val withMetrics = metrics.persistTime 143 | .labelValues(actorLabel, "_all") 144 | .observeEffect(eff) 145 | foldComposites(withMetrics, composites) 146 | 147 | case CompositeEffect(pe, effs) => 148 | loop(pe, CompositeEffect(pe, effs) :: composites) 149 | 150 | case other => foldComposites(other, composites) 151 | } 152 | 153 | loop(effect, Nil) 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/AkkaSensorsExtension.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors 2 | 3 | import akka.Done 4 | import akka.actor.{ActorSystem, ClassicActorSystemProvider, CoordinatedShutdown, ExtendedActorSystem, Extension, ExtensionId, ExtensionIdProvider, Props} 5 | import akka.persistence.typed.scaladsl.EffectBuilder 6 | import akka.sensors.actor.ClusterEventWatchActor 7 | import com.typesafe.config.ConfigFactory 8 | import com.typesafe.scalalogging.LazyLogging 9 | import io.prometheus.metrics.core.datapoints.DistributionDataPoint 10 | import io.prometheus.metrics.core.metrics.{Counter, Gauge, Histogram} 11 | import io.prometheus.metrics.model.registry.{Collector, PrometheusRegistry} 12 | 13 | import java.util.concurrent.{Executors, ScheduledExecutorService, TimeUnit} 14 | import scala.annotation.tailrec 15 | import scala.collection.concurrent.TrieMap 16 | import scala.concurrent.Future 17 | import scala.concurrent.duration.Duration 18 | import scala.util.Try 19 | 20 | object AkkaSensors extends LazyLogging { 21 | 22 | private val config = ConfigFactory.load().getConfig("akka.sensors") 23 | private val defaultPollInterval: Long = Try(config.getDuration("thread-state-snapshot-period", TimeUnit.SECONDS)).getOrElse(1L) 24 | val ClusterWatchEnabled: Boolean = Try(config.getBoolean("cluster-watch-enabled")).getOrElse(false) 25 | 26 | // single-thread dedicated executor for low-frequency (some seconds between calls) sensors' internal business 27 | private val executor: ScheduledExecutorService = Executors.newScheduledThreadPool(1) 28 | private val periodicPolls = new TrieMap[String, Runnable] 29 | 30 | def schedule(id: String, poll: Runnable, interval: Duration = Duration(defaultPollInterval, TimeUnit.SECONDS)): Unit = 31 | periodicPolls.getOrElseUpdate( 32 | id, { 33 | executor.scheduleWithFixedDelay(poll, interval.length, interval.length, interval.unit) 34 | logger.info(s"Scheduled activity: $id") 35 | poll 36 | } 37 | ) 38 | 39 | val prometheusRegistry: PrometheusRegistry = PrometheusRegistry.defaultRegistry // todo how to parametrise/hook to other metric exports? 40 | 41 | /** 42 | * Safer than Class obj's getSimpleName, which may throw Malformed class name error in scala. 43 | * This method mimics scalatest's getSimpleNameOfAnObjectsClass. 44 | */ 45 | def getSimpleName(cls: Class[_]): String = 46 | try cls.getSimpleName 47 | catch { 48 | // TODO: the value returned here isn't even quite right; it returns simple names 49 | // like UtilsSuite$MalformedClassObject$MalformedClass instead of MalformedClass 50 | // The exact value may not matter much as it's used in log statements 51 | case _: InternalError => 52 | stripDollars(stripPackages(cls.getName)) 53 | } 54 | 55 | /** 56 | * Remove the packages from full qualified class name 57 | */ 58 | private def stripPackages(fullyQualifiedName: String): String = 59 | fullyQualifiedName.split("\\.").takeRight(1)(0) 60 | 61 | /** 62 | * Remove trailing dollar signs from qualified class name, 63 | * and return the trailing part after the last dollar sign in the middle 64 | */ 65 | @tailrec 66 | private def stripDollars(s: String): String = { 67 | val lastDollarIndex = s.lastIndexOf('$') 68 | if (lastDollarIndex < s.length - 1) 69 | // The last char is not a dollar sign 70 | if (lastDollarIndex == -1 || !s.contains("$iw")) 71 | // The name does not have dollar sign or is not an interpreter 72 | // generated class, so we should return the full string 73 | s 74 | else 75 | // The class name is interpreter generated, 76 | // return the part after the last dollar sign 77 | // This is the same behavior as getClass.getSimpleName 78 | s.substring(lastDollarIndex + 1) 79 | else { 80 | // The last char is a dollar sign 81 | // Find last non-dollar char 82 | val lastNonDollarChar = s.findLast(_ != '$') 83 | lastNonDollarChar match { 84 | case None => s 85 | case Some(c) => 86 | val lastNonDollarIndex = s.lastIndexOf(c) 87 | if (lastNonDollarIndex == -1) 88 | s 89 | else 90 | // Strip the trailing dollar signs 91 | // Invoke stripDollars again to get the simple name 92 | stripDollars(s.substring(0, lastNonDollarIndex + 1)) 93 | } 94 | } 95 | } 96 | } 97 | 98 | /** 99 | * For overrides, make a subclass and put it's name in 'akka.sensors.extension-class' config value 100 | */ 101 | class AkkaSensorsExtension(system: ExtendedActorSystem) extends Extension with LazyLogging { 102 | 103 | val registry = PrometheusRegistry.defaultRegistry 104 | 105 | logger.info(s"Akka Sensors extension has been activated: ${this.getClass.getName}") 106 | if (AkkaSensors.ClusterWatchEnabled) 107 | system.actorOf(Props(classOf[ClusterEventWatchActor]), s"ClusterEventWatchActor") 108 | 109 | CoordinatedShutdown(system) 110 | .addTask(CoordinatedShutdown.PhaseBeforeActorSystemTerminate, "clearPrometheusRegistry") { () => 111 | allCollectors.foreach(c => this.registry.unregister(c)) 112 | logger.info("Cleared metrics") 113 | Future.successful(Done) 114 | } 115 | 116 | val metrics: SensorMetrics = SensorMetrics.makeAndRegister(this.registry) 117 | 118 | def activityTime: Histogram = metrics.activityTime 119 | def activeActors: Gauge = metrics.activeActors 120 | def unhandledMessages: Counter = metrics.unhandledMessages 121 | def exceptions: Counter = metrics.exceptions 122 | def receiveTime: Histogram = metrics.receiveTime 123 | def receiveTimeouts: Counter = metrics.receiveTimeouts 124 | def clusterEvents: Counter = metrics.clusterEvents 125 | def clusterMembers: Gauge = metrics.clusterMembers 126 | def recoveryTime: Histogram = metrics.recoveryTime 127 | def persistTime: Histogram = metrics.persistTime 128 | def recoveries: Counter = metrics.recoveries 129 | def recoveryEvents: Counter = metrics.recoveryEvents 130 | def persistFailures: Counter = metrics.persistFailures 131 | def recoveryFailures: Counter = metrics.recoveryFailures 132 | def persistRejects: Counter = metrics.persistRejects 133 | def waitingForRecovery: Gauge = metrics.waitingForRecovery 134 | def waitingForRecoveryTime: Histogram = metrics.waitingForRecoveryTime 135 | 136 | def allCollectors: List[Collector] = metrics.allCollectors 137 | } 138 | 139 | object AkkaSensorsExtension extends ExtensionId[AkkaSensorsExtension] with ExtensionIdProvider { 140 | override def lookup: ExtensionId[_ <: Extension] = AkkaSensorsExtension 141 | override def createExtension(system: ExtendedActorSystem): AkkaSensorsExtension = { 142 | val extensionClass = ConfigFactory.load().getString("akka.sensors.extension-class") 143 | Class.forName(extensionClass).getDeclaredConstructor(classOf[ExtendedActorSystem]).newInstance(system) match { 144 | case w: AkkaSensorsExtension => w 145 | case _ => throw new IllegalArgumentException(s"Class $extensionClass must extend com.ing.bakery.baker.Watcher") 146 | } 147 | } 148 | override def get(system: ActorSystem): AkkaSensorsExtension = super.get(system) 149 | override def get(system: ClassicActorSystemProvider): AkkaSensorsExtension = super.get(system) 150 | } 151 | 152 | object MetricOps { 153 | 154 | implicit class HistogramExtensions(val histogram: DistributionDataPoint) { 155 | def observeExecution[A](f: => A): A = { 156 | val timer = histogram.startTimer() 157 | try f 158 | finally timer.observeDuration() 159 | } 160 | 161 | def observeEffect[E, S](eff: EffectBuilder[E, S]): EffectBuilder[E, S] = { 162 | val timer = histogram.startTimer() 163 | eff.thenRun(_ => timer.observeDuration()) 164 | } 165 | } 166 | 167 | } 168 | -------------------------------------------------------------------------------- /akka-sensors/src/main/scala/akka/sensors/dispatch/InstrumentedDispatchers.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors.dispatch 2 | 3 | import java.lang.management.{ManagementFactory, ThreadInfo, ThreadMXBean} 4 | import java.util.concurrent._ 5 | import java.util.concurrent.atomic.LongAdder 6 | import akka.dispatch._ 7 | import akka.event.Logging.{Error, Warning} 8 | import akka.sensors.PrometheusCompat.{GaugeLabelsCompat, HistogramLabelsCompat} 9 | import akka.sensors.dispatch.DispatcherInstrumentationWrapper.Run 10 | import akka.sensors.{AkkaSensors, RunnableWatcher} 11 | import com.typesafe.config.Config 12 | 13 | import scala.PartialFunction.condOpt 14 | import scala.concurrent.duration.{Duration, FiniteDuration} 15 | import akka.sensors.dispatch.ScalaRunnableWrapper 16 | 17 | object AkkaRunnableWrapper { 18 | def unapply(runnable: Runnable): Option[Run => Runnable] = 19 | condOpt(runnable) { 20 | case runnable: Batchable => new BatchableWrapper(runnable, _) 21 | case runnable: Mailbox => new MailboxWrapper(runnable, _) 22 | } 23 | 24 | class BatchableWrapper(self: Batchable, r: Run) extends Batchable { 25 | def run(): Unit = r(() => self.run()) 26 | def isBatchable: Boolean = self.isBatchable 27 | } 28 | 29 | class MailboxWrapper(self: Mailbox, r: Run) extends ForkJoinTask[Unit] with Runnable { 30 | def getRawResult: Unit = self.getRawResult() 31 | def setRawResult(v: Unit): Unit = self.setRawResult(v) 32 | def exec(): Boolean = r(() => self.exec()) 33 | def run(): Unit = { exec(); () } 34 | } 35 | } 36 | 37 | class DispatcherInstrumentationWrapper(config: Config) { 38 | import DispatcherInstrumentationWrapper._ 39 | import Helpers._ 40 | 41 | private val executorConfig = config.getConfig("instrumented-executor") 42 | 43 | private val instruments: List[InstrumentedRun] = 44 | List( 45 | if (executorConfig.getBoolean("measure-runs")) Some(meteredRun(config.getString("id"))) else None, 46 | if (executorConfig.getBoolean("watch-long-runs")) 47 | Some(watchedRun(config.getString("id"), executorConfig.getMillisDuration("watch-too-long-run"), executorConfig.getMillisDuration("watch-check-interval"))) 48 | else None 49 | ) flatten 50 | 51 | def apply(runnable: Runnable, execute: Runnable => Unit): Unit = { 52 | val beforeRuns = for { f <- instruments } yield f() 53 | val run = new Run { 54 | def apply[T](run: () => T): T = { 55 | val afterRuns = for { f <- beforeRuns } yield f() 56 | try run() 57 | finally for { f <- afterRuns } f() 58 | } 59 | } 60 | execute(RunnableWrapper(runnable, run)) 61 | } 62 | } 63 | 64 | object RunnableWrapper { 65 | def apply(runnableParam: Runnable, r: Run): Runnable = 66 | runnableParam match { 67 | case AkkaRunnableWrapper(runnable) => runnable.apply(r) 68 | case ScalaRunnableWrapper(runnable) => runnable.apply(r) 69 | case runnable => new Default(runnable, r) 70 | } 71 | 72 | private class Default(self: Runnable, r: Run) extends Runnable { 73 | def run(): Unit = r(() => self.run()) 74 | } 75 | } 76 | 77 | object DispatcherInstrumentationWrapper { 78 | trait Run { def apply[T](f: () => T): T } 79 | 80 | type InstrumentedRun = () => BeforeRun 81 | type BeforeRun = () => AfterRun 82 | type AfterRun = () => Unit 83 | 84 | val Empty: InstrumentedRun = () => () => () => () 85 | 86 | import DispatcherMetricsRegistration._ 87 | def meteredRun(id: String): InstrumentedRun = { 88 | val currentWorkers = new LongAdder 89 | val queue = queueTime.labels(id) 90 | val run = runTime.labels(id) 91 | val active = activeThreads.labels(id) 92 | 93 | () => { 94 | val created = System.currentTimeMillis() 95 | () => { 96 | val started = System.currentTimeMillis() 97 | queue.observe((started - created).toDouble) 98 | currentWorkers.increment() 99 | active.observe(currentWorkers.intValue()) 100 | () => { 101 | val stopped = System.currentTimeMillis() 102 | run.observe((stopped - started).toDouble) 103 | currentWorkers.decrement() 104 | active.observe(currentWorkers.intValue) 105 | () 106 | } 107 | } 108 | } 109 | } 110 | 111 | def watchedRun(id: String, tooLongThreshold: Duration, checkInterval: Duration): InstrumentedRun = { 112 | val watcher = RunnableWatcher(tooLongRunThreshold = tooLongThreshold, checkInterval = checkInterval) 113 | 114 | () => { () => 115 | val stop = watcher.start() 116 | () => { 117 | stop() 118 | () 119 | } 120 | } 121 | } 122 | } 123 | 124 | class InstrumentedExecutor(val config: Config, val prerequisites: DispatcherPrerequisites) extends ExecutorServiceConfigurator(config, prerequisites) { 125 | 126 | lazy val delegate: ExecutorServiceConfigurator = 127 | serviceConfigurator(config.getString("instrumented-executor.delegate")) 128 | 129 | override def createExecutorServiceFactory(id: String, threadFactory: ThreadFactory): ExecutorServiceFactory = { 130 | val esf = delegate.createExecutorServiceFactory(id, threadFactory) 131 | import DispatcherMetricsRegistration._ 132 | new ExecutorServiceFactory { 133 | def createExecutorService: ExecutorService = { 134 | val es = esf.createExecutorService 135 | 136 | lazy val activeCount = executorValue.labels(id, "activeCount") 137 | lazy val corePoolSize = executorValue.labels(id, "corePoolSize") 138 | lazy val largestPoolSize = executorValue.labels(id, "largestPoolSize") 139 | lazy val maximumPoolSize = executorValue.labels(id, "maximumPoolSize") 140 | lazy val queueSize = executorValue.labels(id, "queueSize") 141 | lazy val completedTasks = executorValue.labels(id, "completedTasks") 142 | lazy val poolSize = executorValue.labels(id, "poolSize") 143 | lazy val steals = executorValue.labels(id, "steals") 144 | lazy val parallelism = executorValue.labels(id, "parallelism") 145 | lazy val queuedSubmissions = executorValue.labels(id, "queuedSubmissions") 146 | lazy val queuedTasks = executorValue.labels(id, "queuedTasks") 147 | lazy val runningThreads = executorValue.labels(id, "runningThreads") 148 | 149 | es match { 150 | case tp: ThreadPoolExecutor => 151 | AkkaSensors.schedule( 152 | id, 153 | () => { 154 | activeCount.set(tp.getActiveCount) 155 | corePoolSize.set(tp.getCorePoolSize) 156 | largestPoolSize.set(tp.getLargestPoolSize) 157 | maximumPoolSize.set(tp.getMaximumPoolSize) 158 | queueSize.set(tp.getQueue.size()) 159 | completedTasks.set(tp.getCompletedTaskCount.toDouble) 160 | poolSize.set(tp.getPoolSize) 161 | } 162 | ) 163 | 164 | case fj: ForkJoinPool => 165 | AkkaSensors.schedule( 166 | id, 167 | () => { 168 | poolSize.set(fj.getPoolSize) 169 | steals.set(fj.getStealCount.toDouble) 170 | parallelism.set(fj.getParallelism) 171 | activeCount.set(fj.getActiveThreadCount) 172 | queuedSubmissions.set(fj.getQueuedSubmissionCount) 173 | queuedTasks.set(fj.getQueuedTaskCount.toDouble) 174 | runningThreads.set(fj.getRunningThreadCount) 175 | } 176 | ) 177 | 178 | case _ => 179 | 180 | } 181 | 182 | es 183 | } 184 | } 185 | } 186 | 187 | def serviceConfigurator(executor: String): ExecutorServiceConfigurator = 188 | executor match { 189 | case null | "" | "fork-join-executor" => new ForkJoinExecutorConfigurator(config.getConfig("fork-join-executor"), prerequisites) 190 | case "thread-pool-executor" => new ThreadPoolExecutorConfigurator(config.getConfig("thread-pool-executor"), prerequisites) 191 | case fqcn => 192 | val args = List(classOf[Config] -> config, classOf[DispatcherPrerequisites] -> prerequisites) 193 | prerequisites.dynamicAccess 194 | .createInstanceFor[ExecutorServiceConfigurator](fqcn, args) 195 | .recover({ 196 | case exception => 197 | throw new IllegalArgumentException( 198 | """Cannot instantiate ExecutorServiceConfigurator ("executor = [%s]"), defined in [%s], 199 | make sure it has an accessible constructor with a [%s,%s] signature""" 200 | .format(fqcn, config.getString("id"), classOf[Config], classOf[DispatcherPrerequisites]), 201 | exception 202 | ) 203 | }) 204 | .get 205 | } 206 | 207 | } 208 | 209 | trait InstrumentedDispatcher extends Dispatcher { 210 | 211 | def actorSystemName: String 212 | private lazy val wrapper = new DispatcherInstrumentationWrapper(configurator.config) 213 | 214 | private val threadMXBean: ThreadMXBean = ManagementFactory.getThreadMXBean 215 | private val interestingStateNames = Set("runnable", "waiting", "timed_waiting", "blocked") 216 | private val interestingStates = Thread.State.values.filter(s => interestingStateNames.contains(s.name().toLowerCase)) 217 | 218 | AkkaSensors.schedule( 219 | s"$id-states", 220 | () => { 221 | val threads = threadMXBean 222 | .getThreadInfo(threadMXBean.getAllThreadIds, 0) 223 | .filter(t => 224 | t != null 225 | && t.getThreadName.startsWith(s"$actorSystemName-$id") 226 | ) 227 | 228 | interestingStates foreach { state => 229 | val stateLabel = state.toString.toLowerCase 230 | DispatcherMetricsRegistration.threadStates 231 | .labels(id, stateLabel) 232 | .set(threads.count(_.getThreadState.name().equalsIgnoreCase(stateLabel))) 233 | } 234 | DispatcherMetricsRegistration.threads 235 | .labels(id) 236 | .set(threads.length) 237 | } 238 | ) 239 | 240 | override def execute(runnable: Runnable): Unit = wrapper(runnable, super.execute) 241 | 242 | protected[akka] override def registerForExecution(mbox: Mailbox, hasMessageHint: Boolean, hasSystemMessageHint: Boolean): Boolean = 243 | if (mbox.canBeScheduledForExecution(hasMessageHint, hasSystemMessageHint)) 244 | if (mbox.setAsScheduled()) 245 | try { 246 | wrapper(mbox, executorService.execute) 247 | true 248 | } catch { 249 | case _: RejectedExecutionException => 250 | try { 251 | wrapper(mbox, executorService.execute) 252 | true 253 | } catch { //Retry once 254 | case e: RejectedExecutionException => 255 | mbox.setAsIdle() 256 | eventStream.publish(Error(e, getClass.getName, getClass, "registerForExecution was rejected twice!")) 257 | throw e 258 | } 259 | } 260 | else false 261 | else false 262 | } 263 | 264 | class InstrumentedDispatcherConfigurator(config: Config, prerequisites: DispatcherPrerequisites) extends MessageDispatcherConfigurator(config, prerequisites) { 265 | 266 | import Helpers._ 267 | 268 | private val instance = new Dispatcher( 269 | this, 270 | config.getString("id"), 271 | config.getInt("throughput"), 272 | config.getNanosDuration("throughput-deadline-time"), 273 | configureExecutor(), 274 | config.getMillisDuration("shutdown-timeout") 275 | ) with InstrumentedDispatcher { 276 | def actorSystemName: String = prerequisites.mailboxes.settings.name 277 | } 278 | 279 | def dispatcher(): MessageDispatcher = instance 280 | 281 | } 282 | 283 | class InstrumentedPinnedDispatcherConfigurator(config: Config, prerequisites: DispatcherPrerequisites) extends MessageDispatcherConfigurator(config, prerequisites) { 284 | import Helpers._ 285 | 286 | private val threadPoolConfig: ThreadPoolConfig = configureExecutor() match { 287 | case e: ThreadPoolExecutorConfigurator => e.threadPoolConfig 288 | case _ => 289 | prerequisites.eventStream.publish( 290 | Warning( 291 | "PinnedDispatcherConfigurator", 292 | this.getClass, 293 | "PinnedDispatcher [%s] not configured to use ThreadPoolExecutor, falling back to default config.".format(config.getString("id")) 294 | ) 295 | ) 296 | ThreadPoolConfig() 297 | } 298 | 299 | override def dispatcher(): MessageDispatcher = 300 | new PinnedDispatcher(this, null, config.getString("id"), config.getMillisDuration("shutdown-timeout"), threadPoolConfig) with InstrumentedDispatcher { 301 | def actorSystemName: String = prerequisites.mailboxes.settings.name 302 | } 303 | 304 | } 305 | 306 | object Helpers { 307 | 308 | /** 309 | * INTERNAL API 310 | */ 311 | private[akka] implicit final class ConfigOps(val config: Config) extends AnyVal { 312 | def getMillisDuration(path: String): FiniteDuration = getDuration(path, TimeUnit.MILLISECONDS) 313 | 314 | def getNanosDuration(path: String): FiniteDuration = getDuration(path, TimeUnit.NANOSECONDS) 315 | 316 | private def getDuration(path: String, unit: TimeUnit): FiniteDuration = 317 | Duration(config.getDuration(path, unit), unit) 318 | } 319 | } 320 | -------------------------------------------------------------------------------- /akka-sensors/src/test/scala/akka/sensors/AkkaSensorsSpec.scala: -------------------------------------------------------------------------------- 1 | package akka.sensors 2 | 3 | import akka.actor.typed.Behavior 4 | import akka.actor.typed.scaladsl.adapter._ 5 | import akka.actor.{Actor, ActorRef, ActorSystem, NoSerializationVerificationNeeded, PoisonPill, Props, ReceiveTimeout} 6 | import akka.pattern.ask 7 | import akka.persistence.typed.PersistenceId 8 | import akka.persistence.typed.scaladsl.{Effect, EventSourcedBehavior} 9 | import akka.persistence.{PersistentActor, RecoveryCompleted} 10 | import akka.sensors.actor.{ActorMetrics, PersistentActorMetrics} 11 | import akka.sensors.behavior.BehaviorMetrics 12 | import akka.util.Timeout 13 | import com.typesafe.scalalogging.LazyLogging 14 | import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter 15 | import io.prometheus.metrics.model.registry.PrometheusRegistry 16 | import org.scalatest.BeforeAndAfterAll 17 | import org.scalatest.concurrent.Eventually 18 | import org.scalatest.freespec.AnyFreeSpec 19 | import org.scalatest.time.{Millis, Seconds, Span} 20 | 21 | import java.io.ByteArrayOutputStream 22 | import scala.Console.println 23 | import scala.concurrent.duration._ 24 | import scala.concurrent.{Await, ExecutionContext} 25 | import scala.util.Random 26 | 27 | class AkkaSensorsSpec extends AnyFreeSpec with LazyLogging with Eventually with BeforeAndAfterAll { 28 | 29 | import InstrumentedActors._ 30 | implicit override val patienceConfig: PatienceConfig = 31 | PatienceConfig(timeout = scaled(Span(2, Seconds)), interval = scaled(Span(5, Millis))) 32 | 33 | implicit val ec: ExecutionContext = ExecutionContext.global 34 | private val system: ActorSystem = ActorSystem("instrumented") 35 | private lazy val probeActor = system.actorOf(Props(classOf[InstrumentedProbe]), s"probe") 36 | private lazy val persistentActor = system.actorOf(Props(classOf[PersistentInstrumentedProbe]), s"persistent") 37 | private lazy val persistentActor2 = system.actorOf(Props(classOf[AnotherPersistentInstrumentedProbe]), s"another-persistent") 38 | implicit val registry: PrometheusRegistry = AkkaSensors.prometheusRegistry 39 | 40 | "Launch akka app, and ensure it works" - { 41 | 42 | "starts actor system and pings the bootstrap actor" in { 43 | pingActor 44 | } 45 | 46 | "ensure prometheus JMX scraping is working" in { 47 | 48 | for (_ <- 1 to 5) probeActor ! KnownError 49 | for (_ <- 1 to 100) probeActor ! UnknownMessage 50 | probeActor ! BlockTooLong 51 | for (_ <- 1 to 1000) 52 | pingActor 53 | 54 | probeActor ! PoisonPill 55 | 56 | for (_ <- 1 to 100) sendEventAck(persistentActor) 57 | 58 | for (_ <- 1 to 100) sendEventAck(persistentActor2) 59 | 60 | persistentActor ! PoisonPill 61 | persistentActor2 ! PoisonPill 62 | 63 | Thread.sleep(100) 64 | 65 | val blockingIo = system.dispatchers.lookup("akka.actor.default-blocking-io-dispatcher") 66 | blockingIo.execute(() => Thread.sleep(100)) 67 | 68 | system.actorOf(Props(classOf[PersistentInstrumentedProbe]), s"persistent") 69 | system.actorOf(Props(classOf[AnotherPersistentInstrumentedProbe]), s"another-persistent") 70 | 71 | val content = metrics.split("\n") 72 | List("akka_sensors_actor", "akka_sensors_dispatchers").foreach(s => assert(content.exists(_.startsWith(s)), s"starts with $s")) 73 | } 74 | 75 | "ensure MANY actors are created and stopped, all accounted for" in { 76 | val actors = 50000 77 | val refs = (1 to actors).map(v => system.actorOf(Props(classOf[MassProbe]), s"mass-$v")) 78 | 79 | implicit val patienceConfig: PatienceConfig = PatienceConfig(20 seconds, 100 milliseconds) 80 | eventually { 81 | assertMetrics( 82 | _.startsWith("akka_sensors_actor_active_actors{actor=\"MassProbe\""), 83 | _.endsWith(s" $actors.0") 84 | ) 85 | } 86 | eventually { 87 | refs.foreach(r => assert(!r.isTerminated)) 88 | } 89 | refs.foreach(_ ! Ping("1")) 90 | eventually { 91 | assertMetrics( 92 | _.startsWith("akka_sensors_actor_receive_time_millis_count{actor=\"MassProbe\""), 93 | _.endsWith(s" $actors") 94 | ) 95 | } 96 | eventually { 97 | assertMetrics( 98 | _.startsWith("akka_sensors_actor_active_actors{actor=\"MassProbe\""), 99 | _.endsWith(s" 0.0") 100 | ) 101 | assertMetrics( 102 | _.startsWith("akka_sensors_actor_receive_timeouts_total{actor=\"MassProbe\""), 103 | _.endsWith(s" $actors.0") 104 | ) 105 | } 106 | assertMetrics( 107 | _.startsWith("akka_sensors_actor_activity_time_seconds_bucket{actor=\"MassProbe\",le=\"10.0\""), 108 | _.endsWith(s" $actors") 109 | ) 110 | } 111 | 112 | "ensure many classic persistent are created and stopped, all accounted for" in new PersistentScope { 113 | def actorName: String = "MassPersistentProbe" 114 | 115 | def createRef(idx: Int): ActorRef = 116 | system.actorOf(Props(classOf[MassPersistentProbe]), s"mass-classic-persistent-$idx") 117 | } 118 | 119 | "ensure many typed persistent are created and stopped, all accounted for" in new PersistentScope { 120 | def actorName: String = "MassTypedPersistentProbe" 121 | 122 | def createRef(idx: Int): ActorRef = 123 | system.spawn(MassTypedPersistentProbe(), s"mass-typed-persistent-$idx").ref.toClassic 124 | } 125 | 126 | trait PersistentScope { 127 | def actors = 1000 128 | def commands = 10 129 | implicit val patienceConfig: PatienceConfig = PatienceConfig(10 seconds, 100 milliseconds) 130 | 131 | def actorName: String 132 | 133 | def createRef(idx: Int): ActorRef 134 | 135 | val refs: Seq[ActorRef] = (1 to actors).map(createRef) 136 | 137 | eventually { 138 | assertMetrics( 139 | _.startsWith(s"""akka_sensors_actor_active_actors{actor="$actorName""""), 140 | _.endsWith(s" $actors.0") 141 | ) 142 | } 143 | eventually { 144 | refs.foreach(r => assert(!r.isTerminated)) 145 | } 146 | eventually { 147 | assertMetrics( 148 | _.startsWith(s"""akka_sensors_actor_active_actors{actor="$actorName""""), 149 | _.endsWith(s" $actors.0") 150 | ) 151 | } 152 | 153 | for (e <- 1 to commands) refs.foreach { a => 154 | a ! ValidCommand(e.toString) 155 | } 156 | 157 | eventually { 158 | assertMetrics( 159 | _.startsWith(s"""akka_sensors_actor_persist_time_millis_count{actor="$actorName"""), 160 | _.endsWith(s" ${actors * commands}") 161 | ) 162 | } 163 | 164 | eventually { 165 | assertMetrics( 166 | _.startsWith(s"""akka_sensors_actor_active_actors{actor="$actorName""""), 167 | _.endsWith(s" 0.0") 168 | ) 169 | assertMetrics( 170 | _.startsWith(s"""akka_sensors_actor_receive_timeouts_total{actor="$actorName""""), 171 | _.endsWith(s" $actors.0") 172 | ) 173 | } 174 | 175 | assertMetrics( 176 | _.startsWith(s"""akka_sensors_actor_activity_time_seconds_bucket{actor="$actorName",le="10.0""""), 177 | _.endsWith(s" $actors") 178 | ) 179 | 180 | assertMetrics( 181 | _.startsWith(s"""akka_sensors_actor_persist_time_millis_count{actor="$actorName",event="ValidEvent""""), 182 | _.endsWith(s" ${actors * commands}") 183 | ) 184 | 185 | val refRecovered = (1 to actors).map(createRef) 186 | 187 | eventually { 188 | assertMetrics( 189 | _.startsWith(s"""akka_sensors_actor_active_actors{actor="$actorName""""), 190 | _.endsWith(s" $actors.0") 191 | ) 192 | } 193 | 194 | for (e <- commands + 1 to commands + 2) refs.foreach(_ ! ValidCommand(e.toString)) 195 | 196 | eventually { 197 | assertMetrics( 198 | _.startsWith(s"""akka_sensors_actor_recoveries_total{actor="$actorName""""), 199 | _.endsWith(s" ${actors * 2}.0") 200 | ) 201 | } 202 | 203 | assertMetrics( 204 | _.startsWith(s"""akka_sensors_actor_recovery_events_total{actor="$actorName""""), 205 | _.endsWith(s" ${actors * commands}.0") 206 | ) 207 | 208 | assertMetrics( 209 | _.startsWith(s"""akka_sensors_actor_waiting_for_recovery_permit_actors{actor="$actorName""""), 210 | _.endsWith(s" 0.0") 211 | ) 212 | 213 | assertMetrics( 214 | _.startsWith(s"""akka_sensors_actor_waiting_for_recovery_permit_time_millis_count{actor="$actorName""""), 215 | _.endsWith(s" ${actors * 2}") 216 | ) 217 | } 218 | } 219 | 220 | private def assertMetrics(filter: String => Boolean, assertion: String => Boolean) = { 221 | val ms = metrics 222 | val m = ms.split("\n").find(filter) 223 | 224 | m.map(m => assert(assertion(m), s"assertion failed for $m: $ms")) 225 | .getOrElse( 226 | fail(s"No metric found in $ms") 227 | ) 228 | } 229 | 230 | def metrics(implicit registry: PrometheusRegistry): String = { 231 | val writer = new ByteArrayOutputStream() 232 | PrometheusTextFormatWriter.builder().build().write(writer, registry.scrape()) 233 | writer.toString 234 | } 235 | 236 | private def pingActor = { 237 | val r = Await.result(probeActor.ask(Ping("1"))(Timeout.durationToTimeout(10 seconds)), 15 seconds) 238 | assert(r.toString == "Pong(1)") 239 | } 240 | 241 | private def sendEventAck(actor: ActorRef) = { 242 | val r = Await.result(actor.ask(ValidCommand("1"))(Timeout.durationToTimeout(10 seconds)), 15 seconds) 243 | assert(r.toString == "Pong(1)") 244 | } 245 | protected override def afterAll(): Unit = 246 | system.terminate() 247 | } 248 | 249 | object InstrumentedActors { 250 | 251 | case class Ping(id: String) extends NoSerializationVerificationNeeded 252 | case object KnownError extends NoSerializationVerificationNeeded 253 | case object UnknownMessage extends NoSerializationVerificationNeeded 254 | case object BlockTooLong extends NoSerializationVerificationNeeded 255 | case class Pong(id: String) extends NoSerializationVerificationNeeded 256 | case class ValidEvent(id: String) extends NoSerializationVerificationNeeded 257 | 258 | sealed trait Commands 259 | case class ValidCommand(id: String) extends Commands with NoSerializationVerificationNeeded 260 | case object ProbeTimeout extends Commands with NoSerializationVerificationNeeded 261 | 262 | class MassProbe extends Actor with ActorMetrics { 263 | context.setReceiveTimeout(2 seconds) 264 | def receive: Receive = { 265 | case Ping(x) => 266 | sender() ! Pong(x) 267 | case ReceiveTimeout => 268 | context.stop(self) 269 | } 270 | } 271 | 272 | class MassPersistentProbe extends PersistentActor with PersistentActorMetrics { 273 | context.setReceiveTimeout(2 seconds) 274 | var counter = 0 275 | 276 | def receiveRecover: Receive = { 277 | case ValidEvent(x) => 278 | counter = x.toInt 279 | case RecoveryCompleted => 280 | } 281 | 282 | def receiveCommand: Receive = { 283 | case ValidCommand(x) => 284 | persist(ValidEvent(x)) { _ => 285 | counter += 1 286 | } 287 | 288 | case ReceiveTimeout => 289 | context.stop(self) 290 | 291 | } 292 | 293 | def persistenceId: String = context.self.actorRef.path.name 294 | 295 | } 296 | 297 | object MassTypedPersistentProbe { 298 | 299 | def apply(): Behavior[Commands] = 300 | BehaviorMetrics[Commands]("MassTypedPersistentProbe") 301 | .withReceiveTimeoutMetrics(ProbeTimeout) 302 | .withPersistenceMetrics 303 | .setup { context => 304 | val commandHandler: (Int, Commands) => Effect[ValidEvent, Int] = (_, cmd) => 305 | cmd match { 306 | case ValidCommand(c) => Effect.persist(ValidEvent(c)) 307 | case ProbeTimeout => Effect.stop() 308 | } 309 | 310 | val eventHandler = 311 | (state: Int, _: ValidEvent) => state + 1 312 | 313 | context.setReceiveTimeout(2.second, ProbeTimeout) 314 | 315 | EventSourcedBehavior( 316 | persistenceId = PersistenceId.ofUniqueId(context.self.path.name), 317 | emptyState = 0, 318 | commandHandler = commandHandler, 319 | eventHandler = eventHandler 320 | ) 321 | } 322 | } 323 | 324 | class InstrumentedProbe extends Actor with ActorMetrics { 325 | def receive: Receive = { 326 | case Ping(x) => 327 | Thread.sleep(Random.nextInt(3)) 328 | sender() ! Pong(x) 329 | case KnownError => 330 | throw new Exception("known") 331 | case BlockTooLong => 332 | Thread.sleep(6000) 333 | } 334 | } 335 | 336 | class PersistentInstrumentedProbe extends PersistentActor with PersistentActorMetrics { 337 | var counter = 0 338 | 339 | def receiveRecover: Receive = { 340 | case _ => 341 | } 342 | 343 | def receiveCommand: Receive = { 344 | case ValidCommand(x) => 345 | val replyTo = sender() 346 | persist(ValidEvent(counter.toString)) { _ => 347 | replyTo ! Pong(x) 348 | counter += 1 349 | } 350 | case x => println(x) 351 | } 352 | 353 | def persistenceId: String = context.self.actorRef.path.name 354 | } 355 | 356 | class AnotherPersistentInstrumentedProbe extends PersistentInstrumentedProbe 357 | 358 | } 359 | --------------------------------------------------------------------------------