├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── build.sbt ├── project ├── build.properties └── plugins.sbt ├── src └── main │ ├── resources │ └── reference.conf │ └── scala │ └── kamon │ └── instrumentation │ └── system │ ├── host │ ├── HostMetrics.scala │ └── HostMetricsCollector.scala │ ├── jvm │ ├── JvmMetrics.scala │ └── JvmMetricsCollector.scala │ └── process │ ├── ProcessMetrics.scala │ └── ProcessMetricsCollector.scala └── version.sbt /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | .history 4 | *.sc 5 | .pygments-cache 6 | .DS_Store 7 | 8 | # sbt specific 9 | dist/* 10 | target/ 11 | lib_managed/ 12 | src_managed/ 13 | project/boot/ 14 | project/plugins/project/ 15 | .ensime 16 | .ensime_cache 17 | 18 | # Scala-IDE specific 19 | .scala_dependencies 20 | .idea 21 | .idea_modules 22 | 23 | # Intellij 24 | .idea/ 25 | *.iml 26 | *.iws 27 | 28 | # Eclipse 29 | .project 30 | .settings 31 | .classpath 32 | .cache 33 | .cache-main 34 | .cache-tests 35 | bin/ 36 | 37 | _site 38 | 39 | # Ignore Play! working directory # 40 | db 41 | eclipse 42 | lib 43 | log 44 | logs 45 | modules 46 | precompiled 47 | project/project 48 | project/target 49 | target 50 | tmp 51 | test-result 52 | server.pid 53 | *.iml 54 | *.eml 55 | 56 | # Default sigar library provision location. 57 | native/ 58 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | script: 3 | - sbt test 4 | scala: 5 | - 2.12.3 6 | jdk: 7 | - openjdk8 8 | before_script: 9 | - mkdir $TRAVIS_BUILD_DIR/tmp 10 | - export SBT_OPTS="-Djava.io.tmpdir=$TRAVIS_BUILD_DIR/tmp" 11 | sudo: false 12 | 13 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing to Kamon 2 | ===================== 3 | 4 | Thanks for your intention on collaborating to the Kamon Project! It doesn't matter if you want to provide a small change 5 | to our docs, are lost in configuration or want contribute a brand new feature, we value all of your contributions and 6 | the time you take to use our tool and prepare a contribution, we only ask you to follow this guidance depending on your 7 | situation: 8 | 9 | If you are experiencing a bug 10 | ----------------------------- 11 | 12 | If you see weird exceptions in your log or something definitely is working improperly please [open an issue] and include 13 | the Kamon, Akka and Spray/Play! versions that you are using along with as many useful information you can find related 14 | to the issue. If you can provide a gist or a short way to reproduce the issue we will be more than happy! 15 | 16 | If you don't know what is wrong 17 | ------------------------------- 18 | 19 | If you don't see any metrics at all or features are not working maybe you have a setup or configuration problem, to 20 | address this kind of problems please send us a emails to our [mailing list] and we will reply as soon as we can! Again, 21 | please include the relevant version and current setup information to speed up the process. If you are in doubt of 22 | whether you have a bug or a configuration problem, email us and we will take care of openning a issue if necessary. 23 | 24 | If you want to make a code contribution to the project 25 | ------------------------------------------------------ 26 | 27 | Awesome! First, please note that we try to follow the [commit message conventions] used by the Spray guys and we need 28 | you to electronically fill our [CLA] before accepting your contribution. Also, if your PR contains various commits, 29 | please squash them into a single commit. Let the PR rain begin! 30 | 31 | 32 | [open an issue]: https://github.com/kamon-io/Kamon/issues/new 33 | [mailing list]: https://groups.google.com/forum/#!forum/kamon-user 34 | [commit message conventions]: http://spray.io/project-info/contributing/ 35 | [CLA]: https://docs.google.com/forms/d/1G_IDrBTFzOMwHvhxfKRBwNtpRelSa_MZ6jecH8lpTlc/viewform 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This software is licensed under the Apache 2 license, quoted below. 2 | 3 | Copyright © 2013-2017 the kamon project 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); you may not 6 | use this file except in compliance with the License. You may obtain a copy of 7 | the License at 8 | 9 | [http://www.apache.org/licenses/LICENSE-2.0] 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | License for the specific language governing permissions and limitations under 15 | the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # This reporsitory has been moved. 2 | 3 | Since March 2020 all the Kamon instrumentation and reporting modules were moved to Kamon's main repository at [https://github.com/kamon-io/kamon](https://github.com/kamon-io/kamon). Please check out the main repository for the latest sources, reporting issues or start contributing. You can also stop by our [Gitter Channel](https://gitter.im/kamon-io/Kamon). -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | /* ========================================================================================= 2 | * Copyright © 2013-2019 the kamon project 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file 5 | * except in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the 10 | * License is distributed on an "AS I 11 | * S" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, 12 | * either express or implied. See the License for the specific language governing permissions 13 | * and limitations under the License. 14 | * ========================================================================================= 15 | */ 16 | 17 | val kamonCore = "io.kamon" %% "kamon-core" % "2.0.1" 18 | val logback = "ch.qos.logback" % "logback-classic" % "1.2.3" 19 | val oshi = "com.github.oshi" % "oshi-core" % "4.2.1" 20 | 21 | name := "kamon-system-metrics" 22 | 23 | libraryDependencies ++= 24 | compileScope(kamonCore, oshi) ++ 25 | testScope(scalatest, logback) 26 | 27 | resolvers += Resolver.bintrayRepo("kamon-io", "releases") 28 | resolvers += Resolver.bintrayRepo("kamon-io", "snapshots") 29 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.2.8 -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | lazy val root = project in file(".") dependsOn(RootProject(uri("git://github.com/kamon-io/kamon-sbt-umbrella.git#kamon-2.x"))) -------------------------------------------------------------------------------- /src/main/resources/reference.conf: -------------------------------------------------------------------------------- 1 | kamon.instrumentation.system { 2 | jvm { 3 | 4 | } 5 | 6 | process { 7 | hiccup-monitor-interval = 1 millisecond 8 | } 9 | 10 | host { 11 | storage { 12 | 13 | # Decides which file system types will be selected for tracking usage and activity metrics. By default we are 14 | # excluding types associated with Docker and Ubuntu Snaps which would usually generate irrelevant metrics. 15 | tracked-mount-types { 16 | includes = [ "**" ] 17 | excludes = ["squashfs", "tmpfs", "aufs"] 18 | } 19 | } 20 | 21 | network { 22 | 23 | # Decides which network interfaces will be selected for tracking network activity metrics. By default we are 24 | # excluding network interface names known to be associated with Docker. 25 | tracked-interfaces { 26 | includes = [ "**" ] 27 | excludes = [ "docker0", "br-*" ] 28 | } 29 | } 30 | } 31 | } 32 | 33 | kamon.modules { 34 | host-metrics { 35 | enabled = yes 36 | name = "Host Metrics" 37 | description = "Periodically collects metrics on CPU, Memory, Swap, Network and Storage usage" 38 | factory = "kamon.instrumentation.system.host.HostMetricsCollector$Factory" 39 | } 40 | 41 | process-metrics { 42 | enabled = yes 43 | name = "Process Metrics" 44 | description = "Collects Process CPU and Ulimit metrics from the JVM process" 45 | factory = "kamon.instrumentation.system.process.ProcessMetricsCollector$Factory" 46 | } 47 | 48 | jvm-metrics { 49 | enabled = yes 50 | name = "JVM Metrics" 51 | description = "Collects CPU, Garbage Collection, Memory, Class Loading and Threads metrics from the local JVM" 52 | factory = "kamon.instrumentation.system.jvm.JvmMetricsCollector$Factory" 53 | } 54 | } -------------------------------------------------------------------------------- /src/main/scala/kamon/instrumentation/system/host/HostMetrics.scala: -------------------------------------------------------------------------------- 1 | package kamon.instrumentation.system.host 2 | 3 | import kamon.Kamon 4 | import kamon.instrumentation.system.host.HostMetrics.StorageDeviceInstruments.DeviceInstruments 5 | import kamon.instrumentation.system.host.HostMetrics.StorageMountInstruments.MountInstruments 6 | import kamon.instrumentation.system.host.HostMetrics.NetworkActivityInstruments.InterfaceInstruments 7 | import kamon.metric.{Counter, Gauge, InstrumentGroup, MeasurementUnit} 8 | import kamon.tag.TagSet 9 | 10 | import scala.collection.mutable 11 | 12 | object HostMetrics { 13 | 14 | val CpuUsage = Kamon.histogram ( 15 | name = "host.cpu.usage", 16 | description = "Samples the CPU usage percentage", 17 | unit = MeasurementUnit.percentage 18 | ) 19 | 20 | val MemoryUsed = Kamon.gauge ( 21 | name = "host.memory.used", 22 | description = "Tracks the amount of used memory", 23 | unit = MeasurementUnit.information.bytes 24 | ) 25 | 26 | val MemoryFree = Kamon.gauge ( 27 | name = "host.memory.free", 28 | description = "Tracks the amount of free memory", 29 | unit = MeasurementUnit.information.bytes 30 | ) 31 | 32 | val MemoryTotal = Kamon.gauge ( 33 | name = "host.memory.total", 34 | description = "Tracks the total memory available", 35 | unit = MeasurementUnit.information.bytes 36 | ) 37 | 38 | val SwapUsed = Kamon.gauge ( 39 | name = "host.swap.used", 40 | description = "Tracks the used Swap space", 41 | unit = MeasurementUnit.information.bytes 42 | ) 43 | 44 | val SwapFree = Kamon.gauge ( 45 | name = "host.swap.free", 46 | description = "Tracks the free Swap space", 47 | unit = MeasurementUnit.information.bytes 48 | ) 49 | 50 | val SwapTotal = Kamon.gauge ( 51 | name = "host.swap.total", 52 | description = "Tracks the total Swap space", 53 | unit = MeasurementUnit.information.bytes 54 | ) 55 | 56 | val LoadAverage = Kamon.gauge ( 57 | name = "host.load.average", 58 | description = "Tracks the system load average" 59 | ) 60 | 61 | val FileSystemMountSpaceUsed = Kamon.gauge ( 62 | name = "host.storage.mount.space.used", 63 | description = "Tracks the used space on a file system mount/volume", 64 | unit = MeasurementUnit.information.bytes 65 | ) 66 | 67 | val FileSystemMountSpaceFree = Kamon.gauge ( 68 | name = "host.storage.mount.space.free", 69 | description = "Tracks the free space on a file system mount/volume", 70 | unit = MeasurementUnit.information.bytes 71 | ) 72 | 73 | val FileSystemMountSpaceTotal = Kamon.gauge ( 74 | name = "host.storage.mount.space.total", 75 | description = "Tracks the total space on a file system mount/volume", 76 | unit = MeasurementUnit.information.bytes 77 | ) 78 | 79 | val StorageDeviceRead = Kamon.counter ( 80 | name = "host.storage.device.data.read", 81 | description = "Counts the amount of byes that have been read from a storage device", 82 | unit = MeasurementUnit.information.bytes 83 | ) 84 | 85 | val StorageDeviceWrite = Kamon.counter ( 86 | name = "host.storage.device.data.write", 87 | description = "Counts the amount of byes that have been written to a storage device", 88 | unit = MeasurementUnit.information.bytes 89 | ) 90 | 91 | val StorageDeviceReadOps = Kamon.counter ( 92 | name = "host.storage.device.ops.read", 93 | description = "Counts the number of read operations executed on a storage device" 94 | ) 95 | 96 | val StorageDeviceWriteOps = Kamon.counter ( 97 | name = "host.storage.device.ops.write", 98 | description = "Counts the number of write operations executed on a storage device" 99 | ) 100 | 101 | val NetworkPacketsRead = Kamon.counter ( 102 | name = "host.network.packets.read.total", 103 | description = "Counts how many packets have been read from a network interface" 104 | ) 105 | 106 | val NetworkPacketsWrite = Kamon.counter ( 107 | name = "host.network.packets.write.total", 108 | description = "Counts how many packets have been written to a network interface" 109 | ) 110 | 111 | val NetworkPacketsReadFailed = Kamon.counter ( 112 | name = "host.network.packets.read.failed", 113 | description = "Counts how many packets failed to be read from a network interface" 114 | ) 115 | 116 | val NetworkPacketsWriteFailed = Kamon.counter ( 117 | name = "host.network.packets.write.failed", 118 | description = "Counts how many packets failed to be written to a network interface" 119 | ) 120 | 121 | val NetworkDataRead = Kamon.counter ( 122 | name = "host.network.data.read", 123 | description = "Counts how many bytes have been read from a network interface", 124 | unit = MeasurementUnit.information.bytes 125 | ) 126 | 127 | val NetworkDataWrite = Kamon.counter ( 128 | name = "host.network.data.write", 129 | description = "Counts how many bytes have been written to a network interface", 130 | unit = MeasurementUnit.information.bytes 131 | ) 132 | 133 | class CpuInstruments(tags: TagSet) extends InstrumentGroup(tags) { 134 | val user = register(CpuUsage, "mode", "user") 135 | val system = register(CpuUsage, "mode", "system") 136 | val iowait = register(CpuUsage, "mode", "wait") 137 | val idle = register(CpuUsage, "mode", "idle") 138 | val stolen = register(CpuUsage, "mode", "stolen") 139 | val combined = register(CpuUsage, "mode", "combined") 140 | } 141 | 142 | class MemoryInstruments(tags: TagSet) extends InstrumentGroup(tags) { 143 | val used = register(MemoryUsed) 144 | val free = register(MemoryFree) 145 | val total = register(MemoryTotal) 146 | } 147 | 148 | class SwapInstruments(tags: TagSet) extends InstrumentGroup(tags) { 149 | val used = register(SwapUsed) 150 | val free = register(SwapFree) 151 | val total = register(SwapTotal) 152 | } 153 | 154 | class LoadAverageInstruments(tags: TagSet) extends InstrumentGroup(tags) { 155 | val oneMinute = register(LoadAverage, "period", "1m") 156 | val fiveMinutes = register(LoadAverage, "period", "5m") 157 | val fifteenMinutes = register(LoadAverage, "period", "15m") 158 | } 159 | 160 | class StorageMountInstruments(tags: TagSet) extends InstrumentGroup(tags) { 161 | // It is ok to use mutable, not-synchronized collections here because they will only be accessed from one Thread 162 | // at a time and that Thread is always the same Thread. 163 | private val _mountsCache = mutable.Map.empty[String, MountInstruments] 164 | 165 | def mountInstruments(mountName: String): MountInstruments = 166 | _mountsCache.getOrElseUpdate(mountName, { 167 | val mount = TagSet.of("mount", mountName) 168 | 169 | MountInstruments ( 170 | register(FileSystemMountSpaceUsed, mount), 171 | register(FileSystemMountSpaceFree, mount), 172 | register(FileSystemMountSpaceTotal, mount) 173 | ) 174 | }) 175 | } 176 | 177 | object StorageMountInstruments { 178 | case class MountInstruments ( 179 | used: Gauge, 180 | free: Gauge, 181 | total: Gauge 182 | ) 183 | } 184 | 185 | class StorageDeviceInstruments(tags: TagSet) extends InstrumentGroup(tags) { 186 | // It is ok to use mutable, not-synchronized collections here because they will only be accessed from one Thread 187 | // at a time and that Thread is always the same Thread. 188 | private val _deviceInstrumentsCache = mutable.Map.empty[String, DeviceInstruments] 189 | 190 | def deviceInstruments(deviceName: String): DeviceInstruments = 191 | _deviceInstrumentsCache.getOrElseUpdate(deviceName, { 192 | val device = TagSet.of("device", deviceName) 193 | 194 | DeviceInstruments ( 195 | DiffCounter(register(StorageDeviceReadOps, device)), 196 | DiffCounter(register(StorageDeviceRead, device)), 197 | DiffCounter(register(StorageDeviceWriteOps, device)), 198 | DiffCounter(register(StorageDeviceWrite, device)) 199 | ) 200 | }) 201 | } 202 | 203 | object StorageDeviceInstruments { 204 | case class DeviceInstruments ( 205 | reads: DiffCounter, 206 | readBytes: DiffCounter, 207 | writes: DiffCounter, 208 | writeBytes: DiffCounter 209 | ) 210 | } 211 | 212 | class NetworkActivityInstruments(tags: TagSet) extends InstrumentGroup(tags) { 213 | // It is ok to use mutable, not-synchronized collections here because they will only be accessed from one Thread 214 | // at a time and that Thread is always the same Thread. 215 | private val _interfaceCache = mutable.Map.empty[String, InterfaceInstruments] 216 | 217 | def interfaceInstruments(interfaceName: String): InterfaceInstruments = 218 | _interfaceCache.getOrElseUpdate(interfaceName, { 219 | val interface = TagSet.of("interface", interfaceName) 220 | 221 | InterfaceInstruments( 222 | DiffCounter(register(NetworkDataRead, interface)), 223 | DiffCounter(register(NetworkPacketsRead, interface)), 224 | DiffCounter(register(NetworkPacketsReadFailed, interface)), 225 | DiffCounter(register(NetworkDataWrite, interface)), 226 | DiffCounter(register(NetworkPacketsWrite, interface)), 227 | DiffCounter(register(NetworkPacketsWriteFailed, interface)) 228 | ) 229 | }) 230 | } 231 | 232 | object NetworkActivityInstruments { 233 | case class InterfaceInstruments ( 234 | receivedBytes: DiffCounter, 235 | receivedPackets: DiffCounter, 236 | receiveErrorPackets: DiffCounter, 237 | sentBytes: DiffCounter, 238 | sentPackets: DiffCounter, 239 | sendErrorPackets: DiffCounter 240 | ) 241 | } 242 | 243 | /** 244 | * A modified Counter that keeps track of a monotonically increasing value and only records the difference between 245 | * the current and previous value on the target counter. 246 | */ 247 | case class DiffCounter(counter: Counter) { 248 | private var _previous = 0L 249 | 250 | def diff(current: Long): Unit = { 251 | if(_previous > 0L) { 252 | val delta = current - _previous 253 | if(delta > 0) 254 | counter.increment(delta) 255 | 256 | } 257 | 258 | _previous = current 259 | } 260 | } 261 | } 262 | -------------------------------------------------------------------------------- /src/main/scala/kamon/instrumentation/system/host/HostMetricsCollector.scala: -------------------------------------------------------------------------------- 1 | package kamon 2 | package instrumentation 3 | package system 4 | package host 5 | 6 | import java.util.concurrent.TimeUnit 7 | 8 | import com.typesafe.config.Config 9 | import kamon.instrumentation.system.host.HostMetrics._ 10 | import kamon.module.{Module, ModuleFactory} 11 | import kamon.tag.TagSet 12 | import kamon.util.Filter 13 | import oshi.SystemInfo 14 | import oshi.hardware.CentralProcessor.TickType 15 | 16 | import scala.concurrent.{ExecutionContext, Future} 17 | 18 | /** 19 | * Collects CPU, Memory, Swap, Storage and Network metrics. The metrics collection is split into two groups: frequent 20 | * and infrequent collector; the frequent collector records metrics that are highly volatile and do not accumulate over 21 | * time, like the CPU usage, while the infrequent collector focuses on metrics that can be updated less frequently like 22 | * swap and memory usage and cumulative metrics like network and storage usage. 23 | */ 24 | class HostMetricsCollector(ec: ExecutionContext) extends Module { 25 | private val _configPath = "kamon.instrumentation.system.host" 26 | @volatile private var _settings: HostMetricsCollector.Settings = readSettings(Kamon.config()) 27 | 28 | private val _frequentCollector = new FrequentCollectionTask 29 | private val _infrequentCollector = new InfrequentCollectionTask 30 | private val _fcSchedule = Kamon.scheduler().scheduleAtFixedRate(scheduleOnModuleEC(_frequentCollector), 1, 1, TimeUnit.SECONDS) 31 | private val _ifcSchedule = Kamon.scheduler().scheduleAtFixedRate(scheduleOnModuleEC(_infrequentCollector), 1, 10, TimeUnit.SECONDS) 32 | 33 | override def stop(): Unit = { 34 | _fcSchedule.cancel(false) 35 | _ifcSchedule.cancel(false) 36 | _frequentCollector.cleanup() 37 | _infrequentCollector.cleanup() 38 | } 39 | 40 | override def reconfigure(newConfig: Config): Unit = 41 | _settings = readSettings(newConfig) 42 | 43 | private def readSettings(config: Config): HostMetricsCollector.Settings = { 44 | val hostConfig = config.getConfig(_configPath) 45 | 46 | HostMetricsCollector.Settings( 47 | Filter.from(hostConfig.getConfig("network.tracked-interfaces")), 48 | Filter.from(hostConfig.getConfig("storage.tracked-mount-types")) 49 | ) 50 | } 51 | 52 | private def scheduleOnModuleEC(task: CollectionTask): Runnable = new Runnable { 53 | override def run(): Unit = 54 | task.schedule(ec) 55 | } 56 | 57 | trait CollectionTask { 58 | def schedule(ec: ExecutionContext): Unit 59 | def cleanup(): Unit 60 | } 61 | 62 | private class FrequentCollectionTask extends CollectionTask { 63 | private val _defaultTags = TagSet.of("component", "host") 64 | private val _systemInfo = new SystemInfo() 65 | private val _hal = _systemInfo.getHardware() 66 | private val _cpuInstruments = new CpuInstruments(_defaultTags) 67 | private var _prevCpuLoadTicks: Array[Long] = Array.ofDim(0) 68 | 69 | def schedule(ec: ExecutionContext): Unit = { 70 | Future { 71 | recordCpuUsage() 72 | }(ec) 73 | } 74 | 75 | def cleanup(): Unit = { 76 | _cpuInstruments.remove() 77 | } 78 | 79 | private def recordCpuUsage(): Unit = { 80 | if(_prevCpuLoadTicks.length > 0) { 81 | val previousTicks = _prevCpuLoadTicks 82 | val currentTicks = _hal.getProcessor().getSystemCpuLoadTicks() 83 | 84 | val user = ticksDiff(previousTicks, currentTicks, TickType.USER) 85 | val nice = ticksDiff(previousTicks, currentTicks, TickType.NICE) 86 | val system = ticksDiff(previousTicks, currentTicks, TickType.SYSTEM) 87 | val idle = ticksDiff(previousTicks, currentTicks, TickType.IDLE) 88 | val iowait = ticksDiff(previousTicks, currentTicks, TickType.IOWAIT) 89 | val irq = ticksDiff(previousTicks, currentTicks, TickType.IRQ) 90 | val softirq = ticksDiff(previousTicks, currentTicks, TickType.SOFTIRQ) 91 | val steal = ticksDiff(previousTicks, currentTicks, TickType.STEAL) 92 | val total = user + nice + system + idle + iowait +irq + softirq + steal 93 | def toPercent(value: Long): Long = ((100D * value.toDouble) / total.toDouble).toLong 94 | 95 | _cpuInstruments.user.record(toPercent(user)) 96 | _cpuInstruments.system.record(toPercent(system)) 97 | _cpuInstruments.iowait.record(toPercent(iowait)) 98 | _cpuInstruments.idle.record(toPercent(idle)) 99 | _cpuInstruments.stolen.record(toPercent(steal)) 100 | _cpuInstruments.combined.record(toPercent(user + system + nice + iowait)) 101 | _prevCpuLoadTicks = currentTicks 102 | 103 | } else { 104 | _prevCpuLoadTicks = _hal.getProcessor().getSystemCpuLoadTicks() 105 | } 106 | } 107 | 108 | private def ticksDiff(previous: Array[Long], current: Array[Long], tickType: TickType): Long = 109 | math.max(current(tickType.getIndex) - previous(tickType.getIndex), 0L) 110 | 111 | } 112 | 113 | private class InfrequentCollectionTask extends CollectionTask { 114 | private val _defaultTags = TagSet.of("component", "host") 115 | private val _systemInfo = new SystemInfo() 116 | private val _hal = _systemInfo.getHardware() 117 | private val _os = _systemInfo.getOperatingSystem 118 | 119 | private val _memoryInstruments = new MemoryInstruments(_defaultTags) 120 | private val _swapInstruments = new SwapInstruments(_defaultTags) 121 | private val _loadAverageInstruments = new LoadAverageInstruments(_defaultTags) 122 | private val _fileSystemUsageInstruments = new StorageMountInstruments(_defaultTags) 123 | private val _fileSystemActivityInstruments = new StorageDeviceInstruments(_defaultTags) 124 | private val _networkActivityInstruments = new NetworkActivityInstruments(_defaultTags) 125 | 126 | def schedule(ec: ExecutionContext): Unit = { 127 | Future { 128 | recordMemoryUsage() 129 | recordLoadAverage() 130 | recordStorageUsage() 131 | recordStorageActivity() 132 | recordNetworkActivity() 133 | }(ec) 134 | } 135 | 136 | def cleanup(): Unit = { 137 | _memoryInstruments.remove() 138 | _swapInstruments.remove() 139 | _loadAverageInstruments.remove() 140 | _fileSystemUsageInstruments.remove() 141 | _fileSystemActivityInstruments.remove() 142 | _networkActivityInstruments.remove() 143 | } 144 | 145 | private def recordMemoryUsage(): Unit = { 146 | val memory = _hal.getMemory 147 | _memoryInstruments.total.update(memory.getTotal()) 148 | _memoryInstruments.free.update(memory.getAvailable()) 149 | _memoryInstruments.used.update(memory.getTotal() - memory.getAvailable()) 150 | 151 | _swapInstruments.total.update(memory.getVirtualMemory.getSwapTotal()) 152 | _swapInstruments.used.update(memory.getVirtualMemory.getSwapUsed()) 153 | _swapInstruments.free.update(memory.getVirtualMemory.getSwapTotal() - memory.getVirtualMemory.getSwapUsed()) 154 | } 155 | 156 | private def recordLoadAverage(): Unit = { 157 | val loadAverage = _hal.getProcessor.getSystemLoadAverage(3) 158 | if(loadAverage(0) >= 0D) _loadAverageInstruments.oneMinute.update(loadAverage(0)) 159 | if(loadAverage(1) >= 0D) _loadAverageInstruments.fiveMinutes.update(loadAverage(1)) 160 | if(loadAverage(2) >= 0D) _loadAverageInstruments.fifteenMinutes.update(loadAverage(2)) 161 | } 162 | 163 | private def recordStorageUsage(): Unit = { 164 | val fileStores = _os.getFileSystem().getFileStores 165 | 166 | fileStores.foreach(fs => { 167 | if(_settings.trackedMounts.accept(fs.getType)) { 168 | val mountInstruments = _fileSystemUsageInstruments.mountInstruments(fs.getMount) 169 | mountInstruments.free.update(fs.getUsableSpace) 170 | mountInstruments.total.update(fs.getTotalSpace) 171 | mountInstruments.used.update(fs.getTotalSpace - fs.getUsableSpace) 172 | } 173 | }) 174 | } 175 | 176 | private def recordStorageActivity(): Unit = { 177 | val devices = _hal.getDiskStores 178 | 179 | devices.foreach(device => { 180 | if(device.getPartitions.nonEmpty) { 181 | val deviceInstruments = _fileSystemActivityInstruments.deviceInstruments(device.getName) 182 | deviceInstruments.reads.diff(device.getReads) 183 | deviceInstruments.writes.diff(device.getWrites) 184 | deviceInstruments.readBytes.diff(device.getReadBytes) 185 | deviceInstruments.writeBytes.diff(device.getWriteBytes) 186 | } 187 | }) 188 | } 189 | 190 | private def recordNetworkActivity(): Unit = { 191 | val interfaces = _hal.getNetworkIFs() 192 | 193 | interfaces.foreach(interface => { 194 | if(_settings.trackedInterfaces.accept(interface.getName)) { 195 | val interfaceInstruments = _networkActivityInstruments.interfaceInstruments(interface.getName) 196 | interfaceInstruments.receivedBytes.diff(interface.getBytesRecv) 197 | interfaceInstruments.sentBytes.diff(interface.getBytesSent) 198 | interfaceInstruments.sentPackets.diff(interface.getPacketsSent) 199 | interfaceInstruments.receivedPackets.diff(interface.getPacketsRecv) 200 | interfaceInstruments.sendErrorPackets.diff(interface.getOutErrors) 201 | interfaceInstruments.receiveErrorPackets.diff(interface.getInErrors) 202 | } 203 | }) 204 | } 205 | } 206 | } 207 | 208 | object HostMetricsCollector { 209 | 210 | class Factory extends ModuleFactory { 211 | override def create(settings: ModuleFactory.Settings): Module = 212 | new HostMetricsCollector(settings.executionContext) 213 | } 214 | 215 | case class Settings ( 216 | trackedInterfaces: Filter, 217 | trackedMounts: Filter 218 | ) 219 | } 220 | -------------------------------------------------------------------------------- /src/main/scala/kamon/instrumentation/system/jvm/JvmMetrics.scala: -------------------------------------------------------------------------------- 1 | package kamon.instrumentation.system.jvm 2 | 3 | import kamon.Kamon 4 | import kamon.instrumentation.system.jvm.JvmMetrics.MemoryUsageInstruments.MemoryRegionInstruments 5 | import kamon.instrumentation.system.jvm.JvmMetricsCollector.{Collector, MemoryPool} 6 | import kamon.metric.{Gauge, Histogram, InstrumentGroup, MeasurementUnit} 7 | import kamon.tag.TagSet 8 | 9 | import scala.collection.mutable 10 | 11 | object JvmMetrics { 12 | 13 | val GC = Kamon.histogram ( 14 | name = "jvm.gc", 15 | description = "Tracks the distribution of GC events duration", 16 | unit = MeasurementUnit.time.milliseconds 17 | ) 18 | 19 | val GcPromotion = Kamon.histogram ( 20 | name = "jvm.gc.promotion", 21 | description = "Tracks the distribution of promoted bytes to the old generation regions after a GC", 22 | unit = MeasurementUnit.information.bytes 23 | ) 24 | 25 | val MemoryUsed = Kamon.histogram ( 26 | name = "jvm.memory.used", 27 | description = "Samples the used space in a memory region", 28 | unit = MeasurementUnit.information.bytes 29 | ) 30 | 31 | val MemoryFree = Kamon.histogram ( 32 | name = "jvm.memory.free", 33 | description = "Samples the free space in a memory region", 34 | unit = MeasurementUnit.information.bytes 35 | ) 36 | 37 | val MemoryCommitted = Kamon.gauge ( 38 | name = "jvm.memory.committed", 39 | description = "Tracks the committed space in a memory region", 40 | unit = MeasurementUnit.information.bytes 41 | ) 42 | 43 | val MemoryMax = Kamon.gauge ( 44 | name = "jvm.memory.max", 45 | description = "Tracks the max space in a memory region", 46 | unit = MeasurementUnit.information.bytes 47 | ) 48 | 49 | val MemoryPoolUsed = Kamon.histogram ( 50 | name = "jvm.memory.pool.used", 51 | description = "Samples the used space in a memory pool", 52 | unit = MeasurementUnit.information.bytes 53 | ) 54 | 55 | val MemoryPoolFree = Kamon.histogram ( 56 | name = "jvm.memory.pool.free", 57 | description = "Samples the free space in a memory pool", 58 | unit = MeasurementUnit.information.bytes 59 | ) 60 | 61 | val MemoryPoolCommitted = Kamon.gauge ( 62 | name = "jvm.memory.pool.committed", 63 | description = "Tracks the committed space in a memory pool", 64 | unit = MeasurementUnit.information.bytes 65 | ) 66 | 67 | val MemoryPoolMax = Kamon.gauge ( 68 | name = "jvm.memory.pool.max", 69 | description = "Tracks the max space in a memory pool", 70 | unit = MeasurementUnit.information.bytes 71 | ) 72 | 73 | val MemoryAllocation = Kamon.counter ( 74 | name = "jvm.memory.allocation", 75 | description = "Tracks the number amount of bytes allocated", 76 | unit = MeasurementUnit.information.bytes 77 | ) 78 | 79 | val ThreadsTotal = Kamon.gauge ( 80 | name = "jvm.threads.total", 81 | description = "Tracks the current number of live threads on the JVM" 82 | ) 83 | 84 | val ThreadsPeak = Kamon.gauge ( 85 | name = "jvm.threads.peak", 86 | description = "Tracks the peak live thread count since the JVM started" 87 | ) 88 | 89 | val ThreadsDaemon = Kamon.gauge ( 90 | name = "jvm.threads.daemon", 91 | description = "Tracks the current number of daemon threads on the JVM" 92 | ) 93 | 94 | class GarbageCollectionInstruments(tags: TagSet) extends InstrumentGroup(tags) { 95 | private val _collectorCache = mutable.Map.empty[String, Histogram] 96 | 97 | val allocation = register(MemoryAllocation) 98 | val promotionToOld = register(GcPromotion, "space", "old") 99 | 100 | def garbageCollectionTime(collector: Collector): Histogram = 101 | _collectorCache.getOrElseUpdate(collector.alias, { 102 | val collectorTags = TagSet.builder() 103 | .add("collector", collector.alias) 104 | .add("generation", collector.generation.toString) 105 | .build() 106 | 107 | register(GC, collectorTags) 108 | }) 109 | } 110 | 111 | class MemoryUsageInstruments(tags: TagSet) extends InstrumentGroup(tags) { 112 | private val _memoryRegionsCache = mutable.Map.empty[String, MemoryRegionInstruments] 113 | private val _memoryPoolsCache = mutable.Map.empty[String, MemoryRegionInstruments] 114 | 115 | def regionInstruments(regionName: String): MemoryRegionInstruments = 116 | _memoryRegionsCache.getOrElseUpdate(regionName, { 117 | val region = TagSet.of("region", regionName) 118 | 119 | MemoryRegionInstruments ( 120 | register(MemoryUsed, region), 121 | register(MemoryFree, region), 122 | register(MemoryCommitted, region), 123 | register(MemoryMax, region) 124 | ) 125 | }) 126 | 127 | def poolInstruments(pool: MemoryPool): MemoryRegionInstruments = 128 | _memoryPoolsCache.getOrElseUpdate(pool.alias, { 129 | val region = TagSet.of("pool", pool.alias) 130 | 131 | MemoryRegionInstruments ( 132 | register(MemoryPoolUsed, region), 133 | register(MemoryPoolFree, region), 134 | register(MemoryPoolCommitted, region), 135 | register(MemoryPoolMax, region) 136 | ) 137 | }) 138 | } 139 | 140 | class ThreadsInstruments extends InstrumentGroup(TagSet.Empty) { 141 | val total = register(ThreadsTotal) 142 | val peak = register(ThreadsPeak) 143 | val daemon = register(ThreadsDaemon) 144 | } 145 | 146 | object MemoryUsageInstruments { 147 | 148 | case class MemoryRegionInstruments ( 149 | used: Histogram, 150 | free: Histogram, 151 | committed: Gauge, 152 | max: Gauge 153 | ) 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /src/main/scala/kamon/instrumentation/system/jvm/JvmMetricsCollector.scala: -------------------------------------------------------------------------------- 1 | package kamon.instrumentation.system.jvm 2 | 3 | import java.lang.management.{ManagementFactory, MemoryUsage} 4 | import java.util.concurrent.TimeUnit 5 | 6 | import com.sun.management.GarbageCollectionNotificationInfo 7 | import com.sun.management.GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION 8 | import com.typesafe.config.Config 9 | import javax.management.openmbean.CompositeData 10 | import javax.management.{Notification, NotificationEmitter, NotificationListener} 11 | import kamon.Kamon 12 | import kamon.instrumentation.system.jvm.JvmMetrics.{GarbageCollectionInstruments, MemoryUsageInstruments, ThreadsInstruments} 13 | import kamon.instrumentation.system.jvm.JvmMetricsCollector.{Collector, MemoryPool} 14 | import kamon.module.{Module, ModuleFactory} 15 | import kamon.tag.TagSet 16 | 17 | import scala.collection.JavaConverters.{collectionAsScalaIterableConverter, mapAsScalaMapConverter} 18 | import scala.collection.concurrent.TrieMap 19 | import scala.concurrent.ExecutionContext 20 | import scala.util.matching.Regex 21 | 22 | class JvmMetricsCollector(ec: ExecutionContext) extends Module { 23 | private val _defaultTags = TagSet.of("component", "jvm") 24 | private val _gcListener = registerGcListener(_defaultTags) 25 | private val _memoryUsageInstruments = new MemoryUsageInstruments(_defaultTags) 26 | private val _threadsUsageInstruments = new ThreadsInstruments() 27 | private val _jmxCollectorTask = new JmxMetricsCollectorTask(_memoryUsageInstruments, _threadsUsageInstruments) 28 | private val _jmxCollectorSchedule = Kamon.scheduler().scheduleAtFixedRate(_jmxCollectorTask, 1, 10, TimeUnit.SECONDS) 29 | 30 | override def stop(): Unit = { 31 | _jmxCollectorSchedule.cancel(false) 32 | deregisterGcListener() 33 | } 34 | 35 | override def reconfigure(newConfig: Config): Unit = {} 36 | 37 | private def registerGcListener(defaultTags: TagSet): NotificationListener = { 38 | val gcInstruments = new GarbageCollectionInstruments(defaultTags) 39 | val gcListener = new GcNotificationListener(gcInstruments) 40 | 41 | ManagementFactory.getGarbageCollectorMXBeans().asScala.foreach(gcBean => { 42 | if(gcBean.isInstanceOf[NotificationEmitter]) 43 | gcBean.asInstanceOf[NotificationEmitter].addNotificationListener(gcListener, null, null) 44 | }) 45 | 46 | gcListener 47 | } 48 | 49 | private def deregisterGcListener(): Unit = { 50 | ManagementFactory.getGarbageCollectorMXBeans().asScala.foreach(gcBean => { 51 | if(gcBean.isInstanceOf[NotificationEmitter]) { 52 | gcBean.asInstanceOf[NotificationEmitter].removeNotificationListener(_gcListener) 53 | } 54 | }) 55 | } 56 | 57 | class GcNotificationListener(val gcInstruments: GarbageCollectionInstruments) extends NotificationListener { 58 | private val _previousUsageAfterGc = TrieMap.empty[String, MemoryUsage] 59 | 60 | override def handleNotification(notification: Notification, handback: Any): Unit = { 61 | if(notification.getType() == GARBAGE_COLLECTION_NOTIFICATION) { 62 | val compositeData = notification.getUserData.asInstanceOf[CompositeData] 63 | val info = GarbageCollectionNotificationInfo.from(compositeData) 64 | val collector = Collector.find(info.getGcName) 65 | 66 | gcInstruments.garbageCollectionTime(collector).record(info.getGcInfo.getDuration) 67 | 68 | val usageBeforeGc = info.getGcInfo.getMemoryUsageBeforeGc.asScala 69 | val usageAfterGc = info.getGcInfo.getMemoryUsageAfterGc.asScala 70 | 71 | usageBeforeGc.foreach { 72 | case (regionName, regionUsageBeforeGc) => 73 | val region = MemoryPool.find(regionName) 74 | 75 | // We assume that if the old generation grew during this GC event then some data was promoted to it and will 76 | // record it as promotion to the old generation. 77 | // 78 | if(region.usage == MemoryPool.Usage.OldGeneration) { 79 | val regionUsageAfterGc = usageAfterGc(regionName) 80 | val diff = regionUsageAfterGc.getUsed - regionUsageBeforeGc.getUsed 81 | 82 | if(diff > 0) 83 | gcInstruments.promotionToOld.record(diff) 84 | 85 | } 86 | 87 | // We will record the growth of Eden spaces in between GC events as the allocation rate. 88 | if(region.usage == MemoryPool.Usage.Eden) { 89 | _previousUsageAfterGc.get(regionName).fold({ 90 | 91 | // We wont have the previous GC value the first time an Eden region is processed so we can assume 92 | // that all used space before GC was just allocation. 93 | gcInstruments.allocation.increment(regionUsageBeforeGc.getUsed): Unit 94 | 95 | })(previousUsageAfterGc => { 96 | val currentUsageBeforeGc = regionUsageBeforeGc 97 | val allocated = currentUsageBeforeGc.getUsed - previousUsageAfterGc.getUsed 98 | if(allocated > 0) 99 | gcInstruments.allocation.increment(allocated) 100 | }) 101 | 102 | _previousUsageAfterGc.put(regionName, usageAfterGc(regionName)) 103 | } 104 | } 105 | } 106 | } 107 | } 108 | 109 | class JmxMetricsCollectorTask(memoryUsageInstruments: MemoryUsageInstruments, threadsInstruments: ThreadsInstruments) 110 | extends Runnable { 111 | 112 | private val _heapUsage = memoryUsageInstruments.regionInstruments("heap") 113 | private val _nonHeapUsage = memoryUsageInstruments.regionInstruments("non-heap") 114 | 115 | override def run(): Unit = { 116 | val threadsMxBen = ManagementFactory.getThreadMXBean() 117 | threadsInstruments.total.update(threadsMxBen.getThreadCount()) 118 | threadsInstruments.peak.update(threadsMxBen.getPeakThreadCount()) 119 | threadsInstruments.daemon.update(threadsMxBen.getDaemonThreadCount()) 120 | 121 | val currentHeapUsage = ManagementFactory.getMemoryMXBean.getHeapMemoryUsage 122 | val freeHeap = Math.max(0L, currentHeapUsage.getMax - currentHeapUsage.getUsed) 123 | _heapUsage.free.record(freeHeap) 124 | _heapUsage.used.record(currentHeapUsage.getUsed) 125 | _heapUsage.max.update(currentHeapUsage.getMax) 126 | _heapUsage.committed.update(currentHeapUsage.getCommitted) 127 | 128 | val currentNonHeapUsage = ManagementFactory.getMemoryMXBean.getNonHeapMemoryUsage 129 | val freeNonHeap = Math.max(0L, currentNonHeapUsage.getMax - currentNonHeapUsage.getUsed) 130 | _nonHeapUsage.free.record(freeNonHeap) 131 | _nonHeapUsage.used.record(currentNonHeapUsage.getUsed) 132 | _nonHeapUsage.max.update(currentNonHeapUsage.getMax) 133 | _nonHeapUsage.committed.update(currentNonHeapUsage.getCommitted) 134 | 135 | ManagementFactory.getMemoryPoolMXBeans.asScala.foreach(memoryBean => { 136 | val poolInstruments = memoryUsageInstruments.poolInstruments(MemoryPool.find(memoryBean.getName)) 137 | val memoryUsage = memoryBean.getUsage 138 | val freeMemory = Math.max(0L, memoryUsage.getMax - memoryUsage.getUsed) 139 | 140 | poolInstruments.free.record(freeMemory) 141 | poolInstruments.used.record(memoryUsage.getUsed) 142 | poolInstruments.max.update(memoryUsage.getMax) 143 | poolInstruments.committed.update(memoryUsage.getCommitted) 144 | }) 145 | } 146 | } 147 | } 148 | 149 | object JvmMetricsCollector { 150 | 151 | class Factory extends ModuleFactory { 152 | override def create(settings: ModuleFactory.Settings): Module = 153 | new JvmMetricsCollector(settings.executionContext) 154 | } 155 | 156 | case class Collector ( 157 | name: String, 158 | alias: String, 159 | generation: Collector.Generation 160 | ) 161 | 162 | object Collector { 163 | 164 | sealed trait Generation 165 | object Generation { 166 | case object Young extends Generation { override def toString: String = "young" } 167 | case object Old extends Generation { override def toString: String = "old" } 168 | case object Unknown extends Generation { override def toString: String = "unknown" } 169 | } 170 | 171 | def find(collectorName: String): Collector = 172 | _collectorMappings.get(collectorName).getOrElse ( 173 | Collector(collectorName, sanitizeCollectorName(collectorName), Collector.Generation.Unknown) 174 | ) 175 | 176 | private val _collectorMappings: Map[String, Collector] = Map ( 177 | "Copy" -> Collector("Copy", "copy", Generation.Young), 178 | "ParNew" -> Collector("ParNew", "par-new", Generation.Young), 179 | "MarkSweepCompact" -> Collector("MarkSweepCompact", "mark-sweep-compact", Generation.Old), 180 | "ConcurrentMarkSweep" -> Collector("ConcurrentMarkSweep", "concurrent-mark-sweep", Generation.Old), 181 | "PS Scavenge" -> Collector("PS Scavenge", "ps-scavenge", Generation.Young), 182 | "PS MarkSweep" -> Collector("PS MarkSweep", "ps-mark-sweep", Generation.Old), 183 | "G1 Young Generation" -> Collector("G1 Young Generation", "g1-young", Generation.Young), 184 | "G1 Old Generation" -> Collector("G1 Old Generation", "g1-old", Generation.Old) 185 | ) 186 | 187 | private def sanitizeCollectorName(name: String): String = 188 | name.replaceAll("""[^\w]""", "-").toLowerCase 189 | } 190 | 191 | case class MemoryPool ( 192 | name: String, 193 | alias: String, 194 | usage: MemoryPool.Usage 195 | ) 196 | 197 | object MemoryPool { 198 | 199 | sealed trait Usage 200 | object Usage { 201 | case object Eden extends Usage 202 | case object YoungGeneration extends Usage 203 | case object OldGeneration extends Usage 204 | case object CodeCache extends Usage 205 | case object Metaspace extends Usage 206 | case object Unknown extends Usage 207 | } 208 | 209 | def find(poolName: String): MemoryPool = 210 | _memoryRegionMappings.get(poolName).getOrElse { 211 | MemoryPool(poolName, sanitize(poolName), if(poolName.endsWith("Eden Space")) Usage.Eden else Usage.Unknown) 212 | } 213 | 214 | private val _memoryRegionMappings: Map[String, MemoryPool] = Map ( 215 | "Metaspace" -> MemoryPool("Metaspace", "metaspace", Usage.Metaspace), 216 | "Code Cache" -> MemoryPool("Code Cache", "code-cache", Usage.CodeCache), 217 | "Compressed Class Space" -> MemoryPool("Compressed Class Space", "compressed-class-space", Usage.CodeCache), 218 | "PS Eden Space" -> MemoryPool("PS Eden Space", "eden", Usage.Eden), 219 | "PS Survivor Space" -> MemoryPool("PS Survivor Space", "survivor", Usage.YoungGeneration), 220 | "PS Old Gen" -> MemoryPool("PS Old Gen", "old", Usage.OldGeneration) 221 | ) 222 | 223 | private val _invalidChars: Regex = """[^a-z0-9]""".r 224 | 225 | def sanitize(name: String): String = 226 | _invalidChars.replaceAllIn(name.toLowerCase, "-") 227 | } 228 | } 229 | -------------------------------------------------------------------------------- /src/main/scala/kamon/instrumentation/system/process/ProcessMetrics.scala: -------------------------------------------------------------------------------- 1 | package kamon.instrumentation.system.process 2 | 3 | import kamon.Kamon 4 | import kamon.metric.{InstrumentGroup, MeasurementUnit} 5 | import kamon.tag.TagSet 6 | 7 | object ProcessMetrics { 8 | 9 | val ProcessCpu = Kamon.histogram( 10 | name = "process.cpu.usage", 11 | description = "Samples the Process CPU usage", 12 | unit = MeasurementUnit.percentage 13 | ) 14 | 15 | val ULimitMaxFileDescriptors = Kamon.gauge ( 16 | name = "process.ulimit.file-descriptors.max", 17 | description = "Tracks the max number of file descriptors that can be used by the process" 18 | ) 19 | val ULimitUsedFileDescriptors = Kamon.gauge ( 20 | name = "process.ulimit.file-descriptors.used", 21 | description = "Tracks the number of file descriptors used by the process" 22 | ) 23 | 24 | val Hiccups = Kamon.timer ( 25 | name = "process.hiccups", 26 | description = "Tracks the process hiccups generated by either garbage collection or OS noise" 27 | ) 28 | 29 | 30 | class ProcessInstruments(tags: TagSet) extends InstrumentGroup(tags) { 31 | val user = register(ProcessCpu, "mode", "user") 32 | val system = register(ProcessCpu, "mode", "system") 33 | val combined = register(ProcessCpu, "mode", "combined") 34 | val openFilesLimit = register(ULimitMaxFileDescriptors) 35 | val openFilesCurrent = register(ULimitUsedFileDescriptors) 36 | val hiccups = register(Hiccups) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/scala/kamon/instrumentation/system/process/ProcessMetricsCollector.scala: -------------------------------------------------------------------------------- 1 | package kamon.instrumentation.system.process 2 | 3 | import java.time.Duration 4 | import java.util.concurrent.{Executors, TimeUnit} 5 | 6 | import com.sun.jna.Platform 7 | import com.typesafe.config.Config 8 | import kamon.instrumentation.system.process.ProcessMetrics.ProcessInstruments 9 | import kamon.metric.Timer 10 | import kamon.module.{Module, ModuleFactory} 11 | import kamon.tag.TagSet 12 | import kamon.Kamon 13 | import oshi.SystemInfo 14 | import oshi.util.{FileUtil, ParseUtil} 15 | 16 | import scala.collection.JavaConverters.iterableAsScalaIterableConverter 17 | import scala.concurrent.{ExecutionContext, Future} 18 | import scala.util.Try 19 | 20 | class ProcessMetricsCollector(ec: ExecutionContext) extends Module { 21 | private val _hiccupIntervalPath = "kamon.instrumentation.system.process.hiccup-monitor-interval" 22 | private val _defaultTags = TagSet.of("component", "process") 23 | private val _processCpuInstruments = new ProcessInstruments(_defaultTags) 24 | private val _collectionTask = new MetricsCollectionTask 25 | private val _collectionSchedule = Kamon.scheduler().scheduleAtFixedRate(scheduleOnModuleEC(_collectionTask), 1, 1, TimeUnit.SECONDS) 26 | private val _hiccupMonitor = startHiccupMonitor() 27 | 28 | override def stop(): Unit = { 29 | _hiccupMonitor.terminate() 30 | _collectionSchedule.cancel(false) 31 | _collectionTask.cleanup() 32 | } 33 | 34 | override def reconfigure(newConfig: Config): Unit = 35 | _hiccupMonitor.updateInterval(newConfig.getDuration(_hiccupIntervalPath)) 36 | 37 | 38 | private def scheduleOnModuleEC(task: MetricsCollectionTask): Runnable = new Runnable { 39 | override def run(): Unit = 40 | task.schedule(ec) 41 | } 42 | 43 | private def startHiccupMonitor(): HiccupMonitor = { 44 | val interval = Kamon.config().getDuration(_hiccupIntervalPath) 45 | val monitorThread = new HiccupMonitor(_processCpuInstruments.hiccups, interval) 46 | monitorThread.setDaemon(true) 47 | monitorThread.setName("hiccup-monitor") 48 | monitorThread.start() 49 | monitorThread 50 | } 51 | 52 | private class MetricsCollectionTask { 53 | private val _systemInfo = new SystemInfo() 54 | private val _hal = _systemInfo.getHardware() 55 | private val _os = _systemInfo.getOperatingSystem 56 | private val _pid = _os.getProcessId() 57 | private val _processorCount = _hal.getProcessor.getLogicalProcessorCount().toDouble 58 | private var _previousProcessCpuTime: Array[Long] = Array.empty[Long] 59 | 60 | def schedule(ec: ExecutionContext): Unit = { 61 | Future { 62 | recordProcessCpu() 63 | recordProcessULimits() 64 | }(ec) 65 | } 66 | 67 | def cleanup(): Unit = { 68 | _processCpuInstruments.remove() 69 | } 70 | 71 | private def recordProcessCpu(): Unit = { 72 | val process = _os.getProcess(_pid) 73 | val previous = _previousProcessCpuTime 74 | val current = Array ( 75 | process.getKernelTime(), 76 | process.getUserTime(), 77 | process.getUpTime() 78 | ) 79 | 80 | if(previous.nonEmpty) { 81 | val kernelTime = math.max(0L, current(0) - previous(0)) 82 | val userTime = math.max(0L, current(1) - previous(1)) 83 | val totalTime = math.max(0L, current(2) - previous(2)) 84 | def toPercent(value: Long): Long = { 85 | if(totalTime > 0) ((100D * value.toDouble) / totalTime.toDouble / _processorCount).toLong else 0 86 | } 87 | 88 | _processCpuInstruments.user.record(toPercent(userTime)) 89 | _processCpuInstruments.system.record(toPercent(kernelTime)) 90 | _processCpuInstruments.combined.record(toPercent(userTime + kernelTime)) 91 | } 92 | 93 | _previousProcessCpuTime = current 94 | } 95 | 96 | private def recordProcessULimits(): Unit = { 97 | val process = _os.getProcess(_pid) 98 | _processCpuInstruments.openFilesCurrent.update(Math.max(process.getOpenFiles(), 0)) 99 | 100 | Try { 101 | if (Platform.isLinux()) { 102 | val allLimits = FileUtil.readFile(String.format(s"/proc/${_pid}/limits")) 103 | allLimits.asScala.find(_.toLowerCase().startsWith("max open files")).map { openFilesLimitLine => 104 | val openFilesLimit = ParseUtil.getNthIntValue(openFilesLimitLine, 1) 105 | _processCpuInstruments.openFilesLimit.update(openFilesLimit) 106 | } 107 | } 108 | } 109 | } 110 | } 111 | 112 | final class HiccupMonitor(hiccupTimeMetric: Timer, duration: Duration) extends Thread { 113 | @volatile private var _hiccupNanos = duration.toNanos 114 | @volatile private var _doRun = true 115 | 116 | override def run(): Unit = { 117 | var shortestObservedDelta = Long.MaxValue 118 | 119 | while (_doRun) { 120 | val hiccupTime = hic(_hiccupNanos) 121 | record(hiccupTime, _hiccupNanos) 122 | } 123 | 124 | def hic(resolution: Long): Long = { 125 | val start = System.nanoTime 126 | TimeUnit.NANOSECONDS.sleep(resolution) 127 | val delta = System.nanoTime() - start 128 | if (delta < shortestObservedDelta) shortestObservedDelta = delta 129 | delta - shortestObservedDelta 130 | } 131 | } 132 | 133 | /** 134 | * We'll need fill in missing measurements as delayed 135 | */ 136 | def record(value: Long, expectedIntervalBetweenValueSamples: Long): Unit = { 137 | hiccupTimeMetric.record(value) 138 | 139 | if (expectedIntervalBetweenValueSamples > 0) { 140 | var missingValue = value - expectedIntervalBetweenValueSamples 141 | 142 | while (missingValue >= expectedIntervalBetweenValueSamples) { 143 | hiccupTimeMetric.record(missingValue) 144 | missingValue -= expectedIntervalBetweenValueSamples 145 | } 146 | } 147 | } 148 | 149 | def terminate():Unit = 150 | _doRun = false 151 | 152 | def updateInterval(duration: Duration): Unit = 153 | _hiccupNanos = duration.toNanos 154 | } 155 | } 156 | 157 | object ProcessMetricsCollector { 158 | 159 | class Factory extends ModuleFactory { 160 | override def create(settings: ModuleFactory.Settings): Module = 161 | new ProcessMetricsCollector(settings.executionContext) 162 | } 163 | } -------------------------------------------------------------------------------- /version.sbt: -------------------------------------------------------------------------------- 1 | version in ThisBuild := "2.0.3-SNAPSHOT" 2 | --------------------------------------------------------------------------------