├── .github ├── Dockerfile ├── config.test ├── event.test └── workflows │ └── ruby.yml ├── .gitignore ├── CHANGELOG.md ├── CONTRIBUTORS ├── DEVELOPER.md ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── docs └── index.asciidoc ├── lib └── logstash │ └── outputs │ └── prometheus.rb ├── logstash-output-prometheus.gemspec ├── script ├── build ├── cibuild └── test └── spec └── outputs └── prometheus_spec.rb /.github/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.elastic.co/logstash/logstash:7.3.0 2 | 3 | USER 0 4 | 5 | COPY *.gem /plugins/ 6 | 7 | RUN logstash-plugin install --no-verify /plugins/*.gem 8 | 9 | COPY .github/*.test / 10 | -------------------------------------------------------------------------------- /.github/config.test: -------------------------------------------------------------------------------- 1 | input{ 2 | file{ 3 | path => ["/event.test"] 4 | mode => "read" 5 | } 6 | } 7 | output { 8 | prometheus { 9 | increment => { 10 | mycounter => { 11 | description => "This is my test counter" 12 | labels => { 13 | value => "%{[message]}" 14 | } 15 | type => "counter" 16 | } 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /.github/event.test: -------------------------------------------------------------------------------- 1 | hi -------------------------------------------------------------------------------- /.github/workflows/ruby.yml: -------------------------------------------------------------------------------- 1 | name: Integration Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - release/* 8 | pull_request: 9 | branches: 10 | - master 11 | 12 | jobs: 13 | build-test: 14 | runs-on: ubuntu-latest 15 | container: 16 | image: jruby:9.2.8 17 | steps: 18 | - uses: actions/checkout@v1 19 | - name: Install git 20 | run: | 21 | apt-get update 22 | apt-get install -y --no-install-recommends git 23 | - name: Install deps and run RSpec 24 | env: 25 | CI_GEM_NAME: logstash-output-prom.gem 26 | run: script/cibuild 27 | - name: Upload build artifact 28 | uses: actions/upload-artifact@v1 29 | with: 30 | name: gem 31 | path: logstash-output-prom.gem 32 | end-to-end: 33 | needs: build-test 34 | runs-on: ubuntu-latest 35 | steps: 36 | - uses: actions/checkout@v1 37 | - name: Download gem 38 | uses: actions/download-artifact@v1 39 | with: 40 | name: gem 41 | - name: Build docker image 42 | run: | 43 | cp gem/*.gem . 44 | docker build -f .github/Dockerfile -t logstash-output-prom:dev . 45 | - name: Test Plugin 46 | run: | 47 | docker run -d -p 9640:9640 logstash-output-prom:dev -f /config.test 48 | sleep 30 49 | curl localhost:9640/metrics > output 50 | cat output 51 | cat output | grep 'mycounter{value="hi"} 1.0' -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | /.config 4 | /coverage/ 5 | /InstalledFiles 6 | /pkg/ 7 | /spec/reports/ 8 | /spec/examples.txt 9 | /test/tmp/ 10 | /test/version_tmp/ 11 | /tmp/ 12 | 13 | # Used by dotenv library to load environment variables. 14 | # .env 15 | 16 | # Ignore Byebug command history file. 17 | .byebug_history 18 | 19 | ## Specific to RubyMotion: 20 | .dat* 21 | .repl_history 22 | build/ 23 | *.bridgesupport 24 | build-iPhoneOS/ 25 | build-iPhoneSimulator/ 26 | 27 | ## Specific to RubyMotion (use of CocoaPods): 28 | # 29 | # We recommend against adding the Pods directory to your .gitignore. However 30 | # you should judge for yourself, the pros and cons are mentioned at: 31 | # https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control 32 | # 33 | # vendor/Pods/ 34 | 35 | ## Documentation cache and generated files: 36 | /.yardoc/ 37 | /_yardoc/ 38 | /doc/ 39 | /rdoc/ 40 | 41 | ## Environment normalization: 42 | /.bundle/ 43 | /vendor/bundle 44 | /lib/bundler/man/ 45 | 46 | # for a library or gem, you might want to ignore these files since the code is 47 | # intended to run in multiple environments; otherwise, check them in: 48 | Gemfile.lock 49 | .ruby-version 50 | .ruby-gemset 51 | 52 | # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: 53 | .rvmrc 54 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.1.3 2 | - Threading problem was proven to be caused by another plugin, unrelated to us. Reverting to a shard concurrency model 3 | - Upgraded prometheus/client to v1.0.0 4 | ## 0.1.2 5 | - Set to be a single threaded plugin while investigation into deadlocks occurs. 6 | ## 0.1.1 7 | - Fixed bug with unique labels under the same metric name for timers 8 | ## 0.1.0 9 | - Plugin created with the logstash plugin generator 10 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | The following is a list of people who have contributed ideas, code, bug 2 | reports, or in general have helped logstash along its way. 3 | 4 | Contributors: 5 | * Spencer Malone 6 | 7 | Note: If you've sent us patches, bug reports, or otherwise contributed to 8 | Logstash, and you aren't on the list above and want to be, please let us know 9 | and we'll make sure you're here. Contributions from folks like you are what make 10 | open source awesome. 11 | -------------------------------------------------------------------------------- /DEVELOPER.md: -------------------------------------------------------------------------------- 1 | # logstash-output-prometheus 2 | Example output plugin. This should help bootstrap your effort to write your own output plugin! 3 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec 3 | 4 | gem 'logstash-core' 5 | gem 'logstash-core-plugin-api' 6 | 7 | gem 'prometheus-client', "1.0.0" 8 | gem "rack", ">= 1.6.11" 9 | 10 | gem 'logstash-devutils', ">= 1.3.1", :group => [:development] 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Licensed under the Apache License, Version 2.0 (the "License"); 2 | you may not use this file except in compliance with the License. 3 | You may obtain a copy of the License at 4 | 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Logstash Plugin 2 | 3 | This is a plugin for [Logstash](https://github.com/elastic/logstash). 4 | 5 | It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way. 6 | 7 | This plugin allows you to expose metrics from logstash to a prometheus exporter, hosted by your logstash instance. 8 | 9 | 10 | ## Building 11 | 12 | ### Requirements 13 | - JRuby 14 | - JDK 15 | - Git 16 | - bundler 17 | 18 | ### Build steps 19 | 20 | `./script/build` 21 | 22 | ## Contributing 23 | 24 | All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin. 25 | 26 | Programming is not a required skill. Whatever you've seen about open source and maintainers or community members saying "send patches or die" - you will not see that here. 27 | 28 | It is more important to the community that you are able to contribute. 29 | 30 | For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file. 31 | 32 | ## Examples 33 | 34 | ``` 35 | logstash -e 'input { stdin { } } 36 | filter { 37 | mutate { 38 | add_field => { 39 | "timer" => "%{[message]}" 40 | } 41 | } 42 | mutate { 43 | convert => { "timer" => "float" } 44 | } 45 | } 46 | output { 47 | prometheus { 48 | timer => { 49 | histogramtest => { 50 | description => "This is my histogram" 51 | value => "%{[timer]}" 52 | type => "histogram" 53 | buckets => [0.1, 1, 5, 10] 54 | labels => { 55 | mylabel => "testlabel" 56 | } 57 | } 58 | } 59 | } 60 | prometheus { 61 | port => 9641 # You can run multiple ports if you want different outputs scraped by different prom instances. 62 | timer => { 63 | summarytest => { 64 | description => "This is my summary" 65 | value => "%{[timer]}" 66 | type => "summary" 67 | } 68 | } 69 | } 70 | }' 71 | ``` 72 | 73 | ``` 74 | logstash -e 'input { stdin { } } 75 | output { 76 | prometheus { 77 | increment => { 78 | mycounter => { 79 | description => "This is my test counter" 80 | labels => { 81 | value => "%{[message]}" 82 | } 83 | by => "1" 84 | type => "counter" 85 | } 86 | } 87 | } 88 | 89 | prometheus { 90 | increment => { 91 | totaleventscustom => { 92 | description => "This is my second test counter" 93 | } 94 | } 95 | } 96 | }' 97 | ``` 98 | 99 | ## Things to keep in mind 100 | 101 | As outlined in https://www.robustperception.io/putting-queues-in-front-of-prometheus-for-reliability, putting a queue in front of Prometheus can have negative effects. When doing this, please ensure you are monitoring the time it takes events to pass through your queue. 102 | 103 | ## Steps towards a 1.0 release 104 | 105 | Things needed for a 1.0 release: 106 | - Official logstash adoption (nice to have). See: https://github.com/elastic/logstash/issues/11153 107 | - ~`prometheus-client` over at https://github.com/prometheus/client_ruby is still in alpha.~ This has now been solved. 108 | - A stable release without known errors for one month. 109 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "logstash/devutils/rake" 2 | -------------------------------------------------------------------------------- /docs/index.asciidoc: -------------------------------------------------------------------------------- 1 | :plugin: prometheus 2 | :type: output 3 | :default_codec: plain 4 | // Update header with plugin name and default codec 5 | 6 | /////////////////////////////////////////// 7 | START - GENERATED VARIABLES, DO NOT EDIT! 8 | /////////////////////////////////////////// 9 | :version: %VERSION% 10 | :release_date: %RELEASE_DATE% 11 | :changelog_url: %CHANGELOG_URL% 12 | :include_path: ../../../../logstash/docs/include 13 | /////////////////////////////////////////// 14 | END - GENERATED VARIABLES, DO NOT EDIT! 15 | /////////////////////////////////////////// 16 | 17 | [id="plugins-{type}s-{plugin}"] 18 | 19 | === Prometheus output plugin 20 | 21 | include::{include_path}/plugin_header.asciidoc[] 22 | 23 | ==== Description 24 | 25 | This logstash output plugin exposes an endpoint for https://prometheus.io/[Prometheus] to scrape. 26 | 27 | Use cases include but are not limited to: 28 | * Monitoring the time it takes for your logging pipeline to reach logstash 29 | * Reusing your logging pipeline for exposing metrics 30 | * Tracking the health of your data 31 | 32 | [id="plugins-{type}s-{plugin}-options"] 33 | ==== Example Output Configuration Options 34 | 35 | [cols="<,<,<",options="header",] 36 | |======================================================================= 37 | |Setting |Input type|Required 38 | | <> |<>|No 39 | | <> |<>|No 40 | | <> |<>|No 41 | | <> |<>|No 42 | | <> |<>|No 43 | |======================================================================= 44 | 45 | [id="plugins-{type}s-{plugin}-decrement"] 46 | ===== `decrement` 47 | 48 | * Value type is <> 49 | * Default value is `{}` 50 | * Default decrement by value is `1` 51 | 52 | Decrement can be used to decrement a gauge. No other types are supported. 53 | 54 | Example: 55 | [source,ruby] 56 | ---------------------------------- 57 | decrement => { 58 | negaevents_total => { 59 | description => "This is my test gauge" 60 | labels => { 61 | mylabel => "%{[message]}" 62 | } 63 | by => "1" 64 | } 65 | } 66 | ---------------------------------- 67 | 68 | [id="plugins-{type}s-{plugin}-increment"] 69 | ===== `increment` 70 | 71 | * Value type is <> 72 | * Default value is `{}` 73 | * Default type of metric is a `counter` 74 | * Default increment by value is `1` 75 | 76 | Increment either a gauge or a counter. 77 | 78 | Example of a `counter`: 79 | [source,ruby] 80 | ---------------------------------- 81 | increment => { 82 | events_count => { 83 | description => "This is my test gauge" 84 | labels => { 85 | mylabel => "%{[message]}" 86 | } 87 | by => "1" 88 | } 89 | } 90 | ---------------------------------- 91 | 92 | Example of a `gauge`: 93 | [source,ruby] 94 | ---------------------------------- 95 | increment => { 96 | events_total => { 97 | description => "This is my test gauge" 98 | labels => { 99 | mylabel => "%{[message]}" 100 | } 101 | by => "5" 102 | type => "gauge" 103 | } 104 | } 105 | ---------------------------------- 106 | 107 | [id="plugins-{type}s-{plugin}-port"] 108 | ===== `port` 109 | 110 | * Value type is <> 111 | * Default value is `9640` 112 | 113 | The port that will be used when exposing a metric name. 114 | With a default port of 9640, Prometheus should be scraping from `https://:9640/metrics` 115 | 116 | [id="plugins-{type}s-{plugin}-host"] 117 | ===== `host` 118 | 119 | * Value type is <> 120 | * Default value is `0.0.0.0` 121 | 122 | The host that will be used to listen. 123 | 124 | [id="plugins-{type}s-{plugin}-set"] 125 | ===== `set` 126 | 127 | * Value type is <> 128 | * Default value is `{}` 129 | 130 | Set can be used to set a gauge to a specific value. No other types are supported. 131 | 132 | Example: 133 | [source,ruby] 134 | ---------------------------------- 135 | set => { 136 | events_total => { 137 | description => "This is my test gauge" 138 | labels => { 139 | mylabel => "%{[message]}" 140 | } 141 | value => "123" 142 | } 143 | } 144 | ---------------------------------- 145 | 146 | [id="plugins-{type}s-{plugin}-timer"] 147 | ===== `timer` 148 | 149 | * Value type is <> 150 | * Default value is `{}` 151 | * Default type of metric is a `summary` 152 | 153 | Observe timing data for a histogram or summary. 154 | Histograms are recommended, but require setting the `buckets` 155 | Summaries will only expose a sum and count, quantile summaries are not supported. 156 | Read https://prometheus.io/docs/practices/histograms/[best practices for summaries and histograms] for more information. 157 | 158 | Example of a `histogram`: 159 | [source,ruby] 160 | ---------------------------------- 161 | histogramtest => { 162 | description => "This is my histogram" 163 | value => "%{[timer]}" 164 | labels => { 165 | mylabel => "%{[message]}" 166 | } 167 | type => "histogram" 168 | buckets => [0.1, 1, 5, 10] 169 | } 170 | ---------------------------------- 171 | 172 | Example of a `summary`: 173 | [source,ruby] 174 | ---------------------------------- 175 | summarytest => { 176 | description => "This is my summary" 177 | value => "%{[timer]}" 178 | labels => { 179 | mylabel => "%{[message]}" 180 | } 181 | } 182 | ---------------------------------- 183 | 184 | [id="plugins-{type}s-{plugin}-further-reading"] 185 | 186 | ==== Best Practices & Further Reading 187 | 188 | https://www.robustperception.io/how-should-pipelines-be-monitored 189 | https://www.robustperception.io/putting-queues-in-front-of-prometheus-for-reliability 190 | 191 | // The full list of Value Types is here: 192 | // https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html 193 | 194 | [id="plugins-{type}s-{plugin}-common-options"] 195 | include::{include_path}/{type}.asciidoc[] 196 | 197 | :default_codec!: 198 | -------------------------------------------------------------------------------- /lib/logstash/outputs/prometheus.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require "logstash/outputs/base" 3 | require 'rack' 4 | require 'prometheus/middleware/exporter' 5 | require 'prometheus/middleware/collector' 6 | require 'prometheus/client' 7 | 8 | # An prometheus output that does nothing. 9 | class LogStash::Outputs::Prometheus < LogStash::Outputs::Base 10 | config_name "prometheus" 11 | concurrency :shared 12 | 13 | config :port, :validate => :number, :default => 9640 14 | 15 | config :host, :validate => :string, :default => "0.0.0.0" 16 | 17 | config :increment, :validate => :hash, :default => {} 18 | # Decrement is only available for gauges 19 | config :decrement, :validate => :hash, :default => {} 20 | # Decrement is only available for gauges 21 | config :set, :validate => :hash, :default => {} 22 | 23 | config :timer, :validate => :hash, :default => {} 24 | 25 | public 26 | def register 27 | $prom_servers ||= {} 28 | $metrics ||= {} 29 | 30 | if $prom_servers[@port].nil? 31 | $prom_servers[@port] = Prometheus::Client::Registry.new 32 | prom_server = $prom_servers[@port] 33 | 34 | app = 35 | Rack::Builder.new(@port) do 36 | use ::Rack::Deflater 37 | use ::Prometheus::Middleware::Exporter, registry: prom_server 38 | 39 | run ->(_) { [200, {'Content-Type' => 'text/html'}, ['Please access /metrics to see exposed metrics for this Logstash instance.']] } 40 | end.to_app 41 | 42 | @thread = Thread.new do 43 | Rack::Handler::WEBrick.run(app, Port: @port, Host: @host) 44 | end 45 | end 46 | 47 | prom_server = $prom_servers[@port] 48 | 49 | @increment.each do |metric_name, val| 50 | val = setup_registry_labels(val) 51 | if $metrics[port.to_s + metric_name].nil? 52 | if val['type'] == "gauge" 53 | metric = prom_server.gauge(metric_name.to_sym, docstring: val['description'], labels: val['labels'].keys) 54 | else 55 | metric = prom_server.counter(metric_name.to_sym, docstring: val['description'], labels: val['labels'].keys) 56 | end 57 | $metrics[port.to_s + metric_name] = metric 58 | end 59 | end 60 | 61 | @decrement.each do |metric_name, val| 62 | val = setup_registry_labels(val) 63 | if $metrics[port.to_s + metric_name].nil? 64 | metric = prom_server.gauge(metric_name.to_sym, docstring: val['description'], labels: val['labels'].keys) 65 | 66 | $metrics[port.to_s + metric_name] = metric 67 | end 68 | end 69 | 70 | @set.each do |metric_name, val| 71 | val = setup_registry_labels(val) 72 | if $metrics[port.to_s + metric_name].nil? 73 | metric = prom_server.gauge(metric_name.to_sym, docstring: val['description'], labels: val['labels'].keys) 74 | 75 | $metrics[port.to_s + metric_name] = metric 76 | end 77 | end 78 | 79 | @timer.each do |metric_name, val| 80 | val = setup_registry_labels(val) 81 | 82 | if $metrics[port.to_s + metric_name].nil? 83 | if val['type'] == "histogram" 84 | metric = prom_server.histogram(metric_name.to_sym, docstring: val['description'], labels: val['labels'].keys, buckets: val['buckets']) 85 | else 86 | metric = prom_server.summary(metric_name.to_sym, labels: val['labels'].keys, docstring: val['description']) 87 | end 88 | 89 | $metrics[port.to_s + metric_name] = metric 90 | end 91 | end 92 | end # def register 93 | 94 | def kill_thread() 95 | @thread.kill 96 | $prom_servers[@port] = nil 97 | end 98 | 99 | protected 100 | def setup_registry_labels(val) 101 | if val['labels'].nil? 102 | val['labels'] = {} 103 | end 104 | 105 | val['labels'].keys.each do |key| 106 | val['labels'][(key.to_sym rescue key) || key] = val['labels'].delete(key) 107 | end 108 | 109 | return val 110 | end 111 | 112 | public 113 | def receive(event) 114 | @increment.each do |metric_name, val| 115 | labels = setup_event_labels(val, event) 116 | by = val["by"] ? event.sprintf(val["by"]).to_i : 1 117 | $metrics[port.to_s + metric_name].increment(by: by, labels:labels) 118 | end 119 | 120 | @decrement.each do |metric_name, val| 121 | labels = setup_event_labels(val, event) 122 | by = val["by"] ? event.sprintf(val["by"]).to_i : 1 123 | $metrics[port.to_s + metric_name].decrement(by: by, labels: labels) 124 | end 125 | 126 | @set.each do |metric_name, val| 127 | labels = setup_event_labels(val, event) 128 | $metrics[port.to_s + metric_name].set(event.sprintf(val['value']).to_f,labels: labels) 129 | end 130 | 131 | @timer.each do |metric_name, val| 132 | labels = setup_event_labels(val, event) 133 | $metrics[port.to_s + metric_name].observe(event.sprintf(val['value']).to_f,labels: labels) 134 | end 135 | end # def event 136 | 137 | protected 138 | def setup_event_labels(val, event) 139 | labels = {} 140 | val['labels'].each do |label, lval| 141 | labels[label] = event.sprintf(lval) 142 | end 143 | 144 | return labels 145 | end 146 | end # class LogStash::Outputs::Prometheus 147 | -------------------------------------------------------------------------------- /logstash-output-prometheus.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | s.name = 'logstash-output-prometheus' 3 | s.version = '0.1.3' 4 | s.licenses = ['Apache-2.0'] 5 | s.summary = 'Output logstash data to a prometheus exporter' 6 | # s.homepage = 'Nada' 7 | s.authors = ['Spencer Malone'] 8 | s.email = 'spencer@mailchimp.com' 9 | s.require_paths = ['lib'] 10 | 11 | # Files 12 | s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT'] 13 | # Tests 14 | s.test_files = s.files.grep(%r{^(test|spec|features)/}) 15 | 16 | # Special flag to let us know this is actually a logstash plugin 17 | s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } 18 | 19 | # Gem dependencies 20 | s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0" 21 | s.add_runtime_dependency "logstash-codec-plain" 22 | s.add_runtime_dependency "prometheus-client", "1.0.0" 23 | s.add_runtime_dependency "rack", ">= 1.6.11" 24 | 25 | s.add_development_dependency "logstash-devutils", "~> 1.3", ">= 1.3.1" 26 | end 27 | -------------------------------------------------------------------------------- /script/build: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | cd $(git rev-parse --show-toplevel) 6 | 7 | bundle install --jobs 4 --retry 3 8 | 9 | gem build logstash-output-prometheus.gemspec 10 | 11 | if [[ -z "${CI_GEM_NAME-}" ]]; then 12 | echo "Successfully built! Install in your logstash by running..." 13 | echo "logstash-plugin install $PWD/logstash-output-prometheus-0.1.3.gem from your logstash directory" 14 | else 15 | echo "Successfully built!" 16 | # Makes the assumption that there is only one build. 17 | mv logstash-output-prometheus*.gem "${CI_GEM_NAME}" 18 | fi -------------------------------------------------------------------------------- /script/cibuild: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | cd $(git rev-parse --show-toplevel) 6 | 7 | gem install bundler 8 | 9 | ./script/build 10 | ./script/test -------------------------------------------------------------------------------- /script/test: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -euo pipefail 4 | 5 | cd $(git rev-parse --show-toplevel) 6 | 7 | bundle exec rspec -------------------------------------------------------------------------------- /spec/outputs/prometheus_spec.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require "logstash/devutils/rspec/spec_helper" 3 | require "logstash/outputs/prometheus" 4 | require "logstash/codecs/plain" 5 | require "logstash/event" 6 | require 'net/http' 7 | 8 | describe LogStash::Outputs::Prometheus do 9 | let(:port) { rand(2000..10000) } 10 | let(:host) { "0.0.0.0" } 11 | let(:output) { LogStash::Outputs::Prometheus.new(properties) } 12 | let(:secondary_output) { 13 | if secondary_properties.nil? 14 | LogStash::Outputs::Prometheus.new(properties) 15 | else 16 | LogStash::Outputs::Prometheus.new(secondary_properties) 17 | end 18 | } 19 | 20 | before do 21 | output.register 22 | secondary_output.register 23 | end 24 | 25 | let(:secondary_properties) do 26 | nil 27 | end 28 | 29 | let(:event) do 30 | LogStash::Event.new( 31 | properties 32 | ) 33 | end 34 | 35 | let(:secondary_event) do 36 | if secondary_properties.nil? 37 | LogStash::Event.new( 38 | secondary_properties 39 | ) 40 | else 41 | event 42 | end 43 | end 44 | 45 | shared_examples "it should expose data" do |*values| 46 | it "should expose data" do 47 | output.receive(event) 48 | 49 | url = URI.parse("http://localhost:#{port}/metrics") 50 | req = Net::HTTP::Get.new(url.to_s) 51 | 52 | attempts = 0 53 | 54 | begin 55 | res = Net::HTTP.start(url.host, url.port) {|http| 56 | http.request(req) 57 | } 58 | rescue 59 | attempts++ 60 | sleep(0.1) 61 | if attempts < 10 62 | retry 63 | end 64 | end 65 | 66 | values.each do |value| 67 | expect(res.body).to include(value) 68 | end 69 | end 70 | end 71 | 72 | shared_examples "it should expose data from multiple outputs" do |*values| 73 | it "should be able to handle unique labels under the same name" do 74 | secondary_output.receive(secondary_event) 75 | 76 | output.receive(event) 77 | 78 | url = URI.parse("http://localhost:#{port}/metrics") 79 | req = Net::HTTP::Get.new(url.to_s) 80 | 81 | attempts = 0 82 | 83 | begin 84 | res = Net::HTTP.start(url.host, url.port) {|http| 85 | http.request(req) 86 | } 87 | rescue 88 | attempts++ 89 | sleep(0.1) 90 | if attempts < 10 91 | retry 92 | end 93 | end 94 | 95 | values.each do |value| 96 | expect(res.body).to include(value) 97 | end 98 | end 99 | end 100 | 101 | describe "counter behavior" do 102 | describe "default increment" do 103 | let(:properties) { 104 | { 105 | "port" => port, 106 | "host" => host, 107 | "increment" => { 108 | "basic_counter" => { 109 | "description" => "Test", 110 | "labels" => { 111 | "mylabel" => "hi" 112 | } 113 | } 114 | } 115 | } 116 | } 117 | 118 | let(:secondary_properties) { 119 | { 120 | "port" => port, 121 | "host" => host, 122 | "increment" => { 123 | "basic_counter" => { 124 | "description" => "Test", 125 | "labels" => { 126 | "mylabel" => "boo" 127 | } 128 | } 129 | } 130 | } 131 | } 132 | 133 | include_examples "it should expose data", 'basic_counter{mylabel="hi"} 1', "# TYPE basic_counter counter", "# HELP basic_counter Test" 134 | include_examples "it should expose data from multiple outputs", 'basic_counter{mylabel="hi"} 1', 'basic_counter{mylabel="boo"} 1' 135 | 136 | end 137 | 138 | describe "custom increment by" do 139 | let(:properties) { 140 | { 141 | "port" => port, 142 | "host" => host, 143 | "increment" => { 144 | "basic_counter" => { 145 | "description" => "Test", 146 | "by" => "5", 147 | "labels" => { 148 | "mylabel" => "hi" 149 | } 150 | } 151 | } 152 | } 153 | } 154 | 155 | let(:secondary_properties) { 156 | { 157 | "port" => port, 158 | "host" => host, 159 | "increment" => { 160 | "basic_counter" => { 161 | "description" => "Test", 162 | "by" => "10", 163 | "labels" => { 164 | "mylabel" => "boo" 165 | } 166 | } 167 | } 168 | } 169 | } 170 | 171 | include_examples "it should expose data", 'basic_counter{mylabel="hi"} 5', "# TYPE basic_counter counter", "# HELP basic_counter Test" 172 | include_examples "it should expose data from multiple outputs", 'basic_counter{mylabel="hi"} 5', 'basic_counter{mylabel="boo"} 10' 173 | 174 | end 175 | end 176 | 177 | describe "gauge behavior" do 178 | describe "increment" do 179 | let(:properties) { 180 | { 181 | "port" => port, 182 | "host" => host, 183 | "increment" => { 184 | "basic_gauge" => { 185 | "description" => "Test1", 186 | "labels" => { 187 | "mylabel" => "hi" 188 | }, 189 | "type" => "gauge" 190 | } 191 | } 192 | } 193 | } 194 | 195 | let(:secondary_properties) { 196 | { 197 | "port" => port, 198 | "host" => host, 199 | "increment" => { 200 | "basic_gauge" => { 201 | "description" => "Test1", 202 | "type" => "gauge", 203 | "labels" => { 204 | "mylabel" => "boo" 205 | } 206 | } 207 | } 208 | } 209 | } 210 | 211 | include_examples "it should expose data", 'basic_gauge{mylabel="hi"} 1.0', "# TYPE basic_gauge gauge", "# HELP basic_gauge Test1" 212 | include_examples "it should expose data from multiple outputs", 'basic_gauge{mylabel="hi"} 1', 'basic_gauge{mylabel="boo"} 1' 213 | end 214 | describe "increment with custom by" do 215 | let(:properties) { 216 | { 217 | "port" => port, 218 | "host" => host, 219 | "increment" => { 220 | "basic_gauge" => { 221 | "description" => "Test1", 222 | "by" => "5", 223 | "type" => "gauge" 224 | } 225 | } 226 | } 227 | } 228 | include_examples "it should expose data", "basic_gauge 5.0", "# TYPE basic_gauge gauge", "# HELP basic_gauge Test1" 229 | end 230 | 231 | describe "decrement" do 232 | let(:properties) { 233 | { 234 | "port" => port, 235 | "host" => host, 236 | "decrement" => { 237 | "basic_gauge" => { 238 | "description" => "Testone", 239 | "type" => "gauge" 240 | } 241 | } 242 | } 243 | } 244 | include_examples "it should expose data", "basic_gauge -1.0", "# TYPE basic_gauge gauge", "# HELP basic_gauge Testone" 245 | end 246 | 247 | describe "decrement with custom by" do 248 | let(:properties) { 249 | { 250 | "port" => port, 251 | "host" => host, 252 | "decrement" => { 253 | "basic_gauge" => { 254 | "description" => "Testone", 255 | "by" => "10", 256 | "type" => "gauge" 257 | } 258 | } 259 | } 260 | } 261 | include_examples "it should expose data", "basic_gauge -10.0", "# TYPE basic_gauge gauge", "# HELP basic_gauge Testone" 262 | end 263 | 264 | describe "set" do 265 | let(:properties) { 266 | { 267 | "port" => port, 268 | "host" => host, 269 | "set" => { 270 | "basic_gauge" => { 271 | "description" => "Testone", 272 | "type" => "gauge", 273 | "value" => "123" 274 | } 275 | } 276 | } 277 | } 278 | include_examples "it should expose data", "basic_gauge 123.0", "# TYPE basic_gauge gauge", "# HELP basic_gauge Testone" 279 | end 280 | end 281 | 282 | describe "summary behavior" do 283 | let(:properties) { 284 | { 285 | "port" => port, 286 | "host" => host, 287 | "timer" => { 288 | "huh" => { 289 | "description" => "noway", 290 | "type" => "summary", 291 | "value" => "11", 292 | "labels" => { 293 | "mylabel" => "hi" 294 | } 295 | } 296 | } 297 | } 298 | } 299 | 300 | let(:secondary_properties) { 301 | { 302 | "port" => port, 303 | "host" => host, 304 | "timer" => { 305 | "huh" => { 306 | "description" => "noway", 307 | "type" => "summary", 308 | "value" => "10", 309 | "labels" => { 310 | "mylabel" => "boo" 311 | } 312 | } 313 | } 314 | } 315 | } 316 | include_examples "it should expose data", 'huh_sum{mylabel="hi"} 11.0', 'huh_count{mylabel="hi"} 1.0', "# TYPE huh summary", "# HELP huh noway" 317 | include_examples "it should expose data from multiple outputs", 'huh_sum{mylabel="hi"} 11.0', 'huh_sum{mylabel="boo"} 10.0' 318 | end 319 | 320 | describe "histogram behavior" do 321 | describe "description" do 322 | let(:properties) { 323 | { 324 | "port" => port, 325 | "host" => host, 326 | "timer" => { 327 | "history" => { 328 | "description" => "abe", 329 | "type" => "histogram", 330 | "buckets" => [1, 5, 10], 331 | "value" => "0", 332 | "labels" => { 333 | "mylabel" => "hi" 334 | } 335 | } 336 | } 337 | } 338 | } 339 | let(:secondary_properties) { 340 | { 341 | "port" => port, 342 | "host" => host, 343 | "timer" => { 344 | "history" => { 345 | "description" => "abe", 346 | "type" => "histogram", 347 | "buckets" => [1, 5, 10], 348 | "value" => "0", 349 | "labels" => { 350 | "mylabel" => "boo" 351 | } 352 | } 353 | } 354 | } 355 | } 356 | include_examples "it should expose data", "# TYPE history histogram", "# HELP history abe" 357 | include_examples "it should expose data from multiple outputs", 'history_sum{mylabel="hi"} 0.0', 'history_sum{mylabel="boo"} 0.0' 358 | end 359 | 360 | describe "sum and count" do 361 | let(:properties) { 362 | { 363 | "port" => port, 364 | "host" => host, 365 | "timer" => { 366 | "history" => { 367 | "description" => "abe", 368 | "type" => "histogram", 369 | "buckets" => [1, 5, 10], 370 | "value" => "111" 371 | } 372 | } 373 | } 374 | } 375 | include_examples "it should expose data", "history_sum 111.0", "history_count 1.0" 376 | end 377 | 378 | describe "minimum histogram" do 379 | let(:properties) { 380 | { 381 | "port" => port, 382 | "host" => host, 383 | "timer" => { 384 | "history" => { 385 | "description" => "abe", 386 | "type" => "histogram", 387 | "buckets" => [1, 5, 10], 388 | "value" => "0" 389 | } 390 | } 391 | } 392 | } 393 | include_examples "it should expose data", 'history_bucket{le="1"} 1.0', 'history_bucket{le="5"} 1.0', 'history_bucket{le="10"} 1.0', 'history_bucket{le="+Inf"} 1.0' 394 | end 395 | 396 | describe "middle histogram" do 397 | let(:properties) { 398 | { 399 | "port" => port, 400 | "host" => host, 401 | "timer" => { 402 | "history" => { 403 | "description" => "abe", 404 | "type" => "histogram", 405 | "buckets" => [1, 5, 10], 406 | "value" => "5" 407 | } 408 | } 409 | } 410 | } 411 | include_examples "it should expose data", 'history_bucket{le="1"} 0.0', 'history_bucket{le="5"} 0.0', 'history_bucket{le="10"} 1.0', 'history_bucket{le="+Inf"} 1.0' 412 | end 413 | 414 | describe "max histogram" do 415 | let(:properties) { 416 | { 417 | "port" => port, 418 | "host" => host, 419 | "timer" => { 420 | "history" => { 421 | "description" => "abe", 422 | "type" => "histogram", 423 | "buckets" => [1, 5, 10], 424 | "value" => "9" 425 | } 426 | } 427 | } 428 | } 429 | include_examples "it should expose data", 'history_bucket{le="1"} 0.0', 'history_bucket{le="5"} 0.0', 'history_bucket{le="10"} 1.0', 'history_bucket{le="+Inf"} 1.0' 430 | end 431 | 432 | describe "beyond max histogram" do 433 | let(:properties) { 434 | { 435 | "port" => port, 436 | "host" => host, 437 | "timer" => { 438 | "history" => { 439 | "description" => "abe", 440 | "type" => "histogram", 441 | "buckets" => [1, 5, 10], 442 | "value" => "100" 443 | } 444 | } 445 | } 446 | } 447 | include_examples "it should expose data", 'history_bucket{le="1"} 0.0', 'history_bucket{le="5"} 0.0', 'history_bucket{le="10"} 0.0', 'history_bucket{le="+Inf"} 1.0' 448 | end 449 | 450 | end 451 | end 452 | --------------------------------------------------------------------------------