├── .github
    ├── dependabot.yml
    └── workflows
    │   └── linux.yml
├── .gitignore
├── .rspec
├── ChangeLog
├── Gemfile
├── LICENSE
├── README.md
├── Rakefile
├── fluent-plugin-prometheus.gemspec
├── lib
    └── fluent
    │   └── plugin
    │       ├── filter_prometheus.rb
    │       ├── in_prometheus.rb
    │       ├── in_prometheus
    │           └── async_wrapper.rb
    │       ├── in_prometheus_monitor.rb
    │       ├── in_prometheus_output_monitor.rb
    │       ├── in_prometheus_tail_monitor.rb
    │       ├── out_prometheus.rb
    │       ├── prometheus.rb
    │       ├── prometheus
    │           └── placeholder_expander.rb
    │       └── prometheus_metrics.rb
├── misc
    ├── fluentd_sample.conf
    ├── nginx_proxy.conf
    ├── prometheus.yaml
    └── prometheus_alerts.yaml
└── spec
    ├── fluent
        └── plugin
        │   ├── filter_prometheus_spec.rb
        │   ├── in_prometheus_monitor_spec.rb
        │   ├── in_prometheus_spec.rb
        │   ├── in_prometheus_tail_monitor_spec.rb
        │   ├── out_prometheus_spec.rb
        │   ├── prometheus
        │       └── placeholder_expander_spec.rb
        │   ├── prometheus_metrics_spec.rb
        │   └── shared.rb
    └── spec_helper.rb


/.github/dependabot.yml:
--------------------------------------------------------------------------------
1 | version: 2
2 | updates:
3 |   - package-ecosystem: 'github-actions'
4 |     directory: '/'
5 |     schedule:
6 |       interval: 'weekly'
7 | 


--------------------------------------------------------------------------------
/.github/workflows/linux.yml:
--------------------------------------------------------------------------------
 1 | name: linux
 2 | on:
 3 |   push:
 4 |     branches: [master]
 5 |   pull_request:
 6 |     branches: [master]
 7 | jobs:
 8 |   build:
 9 |     runs-on: ${{ matrix.os }}
10 |     continue-on-error: ${{ matrix.experimental }}
11 |     strategy:
12 |       fail-fast: false
13 |       matrix:
14 |         ruby: [ '3.4', '3.3', '3.2', '3.1', '3.0', '2.7' ]
15 |         os:
16 |           - ubuntu-latest
17 |         experimental: [false]
18 |     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
19 |     steps:
20 |     - uses: actions/checkout@v4
21 |     - uses: ruby/setup-ruby@v1
22 |       with:
23 |         ruby-version: ${{ matrix.ruby }}
24 |     - name: unit testing
25 |       env:
26 |         CI: true
27 |       run: |
28 |         bundle install --jobs 4 --retry 3
29 |         bundle exec rake spec
30 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | /.bundle/
 2 | /.yardoc
 3 | /Gemfile.lock
 4 | /Gemfile.fluentd.0.10.lock
 5 | /Gemfile.fluentd.0.12.lock
 6 | /_yardoc/
 7 | /coverage/
 8 | /doc/
 9 | /pkg/
10 | /spec/reports/
11 | /tmp/
12 | *.bundle
13 | *.so
14 | *.o
15 | *.a
16 | mkmf.log
17 | 


--------------------------------------------------------------------------------
/.rspec:
--------------------------------------------------------------------------------
1 | --format documentation
2 | --color
3 | 


--------------------------------------------------------------------------------
/ChangeLog:
--------------------------------------------------------------------------------
 1 | Release 2.2.1 - 2025/03/24
 2 | 
 3 | 	* in_prometheus_tail_monitor: Add throttling metrics as `fluentd_tail_file_throttled`. (GitHub#227)
 4 | 
 5 | Release 2.2.0 - 2024/08/02
 6 | 
 7 | 	* in_prometheus: Add gzip support (Add a new parameter `content_encoding gzip`)
 8 | 
 9 | Release 2.1.0 - 2023/06/15
10 | 
11 | 	* Add `initialized` and `initlabels` parameters to `<metric>` element
12 | 	* in_prometheus_tail_monitor: Add file open/closed/rotation metrics
13 | 
14 | Release 2.0.3 - 2022/05/06
15 | 
16 | 	* in_prometheus_output_monitor: Fix a bug where output_status_num_errors metric is missing
17 | 
18 | Release 2.0.2 - 2021/08/02
19 | 
20 | 	* in_prometheus_output_monitor: Follow Fluentd's core metrics mechanism
21 | 
22 | Release 2.0.1 - 2021/04/08
23 | 
24 | 	* out_prometheus: Allow for lookup by symbol as well as string
25 | 
26 | Release 2.0.0 - 2021/02/18
27 | 
28 | 	* Update prometheus-client dependency to 2.1.0 or later
29 | 
30 | Release 1.8.5 - 2020/11/24
31 | 
32 | 	* in_prometheus_monitor: Support USR2 reload
33 | 
34 | Release 1.8.4 - 2020/09/24
35 | 
36 | 	* in_prometheus_output_monitor: Add gauge_all parameter
37 | 
38 | Release 1.8.3 - 2020/08/24
39 | 
40 | 	* Fix resourcr leak in async-http based server
41 | 
42 | Release 1.8.2 - 2020/07/17
43 | 
44 | 	* in_prometheus_output_monitor/in_prometheus_tail_monitor: Support USR2 reload
45 | 
46 | Release 1.8.1 - 2020/07/06
47 | 
48 | 	* Fix aggregate bug with async-http
49 | 
50 | Release 1.8.0 - 2020/04/17
51 | 
52 | 	* Use http_server helper
53 | 	* Require fluentd v1.9.1 or later
54 | 
55 | 
56 | For older releases, see commits on github repository.
57 | 


--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
1 | source 'https://rubygems.org'
2 | 
3 | # Specify your gem's dependencies in fluent-plugin-prometheus.gemspec
4 | gemspec
5 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 | 
  2 |                                  Apache License
  3 |                            Version 2.0, January 2004
  4 |                         http://www.apache.org/licenses/
  5 | 
  6 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  7 | 
  8 |    1. Definitions.
  9 | 
 10 |       "License" shall mean the terms and conditions for use, reproduction,
 11 |       and distribution as defined by Sections 1 through 9 of this document.
 12 | 
 13 |       "Licensor" shall mean the copyright owner or entity authorized by
 14 |       the copyright owner that is granting the License.
 15 | 
 16 |       "Legal Entity" shall mean the union of the acting entity and all
 17 |       other entities that control, are controlled by, or are under common
 18 |       control with that entity. For the purposes of this definition,
 19 |       "control" means (i) the power, direct or indirect, to cause the
 20 |       direction or management of such entity, whether by contract or
 21 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 22 |       outstanding shares, or (iii) beneficial ownership of such entity.
 23 | 
 24 |       "You" (or "Your") shall mean an individual or Legal Entity
 25 |       exercising permissions granted by this License.
 26 | 
 27 |       "Source" form shall mean the preferred form for making modifications,
 28 |       including but not limited to software source code, documentation
 29 |       source, and configuration files.
 30 | 
 31 |       "Object" form shall mean any form resulting from mechanical
 32 |       transformation or translation of a Source form, including but
 33 |       not limited to compiled object code, generated documentation,
 34 |       and conversions to other media types.
 35 | 
 36 |       "Work" shall mean the work of authorship, whether in Source or
 37 |       Object form, made available under the License, as indicated by a
 38 |       copyright notice that is included in or attached to the work
 39 |       (an example is provided in the Appendix below).
 40 | 
 41 |       "Derivative Works" shall mean any work, whether in Source or Object
 42 |       form, that is based on (or derived from) the Work and for which the
 43 |       editorial revisions, annotations, elaborations, or other modifications
 44 |       represent, as a whole, an original work of authorship. For the purposes
 45 |       of this License, Derivative Works shall not include works that remain
 46 |       separable from, or merely link (or bind by name) to the interfaces of,
 47 |       the Work and Derivative Works thereof.
 48 | 
 49 |       "Contribution" shall mean any work of authorship, including
 50 |       the original version of the Work and any modifications or additions
 51 |       to that Work or Derivative Works thereof, that is intentionally
 52 |       submitted to Licensor for inclusion in the Work by the copyright owner
 53 |       or by an individual or Legal Entity authorized to submit on behalf of
 54 |       the copyright owner. For the purposes of this definition, "submitted"
 55 |       means any form of electronic, verbal, or written communication sent
 56 |       to the Licensor or its representatives, including but not limited to
 57 |       communication on electronic mailing lists, source code control systems,
 58 |       and issue tracking systems that are managed by, or on behalf of, the
 59 |       Licensor for the purpose of discussing and improving the Work, but
 60 |       excluding communication that is conspicuously marked or otherwise
 61 |       designated in writing by the copyright owner as "Not a Contribution."
 62 | 
 63 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 64 |       on behalf of whom a Contribution has been received by Licensor and
 65 |       subsequently incorporated within the Work.
 66 | 
 67 |    2. Grant of Copyright License. Subject to the terms and conditions of
 68 |       this License, each Contributor hereby grants to You a perpetual,
 69 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 70 |       copyright license to reproduce, prepare Derivative Works of,
 71 |       publicly display, publicly perform, sublicense, and distribute the
 72 |       Work and such Derivative Works in Source or Object form.
 73 | 
 74 |    3. Grant of Patent License. Subject to the terms and conditions of
 75 |       this License, each Contributor hereby grants to You a perpetual,
 76 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 77 |       (except as stated in this section) patent license to make, have made,
 78 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 79 |       where such license applies only to those patent claims licensable
 80 |       by such Contributor that are necessarily infringed by their
 81 |       Contribution(s) alone or by combination of their Contribution(s)
 82 |       with the Work to which such Contribution(s) was submitted. If You
 83 |       institute patent litigation against any entity (including a
 84 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 85 |       or a Contribution incorporated within the Work constitutes direct
 86 |       or contributory patent infringement, then any patent licenses
 87 |       granted to You under this License for that Work shall terminate
 88 |       as of the date such litigation is filed.
 89 | 
 90 |    4. Redistribution. You may reproduce and distribute copies of the
 91 |       Work or Derivative Works thereof in any medium, with or without
 92 |       modifications, and in Source or Object form, provided that You
 93 |       meet the following conditions:
 94 | 
 95 |       (a) You must give any other recipients of the Work or
 96 |           Derivative Works a copy of this License; and
 97 | 
 98 |       (b) You must cause any modified files to carry prominent notices
 99 |           stating that You changed the files; and
100 | 
101 |       (c) You must retain, in the Source form of any Derivative Works
102 |           that You distribute, all copyright, patent, trademark, and
103 |           attribution notices from the Source form of the Work,
104 |           excluding those notices that do not pertain to any part of
105 |           the Derivative Works; and
106 | 
107 |       (d) If the Work includes a "NOTICE" text file as part of its
108 |           distribution, then any Derivative Works that You distribute must
109 |           include a readable copy of the attribution notices contained
110 |           within such NOTICE file, excluding those notices that do not
111 |           pertain to any part of the Derivative Works, in at least one
112 |           of the following places: within a NOTICE text file distributed
113 |           as part of the Derivative Works; within the Source form or
114 |           documentation, if provided along with the Derivative Works; or,
115 |           within a display generated by the Derivative Works, if and
116 |           wherever such third-party notices normally appear. The contents
117 |           of the NOTICE file are for informational purposes only and
118 |           do not modify the License. You may add Your own attribution
119 |           notices within Derivative Works that You distribute, alongside
120 |           or as an addendum to the NOTICE text from the Work, provided
121 |           that such additional attribution notices cannot be construed
122 |           as modifying the License.
123 | 
124 |       You may add Your own copyright statement to Your modifications and
125 |       may provide additional or different license terms and conditions
126 |       for use, reproduction, or distribution of Your modifications, or
127 |       for any such Derivative Works as a whole, provided Your use,
128 |       reproduction, and distribution of the Work otherwise complies with
129 |       the conditions stated in this License.
130 | 
131 |    5. Submission of Contributions. Unless You explicitly state otherwise,
132 |       any Contribution intentionally submitted for inclusion in the Work
133 |       by You to the Licensor shall be under the terms and conditions of
134 |       this License, without any additional terms or conditions.
135 |       Notwithstanding the above, nothing herein shall supersede or modify
136 |       the terms of any separate license agreement you may have executed
137 |       with Licensor regarding such Contributions.
138 | 
139 |    6. Trademarks. This License does not grant permission to use the trade
140 |       names, trademarks, service marks, or product names of the Licensor,
141 |       except as required for reasonable and customary use in describing the
142 |       origin of the Work and reproducing the content of the NOTICE file.
143 | 
144 |    7. Disclaimer of Warranty. Unless required by applicable law or
145 |       agreed to in writing, Licensor provides the Work (and each
146 |       Contributor provides its Contributions) on an "AS IS" BASIS,
147 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 |       implied, including, without limitation, any warranties or conditions
149 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 |       PARTICULAR PURPOSE. You are solely responsible for determining the
151 |       appropriateness of using or redistributing the Work and assume any
152 |       risks associated with Your exercise of permissions under this License.
153 | 
154 |    8. Limitation of Liability. In no event and under no legal theory,
155 |       whether in tort (including negligence), contract, or otherwise,
156 |       unless required by applicable law (such as deliberate and grossly
157 |       negligent acts) or agreed to in writing, shall any Contributor be
158 |       liable to You for damages, including any direct, indirect, special,
159 |       incidental, or consequential damages of any character arising as a
160 |       result of this License or out of the use or inability to use the
161 |       Work (including but not limited to damages for loss of goodwill,
162 |       work stoppage, computer failure or malfunction, or any and all
163 |       other commercial damages or losses), even if such Contributor
164 |       has been advised of the possibility of such damages.
165 | 
166 |    9. Accepting Warranty or Additional Liability. While redistributing
167 |       the Work or Derivative Works thereof, You may choose to offer,
168 |       and charge a fee for, acceptance of support, warranty, indemnity,
169 |       or other liability obligations and/or rights consistent with this
170 |       License. However, in accepting such obligations, You may act only
171 |       on Your own behalf and on Your sole responsibility, not on behalf
172 |       of any other Contributor, and only if You agree to indemnify,
173 |       defend, and hold each Contributor harmless for any liability
174 |       incurred by, or claims asserted against, such Contributor by reason
175 |       of your accepting any such warranty or additional liability.
176 | 
177 |    END OF TERMS AND CONDITIONS
178 | 
179 |    APPENDIX: How to apply the Apache License to your work.
180 | 
181 |       To apply the Apache License to your work, attach the following
182 |       boilerplate notice, with the fields enclosed by brackets "[]"
183 |       replaced with your own identifying information. (Don't include
184 |       the brackets!)  The text should be enclosed in the appropriate
185 |       comment syntax for the file format. We also recommend that a
186 |       file or class name and description of purpose be included on the
187 |       same "printed page" as the copyright notice for easier
188 |       identification within third-party archives.
189 | 
190 |    Copyright [yyyy] [name of copyright owner]
191 | 
192 |    Licensed under the Apache License, Version 2.0 (the "License");
193 |    you may not use this file except in compliance with the License.
194 |    You may obtain a copy of the License at
195 | 
196 |        http://www.apache.org/licenses/LICENSE-2.0
197 | 
198 |    Unless required by applicable law or agreed to in writing, software
199 |    distributed under the License is distributed on an "AS IS" BASIS,
200 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 |    See the License for the specific language governing permissions and
202 |    limitations under the License.
203 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # fluent-plugin-prometheus, a plugin for [Fluentd](https://www.fluentd.org)
  2 | 
  3 | [![Build Status](https://travis-ci.org/fluent/fluent-plugin-prometheus.svg?branch=master)](https://travis-ci.org/fluent/fluent-plugin-prometheus)
  4 | 
  5 | A fluent plugin that instruments metrics from records and exposes them via web interface. Intended to be used together with a [Prometheus server](https://github.com/prometheus/prometheus).
  6 | 
  7 | ## Requirements
  8 | 
  9 | | fluent-plugin-prometheus | fluentd    | ruby   |
 10 | |--------------------------|------------|--------|
 11 | | 1.x.y                    | >= v1.9.1  | >= 2.4 |
 12 | | 1.[0-7].y                | >= v0.14.8 | >= 2.1 |
 13 | | 0.x.y                    | >= v0.12.0 | >= 1.9 |
 14 | 
 15 | Since v1.8.0, fluent-plugin-prometheus uses [http_server helper](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-http_server) to launch HTTP server.
 16 | If you want to handle lots of connections, install `async-http` gem.
 17 | 
 18 | ## Installation
 19 | 
 20 | Add this line to your application's Gemfile:
 21 | 
 22 | ```ruby
 23 | gem 'fluent-plugin-prometheus'
 24 | ```
 25 | 
 26 | And then execute:
 27 | 
 28 |     $ bundle
 29 | 
 30 | Or install it yourself as:
 31 | 
 32 |     $ gem install fluent-plugin-prometheus
 33 | 
 34 | ## Usage
 35 | 
 36 | fluentd-plugin-prometheus includes 6 plugins.
 37 | 
 38 | - `prometheus` input plugin
 39 | - `prometheus_monitor` input plugin
 40 | - `prometheus_output_monitor` input plugin
 41 | - `prometheus_tail_monitor` input plugin
 42 | - `prometheus` output plugin
 43 | - `prometheus` filter plugin
 44 | 
 45 | See [sample configuration](./misc/fluentd_sample.conf), or try [tutorial](#try-plugin-with-nginx).
 46 | 
 47 | ### prometheus input plugin
 48 | 
 49 | You have to configure this plugin to expose metrics collected by other Prometheus plugins.
 50 | This plugin provides a metrics HTTP endpoint to be scraped by a Prometheus server on 24231/tcp(default).
 51 | 
 52 | With following configuration, you can access http://localhost:24231/metrics on a server where fluentd running.
 53 | 
 54 | ```
 55 | <source>
 56 |   @type prometheus
 57 | </source>
 58 | ```
 59 | 
 60 | More configuration parameters:
 61 | 
 62 | - `bind`: binding interface (default: '0.0.0.0')
 63 | - `port`: listen port (default: 24231)
 64 | - `metrics_path`: metrics HTTP endpoint (default: /metrics)
 65 | - `aggregated_metrics_path`: metrics HTTP endpoint (default: /aggregated_metrics)
 66 | - `content_encoding`: encoding format for the exposed metrics (default: identity). Supported formats are {identity, gzip}
 67 | 
 68 | When using multiple workers, each worker binds to port + `fluent_worker_id`.
 69 | To scrape metrics from all workers at once, you can access http://localhost:24231/aggregated_metrics.
 70 | 
 71 | #### TLS setting
 72 | 
 73 | Use `<trasnport tls>`. See [transport config article](https://docs.fluentd.org/configuration/transport-section) for more details.
 74 | 
 75 | ```
 76 | <source>
 77 |   @type prometheus
 78 |   <transport tls>
 79 |     # TLS parameters...
 80 |   </transport
 81 | </source>
 82 | ```
 83 | 
 84 | ### prometheus_monitor input plugin
 85 | 
 86 | This plugin collects internal metrics in Fluentd. The metrics are similar to/part of [monitor_agent](https://docs.fluentd.org/input/monitor_agent).
 87 | 
 88 | 
 89 | #### Exposed metrics
 90 | 
 91 | - `fluentd_status_buffer_queue_length`
 92 | - `fluentd_status_buffer_total_bytes`
 93 | - `fluentd_status_retry_count`
 94 | - `fluentd_status_buffer_newest_timekey` from fluentd v1.4.2
 95 | - `fluentd_status_buffer_oldest_timekey` from fluentd v1.4.2
 96 | 
 97 | #### Configuration
 98 | 
 99 | With following configuration, those metrics are collected.
100 | 
101 | ```
102 | <source>
103 |   @type prometheus_monitor
104 | </source>
105 | ```
106 | 
107 | More configuration parameters:
108 | 
109 | - `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
110 | - `interval`: interval to update monitor_agent information in seconds (default: 5)
111 | 
112 | ### prometheus_output_monitor input plugin
113 | 
114 | This plugin collects internal metrics for output plugin in Fluentd. This is similar to `prometheus_monitor` plugin, but specialized for output plugin. There are Many metrics `prometheus_monitor` does not include, such as `num_errors`, `retry_wait` and so on.
115 | 
116 | #### Exposed metrics
117 | 
118 | Metrics for output
119 | 
120 | - `fluentd_output_status_retry_count`
121 | - `fluentd_output_status_num_errors`
122 | - `fluentd_output_status_emit_count`
123 | - `fluentd_output_status_retry_wait`
124 |     - current retry_wait computed from last retry time and next retry time
125 | - `fluentd_output_status_emit_records`
126 | - `fluentd_output_status_write_count`
127 | - `fluentd_output_status_rollback_count`
128 | - `fluentd_output_status_flush_time_count` in milliseconds from fluentd v1.6.0
129 | - `fluentd_output_status_slow_flush_count` from fluentd v1.6.0
130 | 
131 | Metrics for buffer
132 | 
133 | - `fluentd_output_status_buffer_total_bytes`
134 | - `fluentd_output_status_buffer_stage_length` from fluentd v1.6.0
135 | - `fluentd_output_status_buffer_stage_byte_size` from fluentd v1.6.0
136 | - `fluentd_output_status_buffer_queue_length`
137 | - `fluentd_output_status_buffer_queue_byte_size` from fluentd v1.6.0
138 | - `fluentd_output_status_buffer_newest_timekey` from fluentd v1.6.0
139 | - `fluentd_output_status_buffer_oldest_timekey` from fluentd v1.6.0
140 | - `fluentd_output_status_buffer_available_space_ratio` from fluentd v1.6.0
141 | 
142 | #### Configuration
143 | 
144 | With following configuration, those metrics are collected.
145 | 
146 | ```
147 | <source>
148 |   @type prometheus_output_monitor
149 | </source>
150 | ```
151 | 
152 | More configuration parameters:
153 | 
154 | - `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
155 | - `interval`: interval to update monitor_agent information in seconds (default: 5)
156 | - `gauge_all`: Specify metric type. If `true`, use `gauge` type. If `false`, use `counter` type. Since v2, this parameter will be removed and use `counter` type.
157 | 
158 | ### prometheus_tail_monitor input plugin
159 | 
160 | This plugin collects internal metrics for in_tail plugin in Fluentd. in_tail plugin holds internal state for files that the plugin is watching. The state is sometimes important to monitor plugins work correctly.
161 | 
162 | This plugin uses internal class of Fluentd, so it's easy to break.
163 | 
164 | #### Exposed metrics
165 | 
166 | - `fluentd_tail_file_position`: Current bytes which plugin reads from the file
167 | - `fluentd_tail_file_inode`: inode of the file
168 | - `fluentd_tail_file_closed`: Number of closed files
169 | - `fluentd_tail_file_opened`: Number of opened files
170 | - `fluentd_tail_file_rotated`: Number of rotated files
171 | - `fluentd_tail_file_throttled`: Number of times files got throttled (only with fluentd version > 1.17)
172 | 
173 | Default labels:
174 | 
175 | - `plugin_id`: a value set for a plugin in configuration.
176 | - `type`: plugin name. `in_tail` only for now.
177 | - `path`: file path
178 | 
179 | #### Configuration
180 | 
181 | With following configuration, those metrics are collected.
182 | 
183 | ```
184 | <source>
185 |   @type prometheus_tail_monitor
186 | </source>
187 | ```
188 | 
189 | More configuration parameters:
190 | 
191 | - `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
192 | - `interval`: interval to update monitor_agent information in seconds (default: 5)
193 | 
194 | ### prometheus output/filter plugin
195 | 
196 | Both output/filter plugins instrument metrics from records. Both plugins have no impact against values of each records, just read.
197 | 
198 | Assuming you have following configuration and receiving message,
199 | 
200 | ```
201 | <match message>
202 |   @type stdout
203 | </match>
204 | ```
205 | 
206 | ```
207 | message {
208 |   "foo": 100,
209 |   "bar": 200,
210 |   "baz": 300
211 | }
212 | ```
213 | 
214 | In filter plugin style,
215 | 
216 | ```
217 | <filter message>
218 |   @type prometheus
219 |   <metric>
220 |     name message_foo_counter
221 |     type counter
222 |     desc The total number of foo in message.
223 |     key foo
224 |   </metric>
225 | </filter>
226 | 
227 | <match message>
228 |   @type stdout
229 | </match>
230 | ```
231 | 
232 | In output plugin style:
233 | 
234 | ```
235 | <filter message>
236 |   @type prometheus
237 |   <metric>
238 |     name message_foo_counter
239 |     type counter
240 |     desc The total number of foo in message.
241 |     key foo
242 |   </metric>
243 | </filter>
244 | 
245 | <match message>
246 |   @type copy
247 |   <store>
248 |     @type prometheus
249 |     <metric>
250 |       name message_foo_counter
251 |       type counter
252 |       desc The total number of foo in message.
253 |       key foo
254 |     </metric>
255 |   </store>
256 |   <store>
257 |     @type stdout
258 |   </store>
259 | </match>
260 | ```
261 | 
262 | With above configuration, the plugin collects a metric named `message_foo_counter` from key `foo` of each records.
263 | 
264 | You can access nested keys in records via dot or bracket notation (https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor#syntax), for example: `$.kubernetes.namespace`, `$['key1'][0]['key2']`. The record accessor is enable only if the value starts with `$.` or `$[`.
265 | 
266 | See Supported Metric Type and Labels for more configuration parameters.
267 | 
268 | ## Supported Metric Types
269 | 
270 | For details of each metric type, see [Prometheus documentation](http://prometheus.io/docs/concepts/metric_types/). Also see [metric name guide](http://prometheus.io/docs/practices/naming/).
271 | 
272 | ### counter type
273 | 
274 | ```
275 | <metric>
276 |   name message_foo_counter
277 |   type counter
278 |   desc The total number of foo in message.
279 |   key foo
280 |   <labels>
281 |     tag ${tag}
282 |     host ${hostname}
283 |     foo bar
284 |   </labels>
285 | </metric>
286 | ```
287 | 
288 | - `name`: metric name (required)
289 | - `type`: metric type (required)
290 | - `desc`: description of this metric (required)
291 | - `key`: key name of record for instrumentation (**optional**)
292 | - `initialized`: boolean controlling initilization of metric (**optional**). See [Metric initialization](#metric-initialization)
293 | - `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
294 | - `<initlabels>`: labels to use for initialization of ReccordAccessors/Placeholder labels (**optional**). See [Metric initialization](#metric-initialization)
295 | 
296 | If key is empty, the metric values is treated as 1, so the counter increments by 1 on each record regardless of contents of the record.
297 | 
298 | ### gauge type
299 | 
300 | ```
301 | <metric>
302 |   name message_foo_gauge
303 |   type gauge
304 |   desc The total number of foo in message.
305 |   key foo
306 |   <labels>
307 |     tag ${tag}
308 |     host ${hostname}
309 |     foo bar
310 |   </labels>
311 | </metric>
312 | ```
313 | 
314 | - `name`: metric name (required)
315 | - `type`: metric type (required)
316 | - `desc`: description of metric (required)
317 | - `key`: key name of record for instrumentation (required)
318 | - `initialized`: boolean controlling initilization of metric (**optional**). See [Metric initialization](#metric-initialization)
319 | - `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
320 | - `<initlabels>`: labels to use for initialization of ReccordAccessors/Placeholder labels (**optional**). See [Metric initialization](#metric-initialization)
321 | 
322 | ### summary type
323 | 
324 | ```
325 | <metric>
326 |   name message_foo
327 |   type summary
328 |   desc The summary of foo in message.
329 |   key foo
330 |   <labels>
331 |     tag ${tag}
332 |     host ${hostname}
333 |     foo bar
334 |   </labels>
335 | </metric>
336 | ```
337 | 
338 | - `name`: metric name (required)
339 | - `type`: metric type (required)
340 | - `desc`: description of metric (required)
341 | - `key`: key name of record for instrumentation (required)
342 | - `initialized`: boolean controlling initilization of metric (**optional**). See [Metric initialization](#metric-initialization)
343 | - `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
344 | - `<initlabels>`: labels to use for initialization of ReccordAccessors/Placeholder labels (**optional**). See [Metric initialization](#metric-initialization)
345 | 
346 | ### histogram type
347 | 
348 | ```
349 | <metric>
350 |   name message_foo
351 |   type histogram
352 |   desc The histogram of foo in message.
353 |   key foo
354 |   buckets 0.1, 1, 5, 10
355 |   <labels>
356 |     tag ${tag}
357 |     host ${hostname}
358 |     foo bar
359 |   </labels>
360 | </metric>
361 | ```
362 | 
363 | - `name`: metric name (required)
364 | - `type`: metric type (required)
365 | - `desc`: description of metric (required)
366 | - `key`: key name of record for instrumentation (required)
367 | - `initialized`: boolean controlling initilization of metric (**optional**). See [Metric initialization](#metric-initialization)
368 | - `buckets`: buckets of record for instrumentation (optional)
369 | - `<labels>`: additional labels for this metric (optional). See [Labels](#labels)
370 | - `<initlabels>`: labels to use for initialization of ReccordAccessors/Placeholder labels (**optional**). See [Metric initialization](#metric-initialization)
371 | 
372 | ## Labels
373 | 
374 | See [Prometheus Data Model](http://prometheus.io/docs/concepts/data_model/) first.
375 | 
376 | You can add labels with static value or dynamic value from records. In `prometheus_monitor` input plugin, you can't use label value from records.
377 | 
378 | ### labels section
379 | 
380 | ```
381 | <labels>
382 |   key1 value1
383 |   key2 value2
384 | </labels>
385 | ```
386 | 
387 | All labels sections has same format. Each lines have key/value for label.
388 | 
389 | You can access nested fields in records via dot or bracket notation (https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor#syntax), for example: `$.kubernetes.namespace`, `$['key1'][0]['key2']`. The record accessor is enable only if the value starts with `$.` or `$[`. Other values are handled as raw string as is and may be expanded by placeholder described later.
390 | 
391 | You can use placeholder for label values. The placeholders will be expanded from reserved values and records.
392 | If you specify `${hostname}`, it will be expanded by value of a hostname where fluentd runs.
393 | The placeholder for records is deprecated. Use record accessor syntax instead.
394 | 
395 | Reserved placeholders are:
396 | 
397 | - `${hostname}`: hostname
398 | - `${worker_id}`: fluent worker id
399 | - `${tag}`: tag name
400 |   - only available in Prometheus output/filter plugin
401 | - `${tag_parts[N]}` refers to the Nth part of the tag.
402 |   - only available in Prometheus output/filter plugin
403 | - `${tag_prefix[N]}` refers to the [0..N] part of the tag.
404 |   - only available in Prometheus output/filter plugin
405 | - `${tag_suffix[N]}` refers to the [`tagsize`-1-N..] part of the tag.
406 |   - where `tagsize` is the size of tag which is splitted with `.` (when tag is `1.2.3`, then `tagsize` is 3)
407 |   - only available in Prometheus output/filter plugin
408 | 
409 | ### Metric initialization
410 | 
411 | You can configure if a metric should be initialized to its zero value before receiving any event. To do so you just need to specify `initialized true`.
412 | 
413 | ```
414 | <metric>
415 |   name message_bar_counter
416 |   type counter
417 |   desc The total number of bar in message.
418 |   key bar
419 |   initialized true
420 |   <labels>
421 |     foo bar
422 |   </labels>
423 | </metric>
424 | ```
425 | 
426 | If your labels contains ReccordAccessors or Placeholders, you must use `<initlabels>` to specify the values your ReccordAccessors/Placeholders will take. This feature is useful only if your Placeholders/ReccordAccessors contain deterministic values. Initialization will create as many zero value metrics as `<initlabels>` blocks you defined.
427 | Potential reserved placeholders `${hostname}` and `${worker_id}`, as well as static labels, are automatically added and should not be specified in `<initlabels>` configuration.
428 | 
429 | ```
430 | <metric>
431 |   name message_bar_counter
432 |   type counter
433 |   desc The total number of bar in message.
434 |   key bar
435 |   initialized true
436 |   <labels>
437 |     key $.foo
438 |     tag ${tag}
439 |     foo bar
440 |     worker_id ${worker_id}
441 |   </labels>
442 |   <initlabels>
443 |     key foo1
444 |     tag tag1
445 |   </initlabels>
446 |   <initlabels>
447 |     key foo2
448 |     tag tag2
449 |   </initlabels>
450 | </metric>
451 | <labels>
452 |   hostname ${hostname}
453 | </labels>
454 | ```
455 | 
456 | ### top-level labels and labels inside metric
457 | 
458 | Prometheus output/filter plugin can have multiple metric section. Top-level labels section specifies labels for all metrics. Labels section inside metric section specifies labels for the metric. Both are specified, labels are merged.
459 | 
460 | ```
461 | <filter message>
462 |   @type prometheus
463 |   <metric>
464 |     name message_foo_counter
465 |     type counter
466 |     desc The total number of foo in message.
467 |     key foo
468 |     <labels>
469 |       key foo
470 |       data_type ${type}
471 |     </labels>
472 |   </metric>
473 |   <metric>
474 |     name message_bar_counter
475 |     type counter
476 |     desc The total number of bar in message.
477 |     key bar
478 |     <labels>
479 |       key bar
480 |     </labels>
481 |   </metric>
482 |   <labels>
483 |     tag ${tag}
484 |     hostname ${hostname}
485 |   </labels>
486 | </filter>
487 | ```
488 | 
489 | In this case, `message_foo_counter` has `tag`, `hostname`, `key` and `data_type` labels.
490 | 
491 | 
492 | ## Try plugin with nginx
493 | 
494 | Checkout repository and setup.
495 | 
496 | ```
497 | $ git clone git://github.com/fluent/fluent-plugin-prometheus.git
498 | $ cd fluent-plugin-prometheus
499 | $ bundle install --path vendor/bundle
500 | ```
501 | 
502 | Download pre-compiled Prometheus binary and start it. It listens on 9090.
503 | 
504 | ```
505 | $ wget https://github.com/prometheus/prometheus/releases/download/v1.5.2/prometheus-1.5.2.linux-amd64.tar.gz -O - | tar zxf -
506 | $ ./prometheus-1.5.2.linux-amd64/prometheus -config.file=./misc/prometheus.yaml -storage.local.path=./prometheus/metrics
507 | ```
508 | 
509 | Install Nginx for sample metrics. It listens on 80 and 9999.
510 | 
511 | ```
512 | $ sudo apt-get install -y nginx
513 | $ sudo cp misc/nginx_proxy.conf /etc/nginx/sites-enabled/proxy
514 | $ sudo chmod 777 /var/log/nginx && sudo chmod +r /var/log/nginx/*.log
515 | $ sudo service nginx restart
516 | ```
517 | 
518 | Start fluentd with sample configuration. It listens on 24231.
519 | 
520 | ```
521 | $ bundle exec fluentd -c misc/fluentd_sample.conf -v
522 | ```
523 | 
524 | Generate some records by accessing nginx.
525 | 
526 | ```
527 | $ curl http://localhost/
528 | $ curl http://localhost:9999/
529 | ```
530 | 
531 | Confirm that some metrics are exported via Fluentd.
532 | 
533 | ```
534 | $ curl http://localhost:24231/metrics
535 | ```
536 | 
537 | Then, make a graph on Prometheus UI. http://localhost:9090/
538 | 
539 | ## Contributing
540 | 
541 | 1. Fork it ( https://github.com/fluent/fluent-plugin-prometheus/fork )
542 | 2. Create your feature branch (`git checkout -b my-new-feature`)
543 | 3. Commit your changes (`git commit -am 'Add some feature'`)
544 | 4. Push to the branch (`git push origin my-new-feature`)
545 | 5. Create a new Pull Request
546 | 
547 | 
548 | ## Copyright
549 | 
550 | <table>
551 |   <tr>
552 |     <td>Author</td><td>Masahiro Sano <sabottenda@gmail.com></td>
553 |   </tr>
554 |   <tr>
555 |     <td>Copyright</td><td>Copyright (c) 2015- Masahiro Sano</td>
556 |   </tr>
557 |   <tr>
558 |     <td>License</td><td>Apache License, Version 2.0</td>
559 |   </tr>
560 | </table>
561 | 


--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
1 | require "bundler/gem_tasks"
2 | require "rspec/core/rake_task"
3 | 
4 | RSpec::Core::RakeTask.new(:spec)
5 | 
6 | task :default => :spec
7 | 
8 | 


--------------------------------------------------------------------------------
/fluent-plugin-prometheus.gemspec:
--------------------------------------------------------------------------------
 1 | Gem::Specification.new do |spec|
 2 |   spec.name          = "fluent-plugin-prometheus"
 3 |   spec.version       = "2.2.1"
 4 |   spec.authors       = ["Masahiro Sano"]
 5 |   spec.email         = ["sabottenda@gmail.com"]
 6 |   spec.summary       = %q{A fluent plugin that collects metrics and exposes for Prometheus.}
 7 |   spec.description   = %q{A fluent plugin that collects metrics and exposes for Prometheus.}
 8 |   spec.homepage      = "https://github.com/fluent/fluent-plugin-prometheus"
 9 |   spec.license       = "Apache-2.0"
10 | 
11 |   spec.files         = `git ls-files -z`.split("\x0")
12 |   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
13 |   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
14 |   spec.require_paths = ["lib"]
15 | 
16 |   spec.add_dependency "fluentd", ">= 1.9.1", "< 2"
17 |   spec.add_dependency "prometheus-client", ">= 2.1.0"
18 |   spec.add_development_dependency "bundler"
19 |   spec.add_development_dependency "rake"
20 |   spec.add_development_dependency "rspec"
21 |   spec.add_development_dependency "test-unit"
22 | end
23 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/filter_prometheus.rb:
--------------------------------------------------------------------------------
 1 | require 'fluent/plugin/prometheus'
 2 | require 'fluent/plugin/filter'
 3 | 
 4 | module Fluent::Plugin
 5 |   class PrometheusFilter < Fluent::Plugin::Filter
 6 |     Fluent::Plugin.register_filter('prometheus', self)
 7 |     include Fluent::Plugin::PrometheusLabelParser
 8 |     include Fluent::Plugin::Prometheus
 9 | 
10 |     def initialize
11 |       super
12 |       @registry = ::Prometheus::Client.registry
13 |     end
14 | 
15 |     def multi_workers_ready?
16 |       true
17 |     end
18 | 
19 |     def configure(conf)
20 |       super
21 |       labels = parse_labels_elements(conf)
22 |       @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
23 |     end
24 | 
25 |     def filter(tag, time, record)
26 |       instrument_single(tag, time, record, @metrics)
27 |       record
28 |     end
29 |   end
30 | end
31 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/in_prometheus.rb:
--------------------------------------------------------------------------------
  1 | require 'fluent/plugin/input'
  2 | require 'fluent/plugin/prometheus'
  3 | require 'fluent/plugin/prometheus_metrics'
  4 | require 'net/http'
  5 | require 'openssl'
  6 | require 'zlib'
  7 | 
  8 | module Fluent::Plugin
  9 |   class PrometheusInput < Fluent::Plugin::Input
 10 |     Fluent::Plugin.register_input('prometheus', self)
 11 | 
 12 |     helpers :thread, :http_server
 13 | 
 14 |     config_param :bind, :string, default: '0.0.0.0'
 15 |     config_param :port, :integer, default: 24231
 16 |     config_param :metrics_path, :string, default: '/metrics'
 17 |     config_param :aggregated_metrics_path, :string, default: '/aggregated_metrics'
 18 | 
 19 |     desc 'Enable ssl configuration for the server'
 20 |     config_section :ssl, required: false, multi: false do
 21 |       config_param :enable, :bool, default: false, deprecated: 'Use <transport tls> section'
 22 | 
 23 |       desc 'Path to the ssl certificate in PEM format.  Read from file and added to conf as "SSLCertificate"'
 24 |       config_param :certificate_path, :string, default: nil, deprecated: 'Use cert_path in <transport tls> section'
 25 | 
 26 |       desc 'Path to the ssl private key in PEM format.  Read from file and added to conf as "SSLPrivateKey"'
 27 |       config_param :private_key_path, :string, default: nil, deprecated: 'Use private_key_path in <transport tls> section'
 28 | 
 29 |       desc 'Path to CA in PEM format.  Read from file and added to conf as "SSLCACertificateFile"'
 30 |       config_param :ca_path, :string, default: nil, deprecated: 'Use ca_path in <transport tls> section'
 31 | 
 32 |       desc 'Additional ssl conf for the server.  Ref: https://github.com/ruby/webrick/blob/master/lib/webrick/ssl.rb'
 33 |       config_param :extra_conf, :hash, default: nil, symbolize_keys: true, deprecated: 'See http helper config'
 34 |     end
 35 | 
 36 |     desc 'Content encoding of the exposed metrics, Currently supported encoding is identity, gzip. Ref: https://prometheus.io/docs/instrumenting/exposition_formats/#basic-info'
 37 |     config_param :content_encoding, :enum, list: [:identity, :gzip], default: :identity
 38 | 
 39 |     def initialize
 40 |       super
 41 |       @registry = ::Prometheus::Client.registry
 42 |       @secure = nil
 43 |     end
 44 | 
 45 |     def configure(conf)
 46 |       super
 47 | 
 48 |       # Get how many workers we have
 49 |       sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
 50 |                   owner.system_config
 51 |                 elsif self.respond_to?(:system_config)
 52 |                   self.system_config
 53 |                 else
 54 |                   nil
 55 |                 end
 56 |       @num_workers = sysconf && sysconf.workers ? sysconf.workers : 1
 57 |       @secure = @transport_config.protocol == :tls || (@ssl && @ssl['enable'])
 58 | 
 59 |       @base_port = @port
 60 |       @port += fluentd_worker_id
 61 |     end
 62 | 
 63 |     def multi_workers_ready?
 64 |       true
 65 |     end
 66 | 
 67 |     def start
 68 |       super
 69 | 
 70 |       scheme = @secure ? 'https' : 'http'
 71 |       log.debug "listening prometheus http server on #{scheme}:://#{@bind}:#{@port}/#{@metrics_path} for worker#{fluentd_worker_id}"
 72 | 
 73 |       proto = @secure ? :tls : :tcp
 74 | 
 75 |       if @ssl && @ssl['enable'] && @ssl['extra_conf']
 76 |         start_webrick
 77 |         return
 78 |       end
 79 | 
 80 |       begin
 81 |         require 'async'
 82 |         require 'fluent/plugin/in_prometheus/async_wrapper'
 83 |         extend AsyncWrapper
 84 |       rescue LoadError => _
 85 |         # ignore
 86 |       end
 87 | 
 88 |       tls_opt = if @ssl && @ssl['enable']
 89 |                   ssl_config = {}
 90 | 
 91 |                   if (@ssl['certificate_path'] && @ssl['private_key_path'].nil?) || (@ssl['certificate_path'].nil? && @ssl['private_key_path'])
 92 |                     raise Fluent::ConfigError.new('both certificate_path and private_key_path must be defined')
 93 |                   end
 94 | 
 95 |                   if @ssl['certificate_path']
 96 |                     ssl_config['cert_path'] = @ssl['certificate_path']
 97 |                   end
 98 | 
 99 |                   if @ssl['private_key_path']
100 |                     ssl_config['private_key_path'] = @ssl['private_key_path']
101 |                   end
102 | 
103 |                   if @ssl['ca_path']
104 |                     ssl_config['ca_path'] = @ssl['ca_path']
105 |                     # Only ca_path is insecure in fluentd
106 |                     # https://github.com/fluent/fluentd/blob/2236ad45197ba336fd9faf56f442252c8b226f25/lib/fluent/plugin_helper/cert_option.rb#L68
107 |                     ssl_config['insecure'] = true
108 |                   end
109 | 
110 |                   ssl_config
111 |                 end
112 | 
113 |       http_server_create_http_server(:in_prometheus_server, addr: @bind, port: @port, logger: log, proto: proto, tls_opts: tls_opt) do |server|
114 |         server.get(@metrics_path) { |_req| all_metrics }
115 |         server.get(@aggregated_metrics_path) { |_req| all_workers_metrics }
116 |       end
117 |     end
118 | 
119 |     def shutdown
120 |       if @webrick_server
121 |         @webrick_server.shutdown
122 |         @webrick_server = nil
123 |       end
124 |       super
125 |     end
126 | 
127 |     private
128 | 
129 |     # For compatiblity because http helper can't support extra_conf option
130 |     def start_webrick
131 |       require 'webrick/https'
132 |       require 'webrick'
133 | 
134 |       config = {
135 |         BindAddress: @bind,
136 |         Port: @port,
137 |         MaxClients: 5,
138 |         Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
139 |         AccessLog: [],
140 |       }
141 |       if (@ssl['certificate_path'] && @ssl['private_key_path'].nil?) || (@ssl['certificate_path'].nil? && @ssl['private_key_path'])
142 |         raise RuntimeError.new("certificate_path and private_key_path most both be defined")
143 |       end
144 | 
145 |       ssl_config = {
146 |         SSLEnable: true,
147 |         SSLCertName: [['CN', 'nobody'], ['DC', 'example']]
148 |       }
149 | 
150 |       if @ssl['certificate_path']
151 |         cert = OpenSSL::X509::Certificate.new(File.read(@ssl['certificate_path']))
152 |         ssl_config[:SSLCertificate] = cert
153 |       end
154 | 
155 |       if @ssl['private_key_path']
156 |         key = OpenSSL::PKey.read(@ssl['private_key_path'])
157 |         ssl_config[:SSLPrivateKey] = key
158 |       end
159 | 
160 |       ssl_config[:SSLCACertificateFile] = @ssl['ca_path'] if @ssl['ca_path']
161 |       ssl_config = ssl_config.merge(@ssl['extra_conf']) if @ssl['extra_conf']
162 |       config = ssl_config.merge(config)
163 | 
164 |       @log.on_debug do
165 |         @log.debug("WEBrick conf: #{config}")
166 |       end
167 | 
168 |       @webrick_server = WEBrick::HTTPServer.new(config)
169 |       @webrick_server.mount_proc(@metrics_path) do |_req, res|
170 |         status, header, body = all_metrics
171 |         res.status = status
172 |         res['Content-Type'] = header['Content-Type']
173 |         res.body = body
174 |         res
175 |       end
176 | 
177 |       @webrick_server.mount_proc(@aggregated_metrics_path) do |_req, res|
178 |         status, header, body = all_workers_metrics
179 |         res.status = status
180 |         res['Content-Type'] = header['Content-Type']
181 |         res.body = body
182 |         res
183 |       end
184 | 
185 |       thread_create(:in_prometheus_webrick) do
186 |         @webrick_server.start
187 |       end
188 |     end
189 | 
190 |     def all_metrics
191 |       response(::Prometheus::Client::Formats::Text.marshal(@registry))
192 |     rescue => e
193 |       [500, { 'Content-Type' => 'text/plain' }, e.to_s]
194 |     end
195 | 
196 |     def all_workers_metrics
197 |       full_result = PromMetricsAggregator.new
198 | 
199 |       send_request_to_each_worker do |resp|
200 |         if resp.code.to_s == '200'
201 |           full_result.add_metrics(resp.body)
202 |         end
203 |       end
204 |       response(full_result.get_metrics)
205 |     rescue => e
206 |       [500, { 'Content-Type' => 'text/plain' }, e.to_s]
207 |     end
208 | 
209 |     def send_request_to_each_worker
210 |       bind = (@bind == '0.0.0.0') ? '127.0.0.1' : @bind
211 |       [*(@base_port...(@base_port + @num_workers))].each do |worker_port|
212 |         do_request(host: bind, port: worker_port, secure: @secure) do |http|
213 |           yield(http.get(@metrics_path))
214 |         end
215 |       end
216 |     end
217 | 
218 |     # might be replaced by AsyncWrapper if async gem is installed
219 |     def do_request(host:, port:, secure:)
220 |       http = Net::HTTP.new(host, port)
221 | 
222 |       if secure
223 |         http.use_ssl = true
224 |         # target is our child process. so it's secure.
225 |         http.verify_mode = OpenSSL::SSL::VERIFY_NONE
226 |       end
227 | 
228 |       http.start do
229 |         yield(http)
230 |       end
231 |     end
232 | 
233 |     def response(metrics)
234 |       body = nil
235 |       case @content_encoding
236 |       when :gzip
237 |         gzip = Zlib::GzipWriter.new(StringIO.new)
238 |         gzip << metrics
239 |         body = gzip.close.string
240 |       when :identity
241 |         body = metrics
242 |       end
243 |       [200, { 'Content-Type' => ::Prometheus::Client::Formats::Text::CONTENT_TYPE, 'Content-Encoding' => @content_encoding.to_s }, body]
244 |     end
245 |   end
246 | end
247 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/in_prometheus/async_wrapper.rb:
--------------------------------------------------------------------------------
 1 | require 'async'
 2 | 
 3 | module Fluent::Plugin
 4 |   class PrometheusInput
 5 |     module AsyncWrapper
 6 |       def do_request(host:, port:, secure:)
 7 |         endpoint =
 8 |           if secure
 9 |             context = OpenSSL::SSL::SSLContext.new
10 |             context.verify_mode = OpenSSL::SSL::VERIFY_NONE
11 |             Async::HTTP::Endpoint.parse("https://#{host}:#{port}", ssl_context: context)
12 |           else
13 |             Async::HTTP::Endpoint.parse("http://#{host}:#{port}")
14 |           end
15 | 
16 |         Async::HTTP::Client.open(endpoint) do |client|
17 |           yield(AsyncHttpWrapper.new(client))
18 |         end
19 |       end
20 | 
21 |       Response = Struct.new(:code, :body, :headers)
22 | 
23 |       class AsyncHttpWrapper
24 |         def initialize(http)
25 |           @http = http
26 |         end
27 | 
28 |         def get(path)
29 |           error = nil
30 |           response = Async::Task.current.async {
31 |             begin
32 |               @http.get(path)
33 |             rescue => e               # Async::Reactor rescue all error. handle it by itself
34 |               error = e
35 |             end
36 |           }.wait
37 | 
38 |           if error
39 |             raise error
40 |           end
41 | 
42 |           Response.new(response.status.to_s, response.read || '', response.headers)
43 |         end
44 |       end
45 |     end
46 |   end
47 | end
48 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/in_prometheus_monitor.rb:
--------------------------------------------------------------------------------
  1 | require 'fluent/plugin/input'
  2 | require 'fluent/plugin/in_monitor_agent'
  3 | require 'fluent/plugin/prometheus'
  4 | 
  5 | module Fluent::Plugin
  6 |   class PrometheusMonitorInput < Fluent::Plugin::Input
  7 |     Fluent::Plugin.register_input('prometheus_monitor', self)
  8 |     include Fluent::Plugin::PrometheusLabelParser
  9 | 
 10 |     helpers :timer
 11 | 
 12 |     config_param :interval, :time, default: 5
 13 |     attr_reader :registry
 14 | 
 15 |     def initialize
 16 |       super
 17 |       @registry = ::Prometheus::Client.registry
 18 |     end
 19 | 
 20 |     def multi_workers_ready?
 21 |       true
 22 |     end
 23 | 
 24 |     def configure(conf)
 25 |       super
 26 |       hostname = Socket.gethostname
 27 |       expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
 28 |       expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
 29 |       @base_labels = parse_labels_elements(conf)
 30 |       @base_labels.each do |key, value|
 31 |         unless value.is_a?(String)
 32 |           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_monitor"
 33 |         end
 34 |         @base_labels[key] = expander.expand(value)
 35 |       end
 36 | 
 37 |       if defined?(Fluent::Plugin) && defined?(Fluent::Plugin::MonitorAgentInput)
 38 |         # from v0.14.6
 39 |         @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
 40 |       else
 41 |         @monitor_agent = Fluent::MonitorAgentInput.new
 42 |       end
 43 | 
 44 |     end
 45 | 
 46 |     def start
 47 |       super
 48 | 
 49 |       @buffer_newest_timekey = get_gauge(
 50 |         :fluentd_status_buffer_newest_timekey,
 51 |         'Newest timekey in buffer.')
 52 |       @buffer_oldest_timekey = get_gauge(
 53 |         :fluentd_status_buffer_oldest_timekey,
 54 |         'Oldest timekey in buffer.')
 55 |       buffer_queue_length = get_gauge(
 56 |         :fluentd_status_buffer_queue_length,
 57 |         'Current buffer queue length.')
 58 |       buffer_total_queued_size = get_gauge(
 59 |         :fluentd_status_buffer_total_bytes,
 60 |         'Current total size of queued buffers.')
 61 |       retry_counts = get_gauge(
 62 |         :fluentd_status_retry_count,
 63 |         'Current retry counts.')
 64 | 
 65 |       @monitor_info = {
 66 |         'buffer_queue_length' => buffer_queue_length,
 67 |         'buffer_total_queued_size' => buffer_total_queued_size,
 68 |         'retry_count' => retry_counts,
 69 |       }
 70 |       timer_execute(:in_prometheus_monitor, @interval, &method(:update_monitor_info))
 71 |     end
 72 | 
 73 |     def update_monitor_info
 74 |       @monitor_agent.plugins_info_all.each do |info|
 75 |         label = labels(info)
 76 | 
 77 |         @monitor_info.each do |name, metric|
 78 |           if info[name]
 79 |             metric.set(info[name], labels: label)
 80 |           end
 81 |         end
 82 | 
 83 |         timekeys = info["buffer_timekeys"]
 84 |         if timekeys && !timekeys.empty?
 85 |           @buffer_newest_timekey.set(timekeys.max, labels: label)
 86 |           @buffer_oldest_timekey.set(timekeys.min, labels: label)
 87 |         end
 88 |       end
 89 |     end
 90 | 
 91 |     def labels(plugin_info)
 92 |       @base_labels.merge(
 93 |         plugin_id: plugin_info["plugin_id"],
 94 |         plugin_category: plugin_info["plugin_category"],
 95 |         type: plugin_info["type"],
 96 |       )
 97 |     end
 98 | 
 99 |     def get_gauge(name, docstring)
100 |       if @registry.exist?(name)
101 |         @registry.get(name)
102 |       else
103 |         @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :plugin_category, :type])
104 |       end
105 |     end
106 |   end
107 | end
108 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/in_prometheus_output_monitor.rb:
--------------------------------------------------------------------------------
  1 | require 'fluent/plugin/input'
  2 | require 'fluent/plugin/in_monitor_agent'
  3 | require 'fluent/plugin/prometheus'
  4 | 
  5 | module Fluent::Plugin
  6 |   class PrometheusOutputMonitorInput < Fluent::Plugin::Input
  7 |     Fluent::Plugin.register_input('prometheus_output_monitor', self)
  8 |     include Fluent::Plugin::PrometheusLabelParser
  9 | 
 10 |     helpers :timer
 11 | 
 12 |     config_param :interval, :time, default: 5
 13 |     config_param :gauge_all, :bool, default: true
 14 |     attr_reader :registry
 15 | 
 16 |     MONITOR_IVARS = [
 17 |       :retry,
 18 | 
 19 |       :num_errors,
 20 |       :emit_count,
 21 | 
 22 |       # for v0.12
 23 |       :last_retry_time,
 24 | 
 25 |       # from v0.14
 26 |       :emit_records,
 27 |       :write_count,
 28 |       :rollback_count,
 29 | 
 30 |       # from v1.6.0
 31 |       :flush_time_count,
 32 |       :slow_flush_count,
 33 |     ]
 34 | 
 35 |     def initialize
 36 |       super
 37 |       @registry = ::Prometheus::Client.registry
 38 |     end
 39 | 
 40 |     def multi_workers_ready?
 41 |       true
 42 |     end
 43 | 
 44 |     def configure(conf)
 45 |       super
 46 |       hostname = Socket.gethostname
 47 |       expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
 48 |       expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
 49 |       @base_labels = parse_labels_elements(conf)
 50 |       @base_labels.each do |key, value|
 51 |         unless value.is_a?(String)
 52 |           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_output_monitor"
 53 |         end
 54 |         @base_labels[key] = expander.expand(value)
 55 |       end
 56 | 
 57 |       @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
 58 | 
 59 |       @gauge_or_counter = @gauge_all ? :gauge : :counter
 60 |     end
 61 | 
 62 |     def start
 63 |       super
 64 | 
 65 |       @metrics = {
 66 |         # Buffer metrics
 67 |         buffer_total_queued_size: get_gauge(
 68 |           :fluentd_output_status_buffer_total_bytes,
 69 |           'Current total size of stage and queue buffers.'),
 70 |         buffer_stage_length: get_gauge(
 71 |           :fluentd_output_status_buffer_stage_length,
 72 |           'Current length of stage buffers.'),
 73 |         buffer_stage_byte_size: get_gauge(
 74 |           :fluentd_output_status_buffer_stage_byte_size,
 75 |           'Current total size of stage buffers.'),
 76 |         buffer_queue_length: get_gauge(
 77 |           :fluentd_output_status_buffer_queue_length,
 78 |           'Current length of queue buffers.'),
 79 |         buffer_queue_byte_size: get_gauge(
 80 |           :fluentd_output_status_buffer_queue_byte_size,
 81 |           'Current total size of queue buffers.'),
 82 |         buffer_available_buffer_space_ratios: get_gauge(
 83 |           :fluentd_output_status_buffer_available_space_ratio,
 84 |           'Ratio of available space in buffer.'),
 85 |         buffer_newest_timekey: get_gauge(
 86 |           :fluentd_output_status_buffer_newest_timekey,
 87 |           'Newest timekey in buffer.'),
 88 |         buffer_oldest_timekey: get_gauge(
 89 |           :fluentd_output_status_buffer_oldest_timekey,
 90 |           'Oldest timekey in buffer.'),
 91 | 
 92 |         # Output metrics
 93 |         retry_counts: get_gauge_or_counter(
 94 |           :fluentd_output_status_retry_count,
 95 |           'Current retry counts.'),
 96 |         num_errors: get_gauge_or_counter(
 97 |           :fluentd_output_status_num_errors,
 98 |           'Current number of errors.'),
 99 |         emit_count: get_gauge_or_counter(
100 |           :fluentd_output_status_emit_count,
101 |           'Current emit counts.'),
102 |         emit_records: get_gauge_or_counter(
103 |           :fluentd_output_status_emit_records,
104 |           'Current emit records.'),
105 |         write_count: get_gauge_or_counter(
106 |           :fluentd_output_status_write_count,
107 |           'Current write counts.'),
108 |         rollback_count: get_gauge(
109 |           :fluentd_output_status_rollback_count,
110 |           'Current rollback counts.'),
111 |         flush_time_count: get_gauge_or_counter(
112 |           :fluentd_output_status_flush_time_count,
113 |           'Total flush time.'),
114 |         slow_flush_count: get_gauge_or_counter(
115 |           :fluentd_output_status_slow_flush_count,
116 |           'Current slow flush counts.'),
117 |         retry_wait: get_gauge(
118 |           :fluentd_output_status_retry_wait,
119 |           'Current retry wait'),
120 |       }
121 |       timer_execute(:in_prometheus_output_monitor, @interval, &method(:update_monitor_info))
122 |     end
123 | 
124 |     def update_monitor_info
125 |       opts = {
126 |         ivars: MONITOR_IVARS,
127 |         with_retry: true,
128 |       }
129 | 
130 |       agent_info = @monitor_agent.plugins_info_all(opts).select {|info|
131 |         info['plugin_category'] == 'output'.freeze
132 |       }
133 | 
134 |       monitor_info = {
135 |         # buffer metrics
136 |         'buffer_total_queued_size' => [@metrics[:buffer_total_queued_size]],
137 |         'buffer_stage_length' => [@metrics[:buffer_stage_length]],
138 |         'buffer_stage_byte_size' => [@metrics[:buffer_stage_byte_size]],
139 |         'buffer_queue_length' => [@metrics[:buffer_queue_length]],
140 |         'buffer_queue_byte_size' => [@metrics[:buffer_queue_byte_size]],
141 |         'buffer_available_buffer_space_ratios' => [@metrics[:buffer_available_buffer_space_ratios]],
142 |         'buffer_newest_timekey' => [@metrics[:buffer_newest_timekey]],
143 |         'buffer_oldest_timekey' => [@metrics[:buffer_oldest_timekey]],
144 | 
145 |         # output metrics
146 |         'retry_count' => [@metrics[:retry_counts], @metrics[:num_errors]],
147 |         # Needed since Fluentd v1.14 due to metrics extensions.
148 |         'write_count' => [@metrics[:write_count]],
149 |         'emit_count' => [@metrics[:emit_count]],
150 |         'emit_records' => [@metrics[:emit_records]],
151 |         'rollback_count' => [@metrics[:rollback_count]],
152 |         'flush_time_count' => [@metrics[:flush_time_count]],
153 |         'slow_flush_count' => [@metrics[:slow_flush_count]],
154 |       }
155 |       # No needed for Fluentd v1.14 but leave as-is for backward compatibility.
156 |       instance_vars_info = {
157 |         num_errors: @metrics[:num_errors],
158 |         write_count: @metrics[:write_count],
159 |         emit_count: @metrics[:emit_count],
160 |         emit_records: @metrics[:emit_records],
161 |         rollback_count: @metrics[:rollback_count],
162 |         flush_time_count: @metrics[:flush_time_count],
163 |         slow_flush_count: @metrics[:slow_flush_count],
164 |       }
165 | 
166 |       agent_info.each do |info|
167 |         label = labels(info)
168 | 
169 |         monitor_info.each do |name, metrics|
170 |           metrics.each do |metric|
171 |             if info[name]
172 |               if metric.is_a?(::Prometheus::Client::Gauge)
173 |                 metric.set(info[name], labels: label)
174 |               elsif metric.is_a?(::Prometheus::Client::Counter)
175 |                 metric.increment(by: info[name] - metric.get(labels: label), labels: label)
176 |               end
177 |             end
178 |           end
179 |         end
180 | 
181 |         if info['instance_variables']
182 |           instance_vars_info.each do |name, metric|
183 |             if info['instance_variables'][name]
184 |               if metric.is_a?(::Prometheus::Client::Gauge)
185 |                 metric.set(info['instance_variables'][name], labels: label)
186 |               elsif metric.is_a?(::Prometheus::Client::Counter)
187 |                 metric.increment(by: info['instance_variables'][name] - metric.get(labels: label), labels: label)
188 |               end
189 |             end
190 |           end
191 |         end
192 | 
193 |         # compute current retry_wait
194 |         if info['retry']
195 |           next_time = info['retry']['next_time']
196 |           start_time = info['retry']['start']
197 |           if start_time.nil? && info['instance_variables']
198 |             # v0.12 does not include start, use last_retry_time instead
199 |             start_time = info['instance_variables'][:last_retry_time]
200 |           end
201 | 
202 |           wait = 0
203 |           if next_time && start_time
204 |             wait = next_time - start_time
205 |           end
206 |           @metrics[:retry_wait].set(wait.to_f, labels: label)
207 |         end
208 |       end
209 |     end
210 | 
211 |     def labels(plugin_info)
212 |       @base_labels.merge(
213 |         plugin_id: plugin_info["plugin_id"],
214 |         type: plugin_info["type"],
215 |       )
216 |     end
217 | 
218 |     def get_gauge(name, docstring)
219 |       if @registry.exist?(name)
220 |         @registry.get(name)
221 |       else
222 |         @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type])
223 |       end
224 |     end
225 | 
226 |     def get_gauge_or_counter(name, docstring)
227 |       if @registry.exist?(name)
228 |         @registry.get(name)
229 |       else
230 |         @registry.public_send(@gauge_or_counter, name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type])
231 |       end
232 |     end
233 |   end
234 | end
235 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/in_prometheus_tail_monitor.rb:
--------------------------------------------------------------------------------
  1 | require 'fluent/plugin/input'
  2 | require 'fluent/plugin/in_monitor_agent'
  3 | require 'fluent/plugin/prometheus'
  4 | 
  5 | module Fluent::Plugin
  6 |   class PrometheusTailMonitorInput < Fluent::Plugin::Input
  7 |     Fluent::Plugin.register_input('prometheus_tail_monitor', self)
  8 |     include Fluent::Plugin::PrometheusLabelParser
  9 | 
 10 |     helpers :timer
 11 | 
 12 |     config_param :interval, :time, default: 5
 13 |     attr_reader :registry
 14 | 
 15 |     MONITOR_IVARS = [
 16 |       :tails,
 17 |     ]
 18 | 
 19 |     def initialize
 20 |       super
 21 |       @registry = ::Prometheus::Client.registry
 22 |     end
 23 | 
 24 |     def multi_workers_ready?
 25 |       true
 26 |     end
 27 | 
 28 |     def configure(conf)
 29 |       super
 30 |       hostname = Socket.gethostname
 31 |       expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
 32 |       expander = expander_builder.build({ 'hostname' => hostname, 'worker_id' => fluentd_worker_id })
 33 |       @base_labels = parse_labels_elements(conf)
 34 |       @base_labels.each do |key, value|
 35 |         unless value.is_a?(String)
 36 |           raise Fluent::ConfigError, "record accessor syntax is not available in prometheus_tail_monitor"
 37 |         end
 38 |         @base_labels[key] = expander.expand(value)
 39 |       end
 40 | 
 41 |       @monitor_agent = Fluent::Plugin::MonitorAgentInput.new
 42 |     end
 43 | 
 44 |     def start
 45 |       super
 46 | 
 47 |       @metrics = {
 48 |         position: get_gauge(
 49 |           :fluentd_tail_file_position,
 50 |           'Current position of file.'),
 51 |         inode: get_gauge(
 52 |           :fluentd_tail_file_inode,
 53 |           'Current inode of file.'),
 54 |         closed_file_metrics: get_gauge(
 55 |           :fluentd_tail_file_closed,
 56 |           'Number of files closed.'),
 57 |         opened_file_metrics: get_gauge(
 58 |           :fluentd_tail_file_opened,
 59 |           'Number of files opened.'),
 60 |         rotated_file_metrics: get_gauge(
 61 |           :fluentd_tail_file_rotated,
 62 |           'Number of files rotated.'),
 63 |         throttled_file_metrics: get_gauge(
 64 |           :fluentd_tail_file_throttled,
 65 |           'Number of times files got throttled.'),
 66 |       }
 67 |       timer_execute(:in_prometheus_tail_monitor, @interval, &method(:update_monitor_info))
 68 |     end
 69 | 
 70 |     def update_monitor_info
 71 |       opts = {
 72 |         ivars: MONITOR_IVARS,
 73 |       }
 74 | 
 75 |       agent_info = @monitor_agent.plugins_info_all(opts).select {|info|
 76 |         info['type'] == 'tail'.freeze
 77 |       }
 78 | 
 79 |       agent_info.each do |info|
 80 |         tails = info['instance_variables'][:tails]
 81 |         next if tails.nil?
 82 | 
 83 |         tails.clone.each do |_, watcher|
 84 |           # Access to internal variable of internal class...
 85 |           # Very fragile implementation
 86 |           pe = watcher.instance_variable_get(:@pe)
 87 |           monitor_info = watcher.instance_variable_get(:@metrics)
 88 |           label = labels(info, watcher.path)
 89 |           @metrics[:inode].set(pe.read_inode, labels: label)
 90 |           @metrics[:position].set(pe.read_pos, labels: label)
 91 |           unless monitor_info.nil?
 92 |             @metrics[:closed_file_metrics].set(monitor_info.closed.get, labels: label)
 93 |             @metrics[:opened_file_metrics].set(monitor_info.opened.get, labels: label)
 94 |             @metrics[:rotated_file_metrics].set(monitor_info.rotated.get, labels: label)
 95 |             @metrics[:throttled_file_metrics].set(monitor_info.throttled.get, labels: label) if monitor_info.members.include?(:throttled)
 96 |           end
 97 |         end
 98 |       end
 99 |     end
100 | 
101 |     def labels(plugin_info, path)
102 |       @base_labels.merge(
103 |         plugin_id: plugin_info["plugin_id"],
104 |         type: plugin_info["type"],
105 |         path: path,
106 |       )
107 |     end
108 | 
109 |     def get_gauge(name, docstring)
110 |       if @registry.exist?(name)
111 |         @registry.get(name)
112 |       else
113 |         @registry.gauge(name, docstring: docstring, labels: @base_labels.keys + [:plugin_id, :type, :path])
114 |       end
115 |     end
116 |   end
117 | end
118 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/out_prometheus.rb:
--------------------------------------------------------------------------------
 1 | require 'fluent/plugin/output'
 2 | require 'fluent/plugin/prometheus'
 3 | 
 4 | module Fluent::Plugin
 5 |   class PrometheusOutput < Fluent::Plugin::Output
 6 |     Fluent::Plugin.register_output('prometheus', self)
 7 |     include Fluent::Plugin::PrometheusLabelParser
 8 |     include Fluent::Plugin::Prometheus
 9 | 
10 |     def initialize
11 |       super
12 |       @registry = ::Prometheus::Client.registry
13 |     end
14 | 
15 |     def multi_workers_ready?
16 |       true
17 |     end
18 | 
19 |     def configure(conf)
20 |       super
21 |       labels = parse_labels_elements(conf)
22 |       @metrics = Fluent::Plugin::Prometheus.parse_metrics_elements(conf, @registry, labels)
23 |     end
24 | 
25 |     def process(tag, es)
26 |       instrument(tag, es, @metrics)
27 |     end
28 |   end
29 | end
30 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/prometheus.rb:
--------------------------------------------------------------------------------
  1 | require 'prometheus/client'
  2 | require 'prometheus/client/formats/text'
  3 | require 'fluent/plugin/prometheus/placeholder_expander'
  4 | 
  5 | module Fluent
  6 |   module Plugin
  7 |     module PrometheusLabelParser
  8 |       def configure(conf)
  9 |         super
 10 |         # Check if running with multiple workers
 11 |         sysconf = if self.respond_to?(:owner) && owner.respond_to?(:system_config)
 12 |           owner.system_config
 13 |         elsif self.respond_to?(:system_config)
 14 |           self.system_config
 15 |         else
 16 |           nil
 17 |         end
 18 |         @multi_worker = sysconf && sysconf.workers ? (sysconf.workers > 1) : false
 19 |       end
 20 | 
 21 |       def parse_labels_elements(conf)
 22 |         base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(conf)
 23 | 
 24 |         if @multi_worker
 25 |           base_labels[:worker_id] = fluentd_worker_id.to_s
 26 |         end
 27 | 
 28 |         base_labels
 29 |       end
 30 |     end
 31 | 
 32 |     module Prometheus
 33 |       class AlreadyRegisteredError < StandardError; end
 34 | 
 35 |       def self.parse_labels_elements(conf)
 36 |         labels = conf.elements.select { |e| e.name == 'labels' }
 37 |         if labels.size > 1
 38 |           raise ConfigError, "labels section must have at most 1"
 39 |         end
 40 | 
 41 |         base_labels = {}
 42 |         unless labels.empty?
 43 |           labels.first.each do |key, value|
 44 |             labels.first.has_key?(key)
 45 | 
 46 |             # use RecordAccessor only for $. and $[ syntax
 47 |             # otherwise use the value as is or expand the value by RecordTransformer for ${} syntax
 48 |             if value.start_with?('$.') || value.start_with?('$[')
 49 |               base_labels[key.to_sym] = PluginHelper::RecordAccessor::Accessor.new(value)
 50 |             else
 51 |               base_labels[key.to_sym] = value
 52 |             end
 53 |           end
 54 |         end
 55 | 
 56 |         base_labels
 57 |       end
 58 | 
 59 |       def self.parse_initlabels_elements(conf, base_labels)
 60 |         base_initlabels = []
 61 | 
 62 |         # We first treat the special case of RecordAccessors and Placeholders labels if any declared
 63 |         conf.elements.select { |e| e.name == 'initlabels' }.each { |block|
 64 |           initlabels = {}
 65 | 
 66 |           block.each do |key, value|
 67 |             if not base_labels.has_key? key.to_sym
 68 |               raise ConfigError, "Key #{key} in <initlabels> is non existent in <labels> for metric #{conf['name']}"
 69 |             end
 70 | 
 71 |             if value.start_with?('$.') || value.start_with?('$[') || value.start_with?('${')
 72 |               raise ConfigError, "Cannot use RecordAccessor or placeholder #{value} (key #{key}) in a <initlabels> in metric #{conf['name']}"
 73 |             end
 74 | 
 75 |             base_label_value = base_labels[key.to_sym]
 76 | 
 77 |             if !(base_label_value.class == Fluent::PluginHelper::RecordAccessor::Accessor) && ! (base_label_value.start_with?('${') )
 78 |               raise ConfigError, "Cannot set <initlabels> on non RecordAccessor/Placeholder key #{key} (value #{value}) in metric #{conf['name']}"
 79 |             end
 80 | 
 81 |             if base_label_value == '${worker_id}' || base_label_value == '${hostname}'
 82 |               raise ConfigError, "Cannot set <initlabels> on reserved placeholder #{base_label_value} for key #{key} in metric #{conf['name']}"
 83 |             end
 84 |             
 85 |             initlabels[key.to_sym] = value
 86 |           end
 87 | 
 88 |           # Now adding all labels that are not RecordAccessor nor Placeholder labels as is
 89 |           base_labels.each do |key, value|
 90 |             if base_labels[key.to_sym].class != Fluent::PluginHelper::RecordAccessor::Accessor
 91 |               if value == '${worker_id}'
 92 |                 # We retrieve fluentd_worker_id this way to not overcomplicate the code
 93 |                 initlabels[key.to_sym] = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
 94 |               elsif value == '${hostname}'
 95 |                 initlabels[key.to_sym] = Socket.gethostname
 96 |               elsif !(value.start_with?('${'))
 97 |                 initlabels[key.to_sym] = value
 98 |               end
 99 |             end
100 |           end
101 | 
102 |           base_initlabels << initlabels
103 |         }
104 | 
105 |         # Testing for RecordAccessor/Placeholder labels missing a declaration in <initlabels> blocks
106 |         base_labels.each do |key, value|
107 |           if value.class == Fluent::PluginHelper::RecordAccessor::Accessor || value.start_with?('${')
108 |             if not base_initlabels.map(&:keys).flatten.include? (key.to_sym)
109 |                 raise ConfigError, "RecordAccessor/Placeholder key #{key} with value #{value} has not been set in a <initlabels> for initialized metric #{conf['name']}"
110 |             end
111 |           end
112 |         end
113 | 
114 |         if base_initlabels.length == 0
115 |           # There were no RecordAccessor nor Placeholder labels, we blunty retrieve the static base_labels
116 |           base_initlabels << base_labels
117 |         end
118 | 
119 |         base_initlabels
120 |       end
121 | 
122 |       def self.parse_metrics_elements(conf, registry, labels = {})
123 |         metrics = []
124 |         conf.elements.select { |element|
125 |           element.name == 'metric'
126 |         }.each { |element|
127 |           if element.has_key?('key') && (element['key'].start_with?('$.') || element['key'].start_with?('$['))
128 |             value = element['key']
129 |             element['key'] = PluginHelper::RecordAccessor::Accessor.new(value)
130 |           end
131 |           case element['type']
132 |           when 'summary'
133 |             metrics << Fluent::Plugin::Prometheus::Summary.new(element, registry, labels)
134 |           when 'gauge'
135 |             metrics << Fluent::Plugin::Prometheus::Gauge.new(element, registry, labels)
136 |           when 'counter'
137 |             metrics << Fluent::Plugin::Prometheus::Counter.new(element, registry, labels)
138 |           when 'histogram'
139 |             metrics << Fluent::Plugin::Prometheus::Histogram.new(element, registry, labels)
140 |           else
141 |             raise ConfigError, "type option must be 'counter', 'gauge', 'summary' or 'histogram'"
142 |           end
143 |         }
144 |         metrics
145 |       end
146 | 
147 |       def self.placeholder_expander(log)
148 |         Fluent::Plugin::Prometheus::ExpandBuilder.new(log: log)
149 |       end
150 | 
151 |       def stringify_keys(hash_to_stringify)
152 |         # Adapted from: https://www.jvt.me/posts/2019/09/07/ruby-hash-keys-string-symbol/
153 |         hash_to_stringify.map do |k,v|
154 |           value_or_hash = if v.instance_of? Hash
155 |                             stringify_keys(v)
156 |                           else
157 |                             v
158 |                           end
159 |           [k.to_s, value_or_hash]
160 |         end.to_h
161 |       end
162 | 
163 |       def configure(conf)
164 |         super
165 |         @placeholder_values = {}
166 |         @placeholder_expander_builder = Fluent::Plugin::Prometheus.placeholder_expander(log)
167 |         @hostname = Socket.gethostname
168 |       end
169 | 
170 |       def instrument_single(tag, time, record, metrics)
171 |         @placeholder_values[tag] ||= {
172 |           'tag' => tag,
173 |           'hostname' => @hostname,
174 |           'worker_id' => fluentd_worker_id,
175 |         }
176 | 
177 |         record = stringify_keys(record)
178 |         placeholders = record.merge(@placeholder_values[tag])
179 |         expander = @placeholder_expander_builder.build(placeholders)
180 |         metrics.each do |metric|
181 |           begin
182 |             metric.instrument(record, expander)
183 |           rescue => e
184 |             log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
185 |             router.emit_error_event(tag, time, record, e)
186 |           end
187 |         end
188 |       end
189 | 
190 |       def instrument(tag, es, metrics)
191 |         placeholder_values = {
192 |           'tag' => tag,
193 |           'hostname' => @hostname,
194 |           'worker_id' => fluentd_worker_id,
195 |         }
196 | 
197 |         es.each do |time, record|
198 |           record = stringify_keys(record)
199 |           placeholders = record.merge(placeholder_values)
200 |           expander = @placeholder_expander_builder.build(placeholders)
201 |           metrics.each do |metric|
202 |             begin
203 |               metric.instrument(record, expander)
204 |             rescue => e
205 |               log.warn "prometheus: failed to instrument a metric.", error_class: e.class, error: e, tag: tag, name: metric.name
206 |               router.emit_error_event(tag, time, record, e)
207 |             end
208 |           end
209 |         end
210 |       end
211 | 
212 |       class Metric
213 |         attr_reader :type
214 |         attr_reader :name
215 |         attr_reader :key
216 |         attr_reader :desc
217 | 
218 |         def initialize(element, registry, labels)
219 |           ['name', 'desc'].each do |key|
220 |             if element[key].nil?
221 |               raise ConfigError, "metric requires '#{key}' option"
222 |             end
223 |           end
224 |           @type = element['type']
225 |           @name = element['name']
226 |           @key = element['key']
227 |           @desc = element['desc']
228 |           element['initialized'].nil? ? @initialized = false : @initialized = element['initialized'] == 'true'
229 |           
230 |           @base_labels = Fluent::Plugin::Prometheus.parse_labels_elements(element)
231 |           @base_labels = labels.merge(@base_labels)
232 | 
233 |           if @initialized
234 |             @base_initlabels = Fluent::Plugin::Prometheus.parse_initlabels_elements(element, @base_labels)
235 |           end
236 |         end
237 | 
238 |         def self.init_label_set(metric, base_initlabels, base_labels)
239 |           base_initlabels.each { |initlabels|
240 |             # Should never happen, but handy test should code evolution break current implementation
241 |             if initlabels.keys.sort != base_labels.keys.sort
242 |               raise ConfigError, "initlabels for metric #{metric.name} must have the same signature than labels " \
243 |                                 "(initlabels given: #{initlabels.keys} vs." \
244 |                                 " expected from labels: #{base_labels.keys})"
245 |             end
246 | 
247 |             metric.init_label_set(initlabels)
248 |           }
249 |         end
250 | 
251 |         def labels(record, expander)
252 |           label = {}
253 |           @base_labels.each do |k, v|
254 |             if v.is_a?(String)
255 |               label[k] = expander.expand(v)
256 |             else
257 |               label[k] = v.call(record)
258 |             end
259 |           end
260 |           label
261 |         end
262 | 
263 |         def self.get(registry, name, type, docstring)
264 |           metric = registry.get(name)
265 | 
266 |           # should have same type, docstring
267 |           if metric.type != type
268 |             raise AlreadyRegisteredError, "#{name} has already been registered as #{type} type"
269 |           end
270 |           if metric.docstring != docstring
271 |             raise AlreadyRegisteredError, "#{name} has already been registered with different docstring"
272 |           end
273 | 
274 |           metric
275 |         end
276 |       end
277 | 
278 |       class Gauge < Metric
279 |         def initialize(element, registry, labels)
280 |           super
281 |           if @key.nil?
282 |             raise ConfigError, "gauge metric requires 'key' option"
283 |           end
284 | 
285 |           begin
286 |             @gauge = registry.gauge(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
287 |           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
288 |             @gauge = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :gauge, element['desc'])
289 |           end
290 | 
291 |           if @initialized
292 |             Fluent::Plugin::Prometheus::Metric.init_label_set(@gauge, @base_initlabels, @base_labels)
293 |           end
294 |         end
295 | 
296 |         def instrument(record, expander)
297 |           if @key.is_a?(String)
298 |             value = record[@key]
299 |           else
300 |             value = @key.call(record)
301 |           end
302 |           if value
303 |             @gauge.set(value, labels: labels(record, expander))
304 |           end
305 |         end
306 |       end
307 | 
308 |       class Counter < Metric
309 |         def initialize(element, registry, labels)
310 |           super
311 |           begin
312 |             @counter = registry.counter(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
313 |           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
314 |             @counter = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :counter, element['desc'])
315 |           end
316 | 
317 |           if @initialized
318 |             Fluent::Plugin::Prometheus::Metric.init_label_set(@counter, @base_initlabels, @base_labels)
319 |           end
320 |         end
321 | 
322 |         def instrument(record, expander)
323 |           # use record value of the key if key is specified, otherwise just increment
324 |           if @key.nil?
325 |             value = 1
326 |           elsif @key.is_a?(String)
327 |             value = record[@key]
328 |           else
329 |             value = @key.call(record)
330 |           end
331 | 
332 |           # ignore if record value is nil
333 |           return if value.nil?
334 | 
335 |           @counter.increment(by: value, labels: labels(record, expander))
336 |         end
337 |       end
338 | 
339 |       class Summary < Metric
340 |         def initialize(element, registry, labels)
341 |           super
342 |           if @key.nil?
343 |             raise ConfigError, "summary metric requires 'key' option"
344 |           end
345 | 
346 |           begin
347 |             @summary = registry.summary(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
348 |           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
349 |             @summary = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :summary, element['desc'])
350 |           end
351 | 
352 |           if @initialized
353 |             Fluent::Plugin::Prometheus::Metric.init_label_set(@summary, @base_initlabels, @base_labels)
354 |           end
355 |         end
356 | 
357 |         def instrument(record, expander)
358 |           if @key.is_a?(String)
359 |             value = record[@key]
360 |           else
361 |             value = @key.call(record)
362 |           end
363 |           if value
364 |             @summary.observe(value, labels: labels(record, expander))
365 |           end
366 |         end
367 |       end
368 | 
369 |       class Histogram < Metric
370 |         def initialize(element, registry, labels)
371 |           super
372 |           if @key.nil?
373 |             raise ConfigError, "histogram metric requires 'key' option"
374 |           end
375 | 
376 |           begin
377 |             if element['buckets']
378 |               buckets = element['buckets'].split(/,/).map(&:strip).map do |e|
379 |                 e[/\A\d+.\d+\Z/] ? e.to_f : e.to_i
380 |               end
381 |               @histogram = registry.histogram(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys, buckets: buckets)
382 |             else
383 |               @histogram = registry.histogram(element['name'].to_sym, docstring: element['desc'], labels: @base_labels.keys)
384 |             end
385 |           rescue ::Prometheus::Client::Registry::AlreadyRegisteredError
386 |             @histogram = Fluent::Plugin::Prometheus::Metric.get(registry, element['name'].to_sym, :histogram, element['desc'])
387 |           end
388 | 
389 |           if @initialized
390 |             Fluent::Plugin::Prometheus::Metric.init_label_set(@histogram, @base_initlabels, @base_labels)
391 |           end
392 |         end
393 | 
394 |         def instrument(record, expander)
395 |           if @key.is_a?(String)
396 |             value = record[@key]
397 |           else
398 |             value = @key.call(record)
399 |           end
400 |           if value
401 |             @histogram.observe(value, labels: labels(record, expander))
402 |           end
403 |         end
404 |       end
405 |     end
406 |   end
407 | end
408 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/prometheus/placeholder_expander.rb:
--------------------------------------------------------------------------------
  1 | module Fluent
  2 |   module Plugin
  3 |     module Prometheus
  4 |       class ExpandBuilder
  5 |         def self.build(placeholder, log:)
  6 |           new(log: log).build(placeholder)
  7 |         end
  8 | 
  9 |         def initialize(log:)
 10 |           @log = log
 11 |         end
 12 | 
 13 |         def build(placeholder_values)
 14 |           placeholders = {}
 15 |           placeholder_values.each do |key, value|
 16 |             case value
 17 |             when Array
 18 |               size = value.size
 19 |               value.each_with_index do |v, i|
 20 |                 placeholders["${#{key}[#{i}]}"] = v
 21 |                 placeholders["${#{key}[#{i - size}]}"] = v
 22 |               end
 23 |             when Hash
 24 |               value.each do |k, v|
 25 |                 placeholders[%(${#{key}["#{k}"]})] = v
 26 |               end
 27 |             else
 28 |               if key == 'tag'
 29 |                 placeholders.merge!(build_tag(value))
 30 |               else
 31 |                 placeholders["${#{key}}"] = value
 32 |               end
 33 |             end
 34 |           end
 35 | 
 36 |           Fluent::Plugin::Prometheus::ExpandBuilder::PlaceholderExpander.new(@log, placeholders)
 37 |         end
 38 | 
 39 |         private
 40 | 
 41 |         def build_tag(tag)
 42 |           tags = tag.split('.')
 43 | 
 44 |           placeholders = { '${tag}' => tag }
 45 | 
 46 |           size = tags.size
 47 | 
 48 |           tags.each_with_index do |v, i|
 49 |             placeholders["${tag_parts[#{i}]}"] = v
 50 |             placeholders["${tag_parts[#{i - size}]}"] = v
 51 |           end
 52 | 
 53 |           tag_prefix(tags).each_with_index do |v, i|
 54 |             placeholders["${tag_prefix[#{i}]}"] = v
 55 |           end
 56 | 
 57 |           tag_suffix(tags).each_with_index do |v, i|
 58 |             placeholders["${tag_suffix[#{i}]}"] = v
 59 |           end
 60 | 
 61 |           placeholders
 62 |         end
 63 | 
 64 |         def tag_prefix(tags)
 65 |           tags = tags.dup
 66 |           return [] if tags.empty?
 67 | 
 68 |           ret = [tags.shift]
 69 |           tags.each.with_index(1) do |tag, i|
 70 |             ret[i] = "#{ret[i-1]}.#{tag}"
 71 |           end
 72 |           ret
 73 |         end
 74 | 
 75 |         def tag_suffix(tags)
 76 |           return [] if tags.empty?
 77 | 
 78 |           tags = tags.dup.reverse
 79 |           ret = [tags.shift]
 80 |           tags.each.with_index(1) do |tag, i|
 81 |             ret[i] = "#{tag}.#{ret[i-1]}"
 82 |           end
 83 |           ret
 84 |         end
 85 | 
 86 |         class PlaceholderExpander
 87 |           PLACEHOLDER_REGEX = /(\${[^\[}]+(\[[^\]]+\])?})/.freeze
 88 | 
 89 |           attr_reader :placeholder
 90 | 
 91 |           def initialize(log, placeholder)
 92 |             @placeholder = placeholder
 93 |             @log = log
 94 |             @expander_cache = {}
 95 |           end
 96 | 
 97 |           def merge_placeholder(placeholder)
 98 |             @placeholder.merge!(placeholder)
 99 |           end
100 | 
101 |           def expand(str, dynamic_placeholders: nil)
102 |             expander = if dynamic_placeholders
103 |                          if @expander_cache[dynamic_placeholders]
104 |                            @expander_cache[dynamic_placeholders]
105 |                          else
106 |                            e = ExpandBuilder.build(dynamic_placeholders, log: @log)
107 |                            e.merge_placeholder(@placeholder)
108 |                            @expander_cache[dynamic_placeholders] = e
109 |                            e
110 |                          end
111 |                        else
112 |                          self
113 |                        end
114 | 
115 |             expander.expand!(str)
116 |           end
117 | 
118 |           protected
119 | 
120 |           def expand!(str)
121 |             str.gsub(PLACEHOLDER_REGEX) { |value|
122 |               @placeholder.fetch(value) do
123 |                 @log.warn("unknown placeholder `#{value}` found")
124 |                 value # return as it is
125 |               end
126 |             }
127 |           end
128 |         end
129 |       end
130 |     end
131 |   end
132 | end
133 | 


--------------------------------------------------------------------------------
/lib/fluent/plugin/prometheus_metrics.rb:
--------------------------------------------------------------------------------
 1 | module Fluent::Plugin
 2 | 
 3 |   ##
 4 |   # PromMetricsAggregator aggregates multiples metrics exposed using Prometheus text-based format
 5 |   # see https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md
 6 | 
 7 | 
 8 |   class PrometheusMetrics
 9 |     def initialize
10 |       @comments = []
11 |       @metrics = []
12 |     end
13 | 
14 |     def to_string
15 |       (@comments + @metrics).join("\n")
16 |     end
17 | 
18 |     def add_comment(comment)
19 |       @comments << comment
20 |     end
21 | 
22 |     def add_metric_value(value)
23 |       @metrics << value
24 |     end
25 | 
26 |     attr_writer :comments, :metrics
27 |   end
28 | 
29 |   class PromMetricsAggregator
30 |     def initialize
31 |       @metrics = {}
32 |     end
33 | 
34 |     def get_metric_name_from_comment(line)
35 |       tokens = line.split(' ')
36 |       if ['HELP', 'TYPE'].include?(tokens[1])
37 |         tokens[2]
38 |       else
39 |         ''
40 |       end
41 |     end
42 | 
43 |     def add_metrics(metrics)
44 |       current_metric = ''
45 |       new_metric = false
46 |       lines = metrics.split("\n")
47 |       for line in lines
48 |         if line[0] == '#'
49 |           # Metric comment (# TYPE, # HELP)
50 |           parsed_metric = get_metric_name_from_comment(line)
51 |           if parsed_metric != ''
52 |             if parsed_metric != current_metric
53 |               # Starting a new metric comment block
54 |               new_metric = !@metrics.key?(parsed_metric)
55 |               if new_metric
56 |                 @metrics[parsed_metric] = PrometheusMetrics.new()
57 |               end
58 |               current_metric = parsed_metric
59 |             end
60 | 
61 |             if new_metric && parsed_metric == current_metric
62 |               # New metric, inject comments (# TYPE, # HELP)
63 |               @metrics[parsed_metric].add_comment(line)
64 |             end
65 |           end
66 |         else
67 |           # Metric value, simply append line
68 |           @metrics[current_metric].add_metric_value(line)
69 |         end
70 |       end
71 |     end
72 | 
73 |     def get_metrics
74 |       @metrics.map{|k,v| v.to_string()}.join("\n") + (@metrics.length ? "\n" : "")
75 |     end
76 |   end
77 | end
78 | 


--------------------------------------------------------------------------------
/misc/fluentd_sample.conf:
--------------------------------------------------------------------------------
  1 | ## Prometheus Input Plugin Configuration
  2 | 
  3 | # input plugin that exports metrics
  4 | <source>
  5 |   @type prometheus
  6 | </source>
  7 | 
  8 | <source>
  9 |   @type monitor_agent
 10 | </source>
 11 | 
 12 | <source>
 13 |   @type forward
 14 | </source>
 15 | 
 16 | # input plugin that collects metrics from MonitorAgent
 17 | <source>
 18 |   @type prometheus_monitor
 19 |   <labels>
 20 |     host ${hostname}
 21 |   </labels>
 22 | </source>
 23 | 
 24 | # input plugin that collects metrics for output plugin
 25 | <source>
 26 |   @type prometheus_output_monitor
 27 |   <labels>
 28 |     host ${hostname}
 29 |   </labels>
 30 | </source>
 31 | 
 32 | # input plugin that collects metrics for in_tail plugin
 33 | <source>
 34 |   @type prometheus_tail_monitor
 35 |   <labels>
 36 |     host ${hostname}
 37 |   </labels>
 38 | </source>
 39 | 
 40 | ## Nginx Access Log Configuration
 41 | 
 42 | <source>
 43 |   @type tail
 44 |   format nginx
 45 |   tag nginx
 46 |   path /var/log/nginx/access.log
 47 |   pos_file /tmp/fluent_nginx.pos
 48 |   types size:integer
 49 | </source>
 50 | 
 51 | <filter nginx>
 52 |   @type prometheus
 53 | 
 54 |   # You can use counter type with specifying a key,
 55 |   # and increments counter by the value
 56 |   <metric>
 57 |     name nginx_size_counter_bytes
 58 |     type counter
 59 |     desc nginx bytes sent
 60 |     key size
 61 |     <labels>
 62 |        host ${hostname}
 63 |        foo bar
 64 |     </labels>
 65 |   </metric>
 66 | 
 67 |   # You can use counter type without specifying a key
 68 |   # This just increments counter by 1
 69 |   <metric>
 70 |     name nginx_record_counts
 71 |     type counter
 72 |     desc the number of emited records
 73 |     <labels>
 74 |        host ${hostname}
 75 |     </labels>
 76 |   </metric>
 77 | </filter>
 78 | 
 79 | <match nginx>
 80 |   @type copy
 81 |   # for MonitorAgent sample
 82 |   <store>
 83 |     @id test_forward
 84 |     @type forward
 85 |     buffer_type memory
 86 |     flush_interval 1s
 87 |     max_retry_wait 2s
 88 |     <buffer>
 89 |     # max_retry_wait 10s
 90 |     flush_interval 1s
 91 |     # retry_type periodic
 92 |     disable_retry_limit
 93 | </buffer>
 94 |     # retry_limit 3
 95 |     disable_retry_limit
 96 |    <server>
 97 |      host 127.0.0.1
 98 |      port 20000
 99 |    </server>
100 |   </store>
101 |   <store>
102 |     @type stdout
103 |   </store>
104 | </match>
105 | 
106 | ## Nginx Proxy Log Configuration
107 | 
108 | <source>
109 |   @type tail
110 |   format ltsv
111 |   tag nginx_proxy
112 |   path /var/log/nginx/access_proxy.log
113 |   pos_file /tmp/fluent_nginx_proxy.pos
114 |   types size:integer,request_length:integer,bytes_sent:integer,body_bytes_sent:integer,request_time:float,upstream_response_time:float
115 | </source>
116 | 
117 | <filter nginx_proxy>
118 |   @type prometheus
119 | 
120 |   # common labels for all metrics
121 |   <labels>
122 |      host ${hostname}
123 |      method ${request_method}
124 |      status ${status}
125 |   </labels>
126 | 
127 |   <metric>
128 |     name nginx_proxy_request_length_total_bytes
129 |     type counter
130 |     desc nginx proxy request length bytes
131 |     key request_length
132 |   </metric>
133 |   <metric>
134 |     name nginx_proxy_bytes_sent_total_bytes
135 |     type counter
136 |     desc nginx proxy bytes sent
137 |     key bytes_sent
138 |   </metric>
139 |   <metric>
140 |     name nginx_proxy_request_duration_total_milliseconds
141 |     type counter
142 |     desc nginx proxy request time
143 |     key request_time
144 |   </metric>
145 |   <metric>
146 |     name nginx_proxy_upstream_response_duration_total_milliseconds
147 |     type counter
148 |     desc nginx proxy upstream response time
149 |     key upstream_response_time
150 |   </metric>
151 |   <metric>
152 |     name nginx_proxy_request_duration_milliseconds
153 |     type summary
154 |     desc nginx proxy request duration summary
155 |     key request_time
156 |   </metric>
157 |   <metric>
158 |     name nginx_proxy_upstream_duration_milliseconds
159 |     type summary
160 |     desc nginx proxy upstream response duration summary
161 |     key upstream_response_time
162 |   </metric>
163 | </filter>
164 | 
165 | <match nginx_proxy>
166 |   @type copy
167 |   <store>
168 |     @type stdout
169 |   </store>
170 | </match>
171 | 


--------------------------------------------------------------------------------
/misc/nginx_proxy.conf:
--------------------------------------------------------------------------------
 1 | log_format ltsv 'time:$time_iso8601\t'
 2 |                 'remote_addr:$remote_addr\t'
 3 |                 'request_method:$request_method\t'
 4 |                 'request_length:$request_length\t'
 5 |                 'request_uri:$request_uri\t'
 6 |                 'uri:$uri\t'
 7 |                 'status:$status\t'
 8 |                 'bytes_sent:$bytes_sent\t'
 9 |                 'body_bytes_sent:$body_bytes_sent\t'
10 |                 'referer:$http_referer\t'
11 |                 'useragent:$http_user_agent\t'
12 |                 'request_time:$request_time\t'
13 |                 'upstream_response_time:$upstream_response_time';
14 | 
15 | server {
16 |   access_log /var/log/nginx/access_proxy.log ltsv;
17 |   listen 9999;
18 |   location / {
19 |     proxy_pass https://www.google.com;
20 |   }
21 | }
22 | 
23 | 


--------------------------------------------------------------------------------
/misc/prometheus.yaml:
--------------------------------------------------------------------------------
 1 | # A job to scrape an endpoint of Fluentd running on localhost.
 2 | scrape_configs:
 3 | - job_name: 'prometheus'
 4 |   scrape_interval: 5s
 5 |   static_configs:
 6 |     - targets:
 7 |       - 'localhost:9090'
 8 | - job_name: fluentd
 9 |   scrape_interval: 5s
10 |   static_configs:
11 |     - targets:
12 |       - 'localhost:24231'
13 |   metrics_path: /metrics
14 | 


--------------------------------------------------------------------------------
/misc/prometheus_alerts.yaml:
--------------------------------------------------------------------------------
 1 | ALERT FluentdNodeDown
 2 |   IF up{job="fluentd"} == 0
 3 |   FOR 10m
 4 |   LABELS {
 5 |     service = "fluentd",
 6 |     severity = "warning"
 7 |   }
 8 |   ANNOTATIONS {
 9 |     summary = "fluentd cannot be scraped",
10 |     description = "Prometheus could not scrape {{ $labels.job }} for more than 10 minutes",
11 |   }
12 |   
13 | ALERT FluentdNodeDown
14 |   IF up{job="fluentd"} == 0
15 |   FOR 30m
16 |   LABELS {
17 |     service = "fluentd",
18 |     severity = "critical"
19 |   }
20 |   ANNOTATIONS {
21 |     summary = "fluentd cannot be scraped",
22 |     description = "Prometheus could not scrape {{ $labels.job }} for more than 30 minutes",
23 |   }
24 |   
25 | ALERT FluentdQueueLength
26 |   IF rate(fluentd_status_buffer_queue_length[5m]) > 0.3
27 |   FOR 1m
28 |   LABELS {
29 |     service = "fluentd",
30 |     severity = "warning"
31 |   }
32 |   ANNOTATIONS {
33 |     summary = "fluentd node are failing",
34 |     description = "In the last 5 minutes, fluentd queues increased 30%. Current value is {{ $value }} ",
35 |   }
36 | 
37 | ALERT FluentdQueueLength
38 |   IF rate(fluentd_status_buffer_queue_length[5m]) > 0.5
39 |   FOR 1m
40 |   LABELS {
41 |     service = "fluentd",
42 |     severity = "critical"
43 |   }
44 |   ANNOTATIONS {
45 |     summary = "fluentd node are critical",
46 |     description = "In the last 5 minutes, fluentd queues increased 50%. Current value is {{ $value }} ",
47 |   }
48 | 
49 | ALERT FluentdRecordsCountsHigh
50 |   IF sum(rate(fluentd_output_status_emit_records{job="fluentd"}[5m])) BY (instance) >  (3 * sum(rate(fluentd_output_status_emit_records{job="fluentd"}[15m])) BY (instance))
51 |   FOR 1m
52 |   LABELS {
53 |     service = "fluentd",
54 |     severity = "critical"
55 |   }
56 |   ANNOTATIONS {
57 |     summary = "fluentd records count are critical",
58 |     description = "In the last 5m, records counts increased 3 times, comparing to the latest 15 min.",
59 |   }
60 | 


--------------------------------------------------------------------------------
/spec/fluent/plugin/filter_prometheus_spec.rb:
--------------------------------------------------------------------------------
 1 | require 'spec_helper'
 2 | require 'fluent/test/driver/filter'
 3 | require 'fluent/plugin/filter_prometheus'
 4 | require_relative 'shared'
 5 | 
 6 | describe Fluent::Plugin::PrometheusFilter do
 7 |   let(:tag) { 'prometheus.test' }
 8 |   let(:driver) { Fluent::Test::Driver::Filter.new(Fluent::Plugin::PrometheusFilter).configure(config) }
 9 |   let(:registry) { ::Prometheus::Client::Registry.new }
10 | 
11 |   before do
12 |     allow(Prometheus::Client).to receive(:registry).and_return(registry)
13 |   end
14 | 
15 |   describe '#configure' do
16 |     it_behaves_like 'output configuration'
17 |   end
18 | 
19 |   describe '#run' do
20 |     let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10} }
21 | 
22 |     context 'simple config' do
23 |       let(:config) {
24 |         BASE_CONFIG + %(
25 |           <metric>
26 |             name simple
27 |             type counter
28 |             desc Something foo.
29 |             key foo
30 |           </metric>
31 |         )
32 |       }
33 | 
34 |       it 'adds a new counter metric' do
35 |         expect(registry.metrics.map(&:name)).not_to eq([:simple])
36 |         driver.run(default_tag: tag) { driver.feed(event_time, message) }
37 |         expect(registry.metrics.map(&:name)).to eq([:simple])
38 |       end
39 | 
40 |       it 'should keep original message' do
41 |         driver.run(default_tag: tag) { driver.feed(event_time, message) }
42 |         expect(driver.filtered_records.first).to eq(message)
43 |       end
44 |     end
45 | 
46 |     it_behaves_like 'instruments record'
47 |   end
48 | end
49 | 


--------------------------------------------------------------------------------
/spec/fluent/plugin/in_prometheus_monitor_spec.rb:
--------------------------------------------------------------------------------
 1 | require 'spec_helper'
 2 | require 'fluent/plugin/in_prometheus_monitor'
 3 | require 'fluent/test/driver/input'
 4 | 
 5 | describe Fluent::Plugin::PrometheusMonitorInput do
 6 |   MONITOR_CONFIG = %[
 7 |   @type prometheus_monitor
 8 |   <labels>
 9 |     host ${hostname}
10 |     foo bar
11 |   </labels>
12 | ]
13 | 
14 |   INVALID_MONITOR_CONFIG = %[
15 |   @type prometheus_monitor
16 | 
17 |   <labels>
18 |     host ${hostname}
19 |     foo bar
20 |     invalid_use1 $.foo.bar
21 |     invalid_use2 $[0][1]
22 |   </labels>
23 | ]
24 | 
25 |   let(:config) { MONITOR_CONFIG }
26 |   let(:driver) { Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusMonitorInput).configure(config) }
27 | 
28 |   describe '#configure' do
29 |     describe 'valid' do
30 |       it 'does not raise error' do
31 |         expect{driver}.not_to raise_error
32 |       end
33 |     end
34 | 
35 |     describe 'invalid' do
36 |       let(:config) { INVALID_MONITOR_CONFIG }
37 |       it 'expect raise error' do
38 |         expect{driver}.to raise_error
39 |       end
40 |     end
41 |   end
42 | end
43 | 


--------------------------------------------------------------------------------
/spec/fluent/plugin/in_prometheus_spec.rb:
--------------------------------------------------------------------------------
  1 | require 'spec_helper'
  2 | require 'fluent/plugin/in_prometheus'
  3 | require 'fluent/test/driver/input'
  4 | 
  5 | require 'net/http'
  6 | require 'zlib'
  7 | 
  8 | describe Fluent::Plugin::PrometheusInput do
  9 |   CONFIG = %[
 10 |   @type prometheus
 11 | ]
 12 | 
 13 |   LOCAL_CONFIG = %[
 14 |   @type prometheus
 15 |   bind 127.0.0.1
 16 | ]
 17 | 
 18 |   let(:config) { CONFIG }
 19 |   let(:port) { 24231 }
 20 |   let(:driver) { Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusInput).configure(config) }
 21 | 
 22 |   describe '#configure' do
 23 |     describe 'bind' do
 24 |       let(:config) { CONFIG + %[
 25 |   bind 127.0.0.1
 26 | ] }
 27 |       it 'should be configurable' do
 28 |         expect(driver.instance.bind).to eq('127.0.0.1')
 29 |       end
 30 |     end
 31 | 
 32 |     describe 'port' do
 33 |       let(:config) { CONFIG + %[
 34 |   port 8888
 35 | ] }
 36 |       it 'should be configurable' do
 37 |         expect(driver.instance.port).to eq(8888)
 38 |       end
 39 |     end
 40 | 
 41 |     describe 'metrics_path' do
 42 |       let(:config) { CONFIG + %[
 43 |   metrics_path /_test
 44 | ] }
 45 |       it 'should be configurable' do
 46 |         expect(driver.instance.metrics_path).to eq('/_test')
 47 |       end
 48 |     end
 49 | 
 50 |     describe 'content_encoding_identity' do
 51 |       let(:config) { CONFIG + %[
 52 |     content_encoding identity
 53 | ] }
 54 |       it 'should be configurable' do
 55 |         expect(driver.instance.content_encoding).to eq(:identity)
 56 |       end
 57 |     end
 58 | 
 59 |     describe 'content_encoding_gzip' do
 60 |       let(:config) { CONFIG + %[
 61 |     content_encoding gzip
 62 | ] }
 63 |       it 'should be configurable' do
 64 |         expect(driver.instance.content_encoding).to eq(:gzip)
 65 |       end
 66 |     end
 67 |   end
 68 | 
 69 |   describe '#start' do
 70 |     context 'with transport section' do
 71 |       let(:config) do
 72 |         %[
 73 |            @type prometheus
 74 |            bind 127.0.0.1
 75 |            <transport tls>
 76 |              insecure true
 77 |            </transport>
 78 |          ]
 79 |       end
 80 | 
 81 |       it 'returns 200' do
 82 |         driver.run(timeout: 1) do
 83 |           Net::HTTP.start('127.0.0.1', port, verify_mode: OpenSSL::SSL::VERIFY_NONE, use_ssl: true) do |http|
 84 |             req = Net::HTTP::Get.new('/metrics')
 85 |             res = http.request(req)
 86 |             expect(res.code).to eq('200')
 87 |           end
 88 |         end
 89 |       end
 90 |     end
 91 | 
 92 |     context 'old parameters are given' do
 93 |       context 'when extra_conf is used' do
 94 |         let(:config) do
 95 |           %[
 96 |             @type prometheus
 97 |             bind 127.0.0.1
 98 |             <ssl>
 99 |               enable true
100 |               extra_conf { "SSLCertName": [["CN", "nobody"], ["DC", "example"]] }
101 |             </ssl>
102 |          ]
103 |         end
104 | 
105 |         it 'uses webrick' do
106 |           expect(driver.instance).to receive(:start_webrick).once
107 |           driver.run(timeout: 1)
108 |         end
109 | 
110 |         it 'returns 200' do
111 |           driver.run(timeout: 1) do
112 |             Net::HTTP.start('127.0.0.1', port, verify_mode: OpenSSL::SSL::VERIFY_NONE, use_ssl: true) do |http|
113 |               req = Net::HTTP::Get.new('/metrics')
114 |               res = http.request(req)
115 |               expect(res.code).to eq('200')
116 |             end
117 |           end
118 |         end
119 |       end
120 | 
121 |       context 'cert_path and private_key_path combination' do
122 |         let(:config) do
123 |           %[
124 |             @type prometheus
125 |             bind 127.0.0.1
126 |             <ssl>
127 |               enable true
128 |               certificate_path path
129 |               private_key_path path1
130 |             </ssl>
131 |           ]
132 |         end
133 | 
134 |         it 'converts them into new transport section' do
135 |           expect(driver.instance).to receive(:http_server_create_http_server).with(
136 |                                        :in_prometheus_server,
137 |                                        addr: anything,
138 |                                        logger: anything,
139 |                                        port: anything,
140 |                                        proto: :tls,
141 |                                        tls_opts: { 'cert_path' => 'path', 'private_key_path' => 'path1' }
142 |                                      ).once
143 | 
144 |           driver.run(timeout: 1)
145 |         end
146 |       end
147 | 
148 |       context 'insecure and ca_path' do
149 |         let(:config) do
150 |           %[
151 |             @type prometheus
152 |             bind 127.0.0.1
153 |             <ssl>
154 |               enable true
155 |               ca_path path
156 |             </ssl>
157 |            ]
158 |         end
159 | 
160 |         it 'converts them into new transport section' do
161 |           expect(driver.instance).to receive(:http_server_create_http_server).with(
162 |                                        :in_prometheus_server,
163 |                                        addr: anything,
164 |                                        logger: anything,
165 |                                        port: anything,
166 |                                        proto: :tls,
167 |                                        tls_opts: { 'ca_path' => 'path', 'insecure' => true }
168 |                                      ).once
169 | 
170 |           driver.run(timeout: 1)
171 |         end
172 |       end
173 | 
174 |       context 'when only private_key_path is geven' do
175 |         let(:config) do
176 |           %[
177 |             @type prometheus
178 |             bind 127.0.0.1
179 |             <ssl>
180 |               enable true
181 |               private_key_path path
182 |             </ssl>
183 |            ]
184 |         end
185 | 
186 |         it 'raises ConfigError' do
187 |           expect { driver.run(timeout: 1) }.to raise_error(Fluent::ConfigError, 'both certificate_path and private_key_path must be defined')
188 |         end
189 |       end
190 |     end
191 |   end
192 | 
193 |   describe '#run' do
194 |     context '/metrics' do
195 |       let(:config) { LOCAL_CONFIG }
196 |       it 'returns 200' do
197 |         driver.run(timeout: 1) do
198 |           Net::HTTP.start("127.0.0.1", port) do |http|
199 |             req = Net::HTTP::Get.new("/metrics")
200 |             res = http.request(req)
201 |             expect(res.code).to eq('200')
202 |           end
203 |         end
204 |       end
205 |     end
206 | 
207 |     context '/foo' do
208 |       let(:config) { LOCAL_CONFIG }
209 |       it 'does not return 200' do
210 |         driver.run(timeout: 1) do
211 |           Net::HTTP.start("127.0.0.1", port) do |http|
212 |             req = Net::HTTP::Get.new("/foo")
213 |             res = http.request(req)
214 |             expect(res.code).not_to eq('200')
215 |           end
216 |         end
217 |       end
218 |     end
219 | 
220 |     context 'response content_encoding identity' do
221 |       let(:config) { LOCAL_CONFIG + %[
222 |         content_encoding identity
223 |   ] }
224 |       it 'exposes metric' do
225 |         driver.run(timeout: 1) do
226 |           registry = driver.instance.instance_variable_get(:@registry)
227 |           registry.counter(:test,docstring: "Testing metrics") unless registry.exist?(:test)
228 |           Net::HTTP.start("127.0.0.1", port) do |http|
229 |             req = Net::HTTP::Get.new("/metrics")
230 |             req['accept-encoding'] = nil
231 |             res = http.request(req)
232 |             expect(res.body).to include("test Testing metrics")
233 |           end
234 |         end
235 |       end
236 |     end
237 | 
238 |     context 'response content_encoding gzip' do
239 |       let(:config) { LOCAL_CONFIG + %[
240 |         content_encoding gzip
241 |   ] }
242 |       it 'exposes metric' do
243 |         driver.run(timeout: 1) do
244 |           registry = driver.instance.instance_variable_get(:@registry)
245 |           registry.counter(:test,docstring: "Testing metrics") unless registry.exist?(:test)
246 |           Net::HTTP.start("127.0.0.1", port) do |http|
247 |             req = Net::HTTP::Get.new("/metrics")
248 |             req['accept-encoding'] = nil
249 |             res = http.request(req)
250 |             gzip = Zlib::GzipReader.new(StringIO.new(res.body.to_s))
251 |             expect(gzip.read).to include("test Testing metrics")
252 |           end
253 |         end
254 |       end
255 |     end
256 |   end
257 | 
258 |   describe '#run_multi_workers' do
259 |     context '/metrics' do
260 |       Fluent::SystemConfig.overwrite_system_config('workers' => 4) do
261 |         let(:config) { FULL_CONFIG + %[
262 |           port #{port - 2}
263 |         ] }
264 | 
265 |         it 'should configure port using sequential number' do
266 |           driver = Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusInput)
267 |           driver.instance.instance_eval{ @_fluentd_worker_id = 2 }
268 |           driver.configure(config)
269 |           expect(driver.instance.port).to eq(port)
270 |           driver.run(timeout: 1) do
271 |             Net::HTTP.start("127.0.0.1", port) do |http|
272 |               req = Net::HTTP::Get.new("/metrics")
273 |               res = http.request(req)
274 |               expect(res.code).to eq('200')
275 |             end
276 |           end
277 |         end
278 |       end
279 |     end
280 |   end
281 | end
282 | 


--------------------------------------------------------------------------------
/spec/fluent/plugin/in_prometheus_tail_monitor_spec.rb:
--------------------------------------------------------------------------------
 1 | require 'spec_helper'
 2 | require 'fluent/plugin/in_prometheus_tail_monitor'
 3 | require 'fluent/test/driver/input'
 4 | 
 5 | describe Fluent::Plugin::PrometheusTailMonitorInput do
 6 |   MONITOR_CONFIG = %[
 7 |   @type prometheus_tail_monitor
 8 |   <labels>
 9 |     host ${hostname}
10 |     foo bar
11 |   </labels>
12 | ]
13 | 
14 |   INVALID_MONITOR_CONFIG = %[
15 |   @type prometheus_tail_monitor
16 | 
17 |   <labels>
18 |     host ${hostname}
19 |     foo bar
20 |     invalid_use1 $.foo.bar
21 |     invalid_use2 $[0][1]
22 |   </labels>
23 | ]
24 | 
25 |   let(:config) { MONITOR_CONFIG }
26 |   let(:driver) { Fluent::Test::Driver::Input.new(Fluent::Plugin::PrometheusTailMonitorInput).configure(config) }
27 | 
28 |   describe '#configure' do
29 |     describe 'valid' do
30 |       it 'does not raise error' do
31 |         expect { driver }.not_to raise_error
32 |       end
33 |     end
34 | 
35 |     describe 'invalid' do
36 |       let(:config) { INVALID_MONITOR_CONFIG }
37 |       it 'expect raise error' do
38 |         expect { driver }.to raise_error
39 |       end
40 |     end
41 |   end
42 | end
43 | 


--------------------------------------------------------------------------------
/spec/fluent/plugin/out_prometheus_spec.rb:
--------------------------------------------------------------------------------
 1 | require 'spec_helper'
 2 | require 'fluent/test/driver/output'
 3 | require 'fluent/plugin/out_prometheus'
 4 | require_relative 'shared'
 5 | 
 6 | describe Fluent::Plugin::PrometheusOutput do
 7 |   let(:tag) { 'prometheus.test' }
 8 |   let(:driver) { Fluent::Test::Driver::Output.new(Fluent::Plugin::PrometheusOutput).configure(config) }
 9 |   let(:registry) { ::Prometheus::Client::Registry.new }
10 | 
11 |   before do
12 |     allow(Prometheus::Client).to receive(:registry).and_return(registry)
13 |   end
14 | 
15 |   describe '#configure' do
16 |     it_behaves_like 'output configuration'
17 |   end
18 | 
19 |   describe '#testinitlabels' do
20 |     it_behaves_like 'initalized metrics'
21 |   end
22 |   
23 |   describe '#run' do
24 |     let(:message) { {"foo" => 100, "bar" => 100, "baz" => 100, "qux" => 10} }
25 | 
26 |     context 'simple config' do
27 |       let(:config) {
28 |         BASE_CONFIG + %(
29 |           <metric>
30 |             name simple
31 |             type counter
32 |             desc Something foo.
33 |             key foo
34 |           </metric>
35 |         )
36 |       }
37 | 
38 |       it 'adds a new counter metric' do
39 |         expect(registry.metrics.map(&:name)).not_to eq([:simple])
40 |         driver.run(default_tag: tag) { driver.feed(event_time, message) }
41 |         expect(registry.metrics.map(&:name)).to eq([:simple])
42 |       end
43 |     end
44 | 
45 |     it_behaves_like 'instruments record'
46 |   end
47 | 
48 |   describe '#run with symbolized keys' do
49 |     let(:message) { {:foo => 100, :bar => 100, :baz => 100, :qux => 10} }
50 | 
51 |     context 'simple config' do
52 |       let(:config) {
53 |         BASE_CONFIG + %(
54 |           <metric>
55 |             name simple
56 |             type counter
57 |             desc Something foo.
58 |             key foo
59 |           </metric>
60 |         )
61 |       }
62 | 
63 |       it 'adds a new counter metric' do
64 |         expect(registry.metrics.map(&:name)).not_to eq([:simple])
65 |         driver.run(default_tag: tag) { driver.feed(event_time, message) }
66 |         expect(registry.metrics.map(&:name)).to eq([:simple])
67 |       end
68 |     end
69 | 
70 |     it_behaves_like 'instruments record'
71 |   end
72 | end
73 | 


--------------------------------------------------------------------------------
/spec/fluent/plugin/prometheus/placeholder_expander_spec.rb:
--------------------------------------------------------------------------------
  1 | require 'logger'
  2 | 
  3 | require 'spec_helper'
  4 | require 'fluent/plugin/prometheus/placeholder_expander'
  5 | require_relative '../shared'
  6 | 
  7 | describe Fluent::Plugin::Prometheus::ExpandBuilder::PlaceholderExpander do
  8 |   let(:log) do
  9 |     Logger.new('/dev/null')
 10 |   end
 11 | 
 12 |   let(:builder) do
 13 |     Fluent::Plugin::Prometheus::ExpandBuilder.new(log: log)
 14 |   end
 15 | 
 16 |   describe '#expand' do
 17 |     context 'with static placeholder' do
 18 |       let(:static_placeholder) do
 19 |         {
 20 |           'hostname' => 'host_value',
 21 |           'tag' => '1.2.3',
 22 |           'ary_value' => ['1', '2', '3'],
 23 |           'hash_value' => { 'key1' => 'val1' },
 24 |         }
 25 |       end
 26 | 
 27 |       let(:dynamic_placeholder) do
 28 |       end
 29 | 
 30 |       it 'expands values' do
 31 |         expander = builder.build(static_placeholder)
 32 |         expect(expander.expand('${hostname}')).to eq('host_value')
 33 |         expect(expander.expand('${ary_value[0]}.${ary_value[1]}.${ary_value[2]}')).to eq('1.2.3')
 34 |         expect(expander.expand('${ary_value[-3]}.${ary_value[-2]}.${ary_value[-1]}')).to eq('1.2.3')
 35 |         expect(expander.expand('${hash_value["key1"]}')).to eq('val1')
 36 | 
 37 |         expect(expander.expand('${tag}')).to eq('1.2.3')
 38 |         expect(expander.expand('${tag_parts[0]}.${tag_parts[1]}.${tag_parts[2]}')).to eq('1.2.3')
 39 |         expect(expander.expand('${tag_parts[-3]}.${tag_parts[-2]}.${tag_parts[-1]}')).to eq('1.2.3')
 40 |         expect(expander.expand('${tag_prefix[0]}.${tag_prefix[1]}.${tag_prefix[2]}')).to eq('1.1.2.1.2.3')
 41 |         expect(expander.expand('${tag_suffix[0]}.${tag_suffix[1]}.${tag_suffix[2]}')).to eq('3.2.3.1.2.3')
 42 |       end
 43 | 
 44 |       it 'does not create new expander' do
 45 |         builder # cached before mock
 46 | 
 47 |         expect(Fluent::Plugin::Prometheus::ExpandBuilder).to receive(:build).with(anything, log: anything).never
 48 |         expander = builder.build(static_placeholder)
 49 |         expander.expand('${hostname}')
 50 |         expander.expand('${hostname}')
 51 |       end
 52 | 
 53 |       context 'when not found placeholder' do
 54 |         it 'prints wanring log and as it is' do
 55 |           expect(log).to receive(:warn).with('unknown placeholder `${tag_prefix[100]}` found').once
 56 | 
 57 |           expander = builder.build(static_placeholder)
 58 |           expect(expander.expand('${tag_prefix[100]}')).to eq('${tag_prefix[100]}')
 59 |         end
 60 |       end
 61 |     end
 62 | 
 63 |     context 'with dynamic placeholder' do
 64 |       let(:static_placeholder) do
 65 |         {
 66 |           'hostname' => 'host_value',
 67 |           'ary_value' => ['1', '2', '3'],
 68 |           'hash_value' => { 'key1' => 'val1' },
 69 |         }
 70 |       end
 71 | 
 72 |       let(:dynamic_placeholder) do
 73 |         { 'tag' => '1.2.3'}
 74 |       end
 75 | 
 76 |       it 'expands values' do
 77 |         expander = builder.build(static_placeholder)
 78 |         expect(expander.expand('${hostname}', dynamic_placeholders: dynamic_placeholder)).to eq('host_value')
 79 |         expect(expander.expand('${ary_value[0]}.${ary_value[1]}.${ary_value[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
 80 |         expect(expander.expand('${ary_value[-3]}.${ary_value[-2]}.${ary_value[-1]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
 81 |         expect(expander.expand('${hash_value["key1"]}', dynamic_placeholders: dynamic_placeholder)).to eq('val1')
 82 | 
 83 |         expect(expander.expand('${tag}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
 84 |         expect(expander.expand('${tag_parts[0]}.${tag_parts[1]}.${tag_parts[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
 85 |         expect(expander.expand('${tag_parts[-3]}.${tag_parts[-2]}.${tag_parts[-1]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.2.3')
 86 |         expect(expander.expand('${tag_prefix[0]}.${tag_prefix[1]}.${tag_prefix[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('1.1.2.1.2.3')
 87 |         expect(expander.expand('${tag_suffix[0]}.${tag_suffix[1]}.${tag_suffix[2]}', dynamic_placeholders: dynamic_placeholder)).to eq('3.2.3.1.2.3')
 88 |       end
 89 | 
 90 |       it 'does not create expander twice if given the same placeholder' do
 91 |         builder # cached before mock
 92 | 
 93 |         expect(Fluent::Plugin::Prometheus::ExpandBuilder).to receive(:build).with(anything, log: anything).once.and_call_original
 94 |         expander = builder.build(static_placeholder)
 95 |         placeholder = { 'tag' => 'val.test' }
 96 |         expander.expand('${hostname}', dynamic_placeholders: placeholder)
 97 |         expander.expand('${hostname}', dynamic_placeholders: placeholder)
 98 |       end
 99 | 
100 |       it 'creates new expander for each placeholder' do
101 |         builder # cached before mock
102 | 
103 |         expect(Fluent::Plugin::Prometheus::ExpandBuilder).to receive(:build).with(anything, log: anything).twice.and_call_original
104 |         expander = builder.build(static_placeholder)
105 |         expander.expand('${hostname}', dynamic_placeholders: { 'tag' => 'val.test' })
106 |         expander.expand('${hostname}', dynamic_placeholders: { 'tag' => 'val.test2' })
107 |       end
108 |     end
109 |   end
110 | end
111 | 


--------------------------------------------------------------------------------
/spec/fluent/plugin/prometheus_metrics_spec.rb:
--------------------------------------------------------------------------------
  1 | require 'spec_helper'
  2 | require 'fluent/plugin/in_prometheus'
  3 | require 'fluent/test/driver/input'
  4 | 
  5 | require 'net/http'
  6 | 
  7 | describe Fluent::Plugin::PromMetricsAggregator do
  8 | 
  9 |   metrics_worker_1 = %[# TYPE fluentd_status_buffer_queue_length gauge
 10 | # HELP fluentd_status_buffer_queue_length Current buffer queue length.
 11 | fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
 12 | fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
 13 | # TYPE fluentd_status_buffer_total_bytes gauge
 14 | # HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
 15 | fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
 16 | fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
 17 | # TYPE log_counter counter
 18 | # HELP log_counter the number of received logs
 19 | log_counter{worker_id="0",host="0123456789ab",tag="fluent.info"} 1.0
 20 | # HELP empty_metric A metric with no data
 21 | # TYPE empty_metric gauge
 22 | # HELP http_request_duration_seconds The HTTP request latencies in seconds.
 23 | # TYPE http_request_duration_seconds histogram
 24 | http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.005"} 58
 25 | http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.01"} 58
 26 | http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.05"} 59
 27 | http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.1"} 59
 28 | http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="1"} 59
 29 | http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="10"} 59
 30 | http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="+Inf"} 59
 31 | http_request_duration_seconds_sum{code="200",worker_id="0",method="GET"} 0.05046115500000003
 32 | http_request_duration_seconds_count{code="200",worker_id="0",method="GET"} 59
 33 | ]
 34 | 
 35 |   metrics_worker_2 = %[# TYPE fluentd_output_status_buffer_queue_length gauge
 36 | # HELP fluentd_output_status_buffer_queue_length Current buffer queue length.
 37 | fluentd_output_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",type="s3"} 0.0
 38 | fluentd_output_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",type="s3"} 0.0
 39 | # TYPE fluentd_output_status_buffer_total_bytes gauge
 40 | # HELP fluentd_output_status_buffer_total_bytes Current total size of queued buffers.
 41 | fluentd_output_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",type="s3"} 0.0
 42 | fluentd_output_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",type="s3"} 0.0
 43 | ]
 44 | 
 45 |   metrics_worker_3 = %[# TYPE fluentd_status_buffer_queue_length gauge
 46 | # HELP fluentd_status_buffer_queue_length Current buffer queue length.
 47 | fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
 48 | fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
 49 | # TYPE fluentd_status_buffer_total_bytes gauge
 50 | # HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
 51 | fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
 52 | fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
 53 | # HELP http_request_duration_seconds The HTTP request latencies in seconds.
 54 | # TYPE http_request_duration_seconds histogram
 55 | http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.005"} 70
 56 | http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.01"} 70
 57 | http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.05"} 71
 58 | http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.1"} 71
 59 | http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="1"} 71
 60 | http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="10"} 71
 61 | http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="+Inf"} 71
 62 | http_request_duration_seconds_sum{code="200",worker_id="1",method="GET"} 0.05646315600000003
 63 | http_request_duration_seconds_count{code="200",worker_id="1",method="GET"} 71
 64 | ]
 65 | 
 66 |   metrics_merged_1_and_3 = %[# TYPE fluentd_status_buffer_queue_length gauge
 67 | # HELP fluentd_status_buffer_queue_length Current buffer queue length.
 68 | fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
 69 | fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
 70 | fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
 71 | fluentd_status_buffer_queue_length{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
 72 | # TYPE fluentd_status_buffer_total_bytes gauge
 73 | # HELP fluentd_status_buffer_total_bytes Current total size of queued buffers.
 74 | fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
 75 | fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="0",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
 76 | fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-1",plugin_category="output",type="s3"} 0.0
 77 | fluentd_status_buffer_total_bytes{host="0123456789ab",worker_id="1",plugin_id="plugin-2",plugin_category="output",type="s3"} 0.0
 78 | # TYPE log_counter counter
 79 | # HELP log_counter the number of received logs
 80 | log_counter{worker_id="0",host="0123456789ab",tag="fluent.info"} 1.0
 81 | # HELP empty_metric A metric with no data
 82 | # TYPE empty_metric gauge
 83 | # HELP http_request_duration_seconds The HTTP request latencies in seconds.
 84 | # TYPE http_request_duration_seconds histogram
 85 | http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.005"} 58
 86 | http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.01"} 58
 87 | http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.05"} 59
 88 | http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="0.1"} 59
 89 | http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="1"} 59
 90 | http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="10"} 59
 91 | http_request_duration_seconds_bucket{code="200",worker_id="0",method="GET",le="+Inf"} 59
 92 | http_request_duration_seconds_sum{code="200",worker_id="0",method="GET"} 0.05046115500000003
 93 | http_request_duration_seconds_count{code="200",worker_id="0",method="GET"} 59
 94 | http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.005"} 70
 95 | http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.01"} 70
 96 | http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.05"} 71
 97 | http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="0.1"} 71
 98 | http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="1"} 71
 99 | http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="10"} 71
100 | http_request_duration_seconds_bucket{code="200",worker_id="1",method="GET",le="+Inf"} 71
101 | http_request_duration_seconds_sum{code="200",worker_id="1",method="GET"} 0.05646315600000003
102 | http_request_duration_seconds_count{code="200",worker_id="1",method="GET"} 71
103 | ]
104 | 
105 |   describe 'add_metrics' do
106 |     context '1st_metrics' do
107 |       it 'adds all fields' do
108 |         all_metrics = Fluent::Plugin::PromMetricsAggregator.new
109 |         all_metrics.add_metrics(metrics_worker_1)
110 |         result_str = all_metrics.get_metrics
111 | 
112 |         expect(result_str).to eq(metrics_worker_1)
113 |       end
114 |     end
115 |     context '2nd_metrics' do
116 |       it 'append new metrics' do
117 |         all_metrics = Fluent::Plugin::PromMetricsAggregator.new
118 |         all_metrics.add_metrics(metrics_worker_1)
119 |         all_metrics.add_metrics(metrics_worker_2)
120 |         result_str = all_metrics.get_metrics
121 | 
122 |         expect(result_str).to eq(metrics_worker_1 + metrics_worker_2)
123 |       end
124 |     end
125 | 
126 |     context '3rd_metrics' do
127 |       it 'append existing metrics in the right place' do
128 |         all_metrics = Fluent::Plugin::PromMetricsAggregator.new
129 |         all_metrics.add_metrics(metrics_worker_1)
130 |         all_metrics.add_metrics(metrics_worker_2)
131 |         all_metrics.add_metrics(metrics_worker_3)
132 |         result_str = all_metrics.get_metrics
133 | 
134 |         expect(result_str).to eq(metrics_merged_1_and_3 + metrics_worker_2)
135 |       end
136 |     end
137 |   end
138 | end
139 | 


--------------------------------------------------------------------------------
/spec/fluent/plugin/shared.rb:
--------------------------------------------------------------------------------
  1 | 
  2 | BASE_CONFIG = %[
  3 |   @type prometheus
  4 | ]
  5 | 
  6 | SIMPLE_CONFIG = BASE_CONFIG + %[
  7 |   <metric>
  8 |     name simple_foo
  9 |     type counter
 10 |     desc Something foo.
 11 |     key foo
 12 |   </metric>
 13 | ]
 14 | 
 15 | FULL_CONFIG = BASE_CONFIG + %[
 16 |   <metric>
 17 |     name full_foo
 18 |     type counter
 19 |     desc Something foo.
 20 |     key foo
 21 |     <labels>
 22 |       key foo1
 23 |     </labels>
 24 |   </metric>
 25 |   <metric>
 26 |     name full_bar
 27 |     type gauge
 28 |     desc Something bar.
 29 |     key bar
 30 |     initialized true
 31 |     <labels>
 32 |       key foo2
 33 |     </labels>
 34 |   </metric>
 35 |   <metric>
 36 |     name full_baz
 37 |     type summary
 38 |     desc Something baz.
 39 |     key baz
 40 |     initialized true
 41 |     <labels>
 42 |       key foo3
 43 |     </labels>
 44 |   </metric>
 45 |   <metric>
 46 |     name full_qux
 47 |     type histogram
 48 |     desc Something qux.
 49 |     key qux
 50 |     buckets 0.1, 1, 5, 10
 51 |     initialized true
 52 |     <labels>
 53 |       key foo4
 54 |     </labels>
 55 |   </metric>
 56 |   <metric>
 57 |     name full_accessor1
 58 |     type summary
 59 |     desc Something with accessor.
 60 |     key $.foo
 61 |     <labels>
 62 |       key foo5
 63 |     </labels>
 64 |   </metric>
 65 |   <metric>
 66 |     name full_accessor2
 67 |     type counter
 68 |     desc Something with accessor
 69 |     key $.foo
 70 |     initialized true
 71 |     <labels>
 72 |       key foo6
 73 |     </labels>
 74 |   </metric>
 75 |   <metric>
 76 |     name full_accessor3
 77 |     type counter
 78 |     desc Something with accessor and several initialized metrics
 79 |     initialized true
 80 |     <labels>
 81 |       key $.foo
 82 |       key2 $.foo2
 83 |       key3 footix
 84 |     </labels>
 85 |     <initlabels>
 86 |       key foo6
 87 |       key2 foo7
 88 |     </initlabels>
 89 |     <initlabels>
 90 |       key foo8
 91 |       key2 foo9
 92 |     </initlabels>
 93 |   </metric>
 94 |   <labels>
 95 |     test_key test_value
 96 |   </labels>
 97 | ]
 98 | 
 99 | PLACEHOLDER_CONFIG = BASE_CONFIG + %[
100 |   <metric>
101 |     name placeholder_foo
102 |     type counter
103 |     desc Something foo.
104 |     key foo
105 |     initialized true
106 |     <labels>
107 |       foo ${foo}
108 |       foo2 foo2
109 |     </labels>
110 |     <initlabels>
111 |       tag tag
112 |       foo foo
113 |     </initlabels>
114 |   </metric>
115 |   <labels>
116 |     tag ${tag}
117 |     hostname ${hostname}
118 |     workerid ${worker_id}
119 |   </labels>
120 | ]
121 | 
122 | ACCESSOR_CONFIG = BASE_CONFIG + %[
123 |   <metric>
124 |     name accessor_foo
125 |     type counter
126 |     desc Something foo.
127 |     key foo
128 |     <labels>
129 |       foo $.foo
130 |     </labels>
131 |   </metric>
132 | ]
133 | 
134 | COUNTER_WITHOUT_KEY_CONFIG = BASE_CONFIG + %[
135 |   <metric>
136 |     name without_key_foo
137 |     type counter
138 |     desc Something foo.
139 |   </metric>
140 | ]
141 | 
142 | shared_examples_for 'output configuration' do
143 |   context 'base config' do
144 |     let(:config) { BASE_CONFIG }
145 |     it { expect { driver }.not_to raise_error }
146 |   end
147 | 
148 |   context 'with simple configuration' do
149 |     let(:config) { SIMPLE_CONFIG }
150 |     it { expect { driver }.not_to raise_error }
151 |   end
152 | 
153 |   context 'with full configuration' do
154 |     let(:config) { FULL_CONFIG }
155 |     it { expect { driver }.not_to raise_error }
156 |   end
157 | 
158 |   context 'with placeholder configuration' do
159 |     let(:config) { PLACEHOLDER_CONFIG }
160 |     it { expect { driver }.not_to raise_error }
161 |   end
162 | 
163 |   context 'with accessor configuration' do
164 |     let(:config) {  ACCESSOR_CONFIG }
165 |     it { expect { driver }.not_to raise_error }
166 |   end
167 | 
168 |   describe 'with counter without key configuration' do
169 |     let(:config) { COUNTER_WITHOUT_KEY_CONFIG }
170 |     it { expect { driver }.not_to raise_error }
171 |   end
172 | 
173 |   context 'with unknown type' do
174 |     let(:config) do
175 |       BASE_CONFIG + %[
176 |       <metric>
177 |         type foo
178 |       </metric>
179 |       ]
180 |     end
181 |     it { expect { driver }.to raise_error(Fluent::ConfigError) }
182 |   end
183 | 
184 | 
185 |   context 'with missing <initlabels>' do
186 |     let(:config) do
187 |       BASE_CONFIG + %[
188 |       <metric>
189 |         name simple_foo
190 |         type counter
191 |         desc Something foo but incorrect
192 |         key foo
193 |         initialized true
194 |         <labels>
195 |           key $.accessor
196 |         </labels>
197 |       </metric>
198 |       ]
199 |     end
200 |     it { expect { driver }.to raise_error(Fluent::ConfigError) }
201 |   end
202 | 
203 |   context 'with RecordAccessor set in <initlabels>' do
204 |     let(:config) do
205 |       BASE_CONFIG + %[
206 |       <metric>
207 |         name simple_foo
208 |         type counter
209 |         desc Something foo but incorrect
210 |         key foo
211 |         initialized true
212 |         <labels>
213 |           key $.accessor
214 |         </labels>
215 |         <initlabels>
216 |           key $.accessor2
217 |         <initlabels>
218 |       </metric>
219 |       ]
220 |     end
221 |     it { expect { driver }.to raise_error(Fluent::ConfigError) }
222 |   end
223 | 
224 |   context 'with PlaceHolder set in <initlabels>' do
225 |     let(:config) do
226 |       BASE_CONFIG + %[
227 |       <metric>
228 |         name simple_foo
229 |         type counter
230 |         desc Something foo but incorrect
231 |         key foo
232 |         initialized true
233 |         <labels>
234 |           key ${foo}
235 |         </labels>
236 |         <initlabels>
237 |           key ${foo}
238 |         <initlabels>
239 |       </metric>
240 |       ]
241 |     end
242 |     it { expect { driver }.to raise_error(Fluent::ConfigError) }
243 |   end
244 | 
245 |   context 'with non RecordAccessor label set in <initlabels>' do
246 |     let(:config) do
247 |       BASE_CONFIG + %[
248 |       <metric>
249 |         name simple_foo
250 |         type counter
251 |         desc Something foo but incorrect
252 |         key foo
253 |         initialized true
254 |         <labels>
255 |           key $.accessor
256 |           key2 foo2
257 |         </labels>
258 |         <initlabels>
259 |           key foo
260 |           key2 foo2
261 |         <initlabels>
262 |       </metric>
263 |       ]
264 |     end
265 |     it { expect { driver }.to raise_error(Fluent::ConfigError) }
266 |   end
267 | 
268 |   context 'with non-matching label keys set in <initlabels>' do
269 |     let(:config) do
270 |       BASE_CONFIG + %[
271 |       <metric>
272 |         name simple_foo
273 |         type counter
274 |         desc Something foo but incorrect
275 |         key foo
276 |         initialized true
277 |         <labels>
278 |           key $.accessor
279 |         </labels>
280 |         <initlabels>
281 |           key2 foo
282 |         <initlabels>
283 |       </metric>
284 |       ]
285 |     end
286 |     it { expect { driver }.to raise_error(Fluent::ConfigError) }
287 |   end
288 | end
289 | 
290 | shared_examples_for 'instruments record' do
291 |   before do
292 |     driver.run(default_tag: tag) { driver.feed(event_time, message) }
293 |   end
294 | 
295 |   context 'full config' do
296 |     let(:config) { FULL_CONFIG }
297 |     let(:counter) { registry.get(:full_foo) }
298 |     let(:gauge) { registry.get(:full_bar) }
299 |     let(:summary) { registry.get(:full_baz) }
300 |     let(:histogram) { registry.get(:full_qux) }
301 |     let(:summary_with_accessor) { registry.get(:full_accessor1) }
302 |     let(:counter_with_accessor) { registry.get(:full_accessor2) }
303 |     let(:counter_with_two_accessors) { registry.get(:full_accessor3) }
304 | 
305 |     it 'adds all metrics' do
306 |       expect(registry.metrics.map(&:name)).to eq(%i[full_foo full_bar full_baz full_qux full_accessor1 full_accessor2 full_accessor3])
307 |       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
308 |       expect(gauge).to be_kind_of(::Prometheus::Client::Metric)
309 |       expect(summary).to be_kind_of(::Prometheus::Client::Metric)
310 |       expect(summary_with_accessor).to be_kind_of(::Prometheus::Client::Metric)
311 |       expect(counter_with_accessor).to be_kind_of(::Prometheus::Client::Metric)
312 |       expect(counter_with_two_accessors).to be_kind_of(::Prometheus::Client::Metric)
313 |       expect(histogram).to be_kind_of(::Prometheus::Client::Metric)
314 |     end
315 | 
316 |     it 'instruments counter metric' do
317 |       expect(counter.type).to eq(:counter)
318 |       expect(counter.get(labels: {test_key: 'test_value', key: 'foo1'})).to be_kind_of(Numeric)
319 |       expect(counter_with_accessor.get(labels: {test_key: 'test_value', key: 'foo6'})).to be_kind_of(Numeric)
320 |       expect(counter_with_two_accessors.get(labels: {test_key: 'test_value', key: 'foo6', key2: 'foo7', key3: 'footix'})).to be_kind_of(Numeric)
321 |     end
322 | 
323 |     it 'instruments gauge metric' do
324 |       expect(gauge.type).to eq(:gauge)
325 |       expect(gauge.get(labels: {test_key: 'test_value', key: 'foo2'})).to eq(100)
326 |     end
327 | 
328 |     it 'instruments summary metric' do
329 |       expect(summary.type).to eq(:summary)
330 |       expect(summary.get(labels: {test_key: 'test_value', key: 'foo3'})).to be_kind_of(Hash)
331 |       expect(summary_with_accessor.get(labels: {test_key: 'test_value', key: 'foo5'})["sum"]).to eq(100)
332 |     end
333 | 
334 |     it 'instruments histogram metric' do
335 |       driver.run(default_tag: tag) do
336 |         4.times { driver.feed(event_time, message) }
337 |       end
338 | 
339 |       expect(histogram.type).to eq(:histogram)
340 |       expect(histogram.get(labels: {test_key: 'test_value', key: 'foo4'})).to be_kind_of(Hash)
341 |       expect(histogram.get(labels: {test_key: 'test_value', key: 'foo4'})["10"]).to eq(5) # 4 + `es` in before
342 |     end
343 |   end
344 | 
345 |   context 'placeholder config' do
346 |     let(:config) { PLACEHOLDER_CONFIG }
347 |     let(:counter) { registry.get(:placeholder_foo) }
348 | 
349 |     it 'expands placeholders with record values' do
350 |       expect(registry.metrics.map(&:name)).to eq([:placeholder_foo])
351 |       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
352 |       key, _ = counter.values.find {|k,v| v ==  100 }
353 |       expect(key).to be_kind_of(Hash)
354 |       expect(key[:tag]).to eq(tag)
355 |       expect(key[:hostname]).to be_kind_of(String)
356 |       expect(key[:hostname]).not_to eq("${hostname}")
357 |       expect(key[:hostname]).not_to be_empty
358 |       expect(key[:foo]).to eq("100")
359 |     end
360 |   end
361 | 
362 |   context 'accessor config' do
363 |     let(:config) { ACCESSOR_CONFIG }
364 |     let(:counter) { registry.get(:accessor_foo) }
365 | 
366 |     it 'expands accessor with record values' do
367 |       expect(registry.metrics.map(&:name)).to eq([:accessor_foo])
368 |       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
369 |       key, _ = counter.values.find {|k,v| v ==  100 }
370 |       expect(key).to be_kind_of(Hash)
371 |       expect(key[:foo]).to eq("100")
372 |     end
373 |   end
374 | 
375 |   context 'counter_without config' do
376 |     let(:config) { COUNTER_WITHOUT_KEY_CONFIG }
377 |     let(:counter) { registry.get(:without_key_foo) }
378 | 
379 |     it 'just increments by 1' do
380 |       expect(registry.metrics.map(&:name)).to eq([:without_key_foo])
381 |       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
382 |       _, value = counter.values.find {|k,v| k == {} }
383 |       expect(value).to eq(1)
384 |     end
385 |   end
386 | end
387 | 
388 | shared_examples_for 'initalized metrics' do
389 |   before do
390 |     driver.run(default_tag: tag)
391 |   end
392 | 
393 |   context 'full config' do
394 |     let(:config) { FULL_CONFIG }
395 |     let(:counter) { registry.get(:full_foo) }
396 |     let(:gauge) { registry.get(:full_bar) }
397 |     let(:summary) { registry.get(:full_baz) }
398 |     let(:histogram) { registry.get(:full_qux) }
399 |     let(:summary_with_accessor) { registry.get(:full_accessor1) }
400 |     let(:counter_with_accessor) { registry.get(:full_accessor2) }
401 |     let(:counter_with_two_accessors) { registry.get(:full_accessor3) }
402 |   
403 |     it 'adds all metrics' do
404 |       expect(registry.metrics.map(&:name)).to eq(%i[full_foo full_bar full_baz full_qux full_accessor1 full_accessor2 full_accessor3])
405 |       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
406 |       expect(gauge).to be_kind_of(::Prometheus::Client::Metric)
407 |       expect(summary).to be_kind_of(::Prometheus::Client::Metric)
408 |       expect(summary_with_accessor).to be_kind_of(::Prometheus::Client::Metric)
409 |       expect(counter_with_accessor).to be_kind_of(::Prometheus::Client::Metric)
410 |       expect(counter_with_two_accessors).to be_kind_of(::Prometheus::Client::Metric)
411 |       expect(histogram).to be_kind_of(::Prometheus::Client::Metric)
412 |     end
413 | 
414 |     it 'tests uninitialized metrics' do
415 |       expect(counter.values).to eq({})
416 |       expect(summary_with_accessor.values).to eq({})
417 |     end
418 | 
419 |     it 'tests initialized metrics' do
420 |       expect(gauge.values).to eq({{:key=>"foo2", :test_key=>"test_value"}=>0.0})
421 |       expect(summary.values).to eq({:key=>"foo3", :test_key=>"test_value"}=>{"count"=>0.0, "sum"=>0.0})
422 |       expect(histogram.values).to eq({:key=>"foo4", :test_key=>"test_value"} => {"+Inf"=>0.0, "0.1"=>0.0, "1"=>0.0, "10"=>0.0, "5"=>0.0, "sum"=>0.0})
423 |       expect(counter_with_accessor.values).to eq({{:key=>"foo6", :test_key=>"test_value"}=>0.0})
424 |       expect(counter_with_two_accessors.values).to eq({{:key=>"foo6", :key2=>"foo7", :key3=>"footix", :test_key=>"test_value"}=>0.0, {:key=>"foo8", :key2=>"foo9", :key3=>"footix", :test_key=>"test_value"}=>0.0})
425 |     end
426 |   end
427 | 
428 |   context 'placeholder config' do
429 |     let(:config) { PLACEHOLDER_CONFIG }
430 |     let(:counter) { registry.get(:placeholder_foo) }
431 | 
432 |     it 'expands placeholders with record values' do
433 |       expect(registry.metrics.map(&:name)).to eq([:placeholder_foo])
434 |       expect(counter).to be_kind_of(::Prometheus::Client::Metric)
435 | 
436 |       key, _ = counter.values.find {|k,v| v ==  0.0 }
437 |       expect(key).to be_kind_of(Hash)
438 |       expect(key[:foo]).to eq("foo")
439 |       expect(key[:foo2]).to eq("foo2")
440 |       expect(key[:hostname]).to be_kind_of(String)
441 |       expect(key[:hostname]).not_to eq("${hostname}")
442 |       expect(key[:hostname]).not_to be_empty
443 |       expect(key[:workerid]).to be_kind_of(String)
444 |       expect(key[:workerid]).not_to eq("${worker_id}")
445 |       expect(key[:workerid]).not_to be_empty
446 |       expect(key[:tag]).to eq("tag")
447 |     end
448 |   end
449 | end


--------------------------------------------------------------------------------
/spec/spec_helper.rb:
--------------------------------------------------------------------------------
 1 | $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 2 | require 'fluent/test'
 3 | require 'fluent/test/helpers'
 4 | require 'fluent/plugin/prometheus'
 5 | 
 6 | # Disable Test::Unit
 7 | Test::Unit::AutoRunner.need_auto_run = false
 8 | 
 9 | Fluent::Test.setup
10 | include Fluent::Test::Helpers
11 | 


--------------------------------------------------------------------------------