├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── arch_federate.png ├── arch_hybrid.png ├── arch_remotewrite.png ├── arch_simple.png ├── modinput_prometheus ├── README.md ├── README │ └── inputs.conf.spec ├── default │ ├── app.conf │ ├── inputs.conf │ ├── props.conf │ └── transforms.conf ├── linux_x86_64 │ └── bin │ │ └── README ├── metadata │ └── default.meta └── static │ ├── appIcon.png │ └── appIcon_2x.png ├── prometheus └── prometheus.go └── prometheusrw ├── prometheusrw.go └── prometheusrw_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | modinput_prometheus/linux_x86_64/bin/prometheusrw 2 | modinput_prometheus/linux_x86_64/bin/prometheus 3 | venv/* 4 | *.tar.gz 5 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.11 as builder 2 | 3 | RUN apt-get update; DEBIAN_FRONTEND=noninteractive apt-get install -y git 4 | WORKDIR /go/src/splunk_modinput_prometheus 5 | COPY . /go/src/splunk_modinput_prometheus 6 | 7 | RUN go get github.com/gogo/protobuf/proto 8 | RUN go get github.com/golang/snappy 9 | RUN go get github.com/prometheus/common/model 10 | RUN go get github.com/prometheus/prometheus/prompb 11 | RUN go get github.com/gobwas/glob 12 | RUN go get github.com/prometheus/prometheus/pkg/textparse 13 | 14 | FROM debian:stretch-slim 15 | COPY --from=builder /go/src/splunk_modinput_prometheus/modinput_prometheus/ /opt/modinput_prometheus 16 | 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | build: 2 | make -B prometheusrw 3 | make -B prometheus 4 | 5 | prometheusrw: prometheusrw/prometheusrw.go 6 | go build -o ./out/prometheusrw ./prometheusrw/prometheusrw.go 7 | 8 | mv ./out/prometheusrw modinput_prometheus/linux_x86_64/bin/ 9 | 10 | prometheus: prometheus/prometheus.go 11 | go build -o ./out/prometheus ./prometheus/prometheus.go 12 | mv ./out/prometheus modinput_prometheus/linux_x86_64/bin/ 13 | 14 | package: 15 | make build 16 | tar cvfz modinput_prometheus.tar.gz modinput_prometheus 17 | 18 | # To use the validate target, install a Python venv in this directory, and install splunk-appinspect within it 19 | # http://dev.splunk.com/view/appinspect/SP-CAAAFAW#installinvirtualenv 20 | validate: 21 | make package 22 | bash -c 'source venv/bin/activate && splunk-appinspect inspect ./modinput_prometheus.tar.gz' 23 | 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deprecation Warning # 2 | Unfortunately, my available time to maintain this project is very low. 3 | 4 | I would suggest that you investigate using the Open Telemetry collector to achieve Prometheus metrics to Splunk and see if it meets your needs first: 5 | * https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/prometheusreceiver 6 | * https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/splunkhecexporter 7 | 8 | # Prometheus Metrics for Splunk 9 | 10 | Prometheus [prometheus.io](https://prometheus.io), a [Cloud Native Computing Foundation](https://cncf.io/) project, is a systems and service monitoring system. It collects metrics from configured targets at given intervals, evaluates rule expressions, displays the results, and can trigger alerts if some condition is observed to be true. 11 | 12 | Splunk [splunk.com](https://www.splunk.com) is a platform for machine data analysis, providing real-time visibility, actionable insights and intelligence across all forms of machine data. Splunk Enterprise since version 7.0 includes the Metrics store for large-scale capture and analysis of time-series metrics alongside log data. 13 | 14 | This Splunk add-on provides two modular inputs to ingest Splunk metrics from Prometheus: 15 | 16 | **[prometheus://]** 17 | 18 | A scraping input which polls a Prometheus exporter and indexes all exposed metrics in Splunk. 19 | 20 | It is also designed to be able to poll a Prometheus servers "/federate" endpoint, so that Prometheus can be responsible for all the metrics gathering details (e.g. service discovery and label rewriting) and Splunk can easily ingest a desired subset of metrics from Prometheus at the desired resolution. In this way it acts much like Prometheus hierarchical federation. 21 | 22 | It will also successfully scrape a statically configured Prometheus exporter, and for this use case does not require a Prometheus server at all. 23 | 24 | **[prometheusrw://]** 25 | 26 | A bridge so that the Prometheus remote-write feature can continuously push metrics to a Splunk Enterprise system. When installed and enabled, this input will add a new listening port to your Splunk server which can be the remote write target for multiple Prometheus servers. 27 | 28 | It has been designed to mimic the Splunk HTTP Event Collector for it's configuration, however the endpoint is much simpler as it only supports Prometheus remote-write. The HEC is not used for this as Prometheus remote-write requires Speedy compression and Protocol Buffer encoding, both of which do not work with the HEC. 29 | 30 | ## Requirements 31 | 32 | - Splunk 8.x and above 33 | - Prometheus 2.x 34 | - Recent Linux x64 35 | 36 | The most testing has been performed on Splunk 8.2 and Prometheus 2.36 on Ubuntu 20.04 37 | 38 | ## Architecture overview 39 | 40 | ### Static exporter 41 | In this configuration, the modular input can poll any statically configured Prometheus exporter at a defined interval. 42 | 43 | Pros: 44 | 45 | - Simple 46 | - Requires no Prometheus server 47 | 48 | Cons: 49 | 50 | - Static configuration only -- very manual to add lots of systems 51 | - HA of Splunk polling is difficult 52 | 53 | ![](https://raw.githubusercontent.com/lukemonahan/splunk_modinput_prometheus/master/arch_simple.png) 54 | 55 | ### Federate server 56 | With this configuration, the modular input is setup to poll a Prometheus server that is exposing the metrics from exporters and other Prometheus servers on it's federate endpoint. 57 | 58 | Pros: 59 | 60 | - Allows Prometheus to handle service discovery and other low-level functions 61 | - High level of control of what Splunk gathers and when using polling interval and match vectors 62 | - Allows scenarios such as using Prometheus to gather high-resolution metrics, and ingesting into Splunk at reduced frequency 63 | 64 | Cons: 65 | 66 | - HA of Splunk polling is difficult 67 | - Could run into scalability issues if you want to gather large numbers of metrics from a single Prometheus server at a high rate 68 | 69 | ![](https://raw.githubusercontent.com/lukemonahan/splunk_modinput_prometheus/master/arch_federate.png) 70 | 71 | ### Prometheus remote-write 72 | With this configuration, Prometheus pushes metrics to Splunk with it's remote_write functionality. 73 | 74 | Pros: 75 | 76 | - Most efficient way to ingest all, or nearly all, metrics from a Prometheus server into Splunk 77 | - HA and scaling of Splunk ingestion is achievable with HTTP load balancers 78 | 79 | Cons: 80 | 81 | - Must send metrics to Splunk with same frequency as they are gathered into Prometheus 82 | 83 | ![](https://raw.githubusercontent.com/lukemonahan/splunk_modinput_prometheus/master/arch_remotewrite.png) 84 | 85 | ### Hybrid 86 | All metrics gathered by the above methods are in a consistent format in Splunk, and reporting over them will be no different no matter how they are gathered. Because of this, different ways of delivering metrics for different use cases could be implemented. 87 | 88 | ![](https://raw.githubusercontent.com/ltmon/splunk_modinput_prometheus/master/arch_hybrid.png) 89 | 90 | ## Download 91 | 92 | This add-on will be hosted at apps.splunk.com in the near future. It will be uploaded there when some further testing has been completed. 93 | 94 | In the meantime, the latest build is available in the Github releases tab. 95 | 96 | ## Build 97 | 98 | This assumes you have a relatively up-to-date Go build environment set up. 99 | 100 | You will need some dependencies installed: 101 | 102 | ``` 103 | $ go get github.com/gogo/protobuf/proto 104 | $ go get github.com/golang/snappy 105 | $ go get github.com/prometheus/common/model 106 | $ go get github.com/prometheus/prometheus/prompb 107 | $ go get github.com/gobwas/glob 108 | $ go get github.com/prometheus/prometheus/model/textparse 109 | ``` 110 | 111 | The "build" make target will build the modular input binaries, and copy them into the correct place in `modinput_prometheus`, which forms the root of the Splunk app. 112 | 113 | ``` 114 | $ make build 115 | ``` 116 | 117 | ## Install and configure 118 | 119 | This add-on is installed just like any Splunk app: either through the web UI, deployment server or copying directly to $SPLUNK_HOME/etc/apps. 120 | 121 | We recommend installing on a heavy forwarder, so the processing of events into metrics occurs at the collection point and not on indexers. The app is only tested on a heavy instance so far, but if you use a Universal Forwarder be sure to also install on your HFs/Indexers as there are index-time transforms to process the received metrics. 122 | 123 | All available parameters for the modular inputs are described in [inputs.conf.spec](https://github.com/ltmon/splunk_modinput_prometheus/blob/master/modinput_prometheus/README/inputs.conf.spec). 124 | 125 | ### Static exporter 126 | 127 | The most basic configuration to poll a Prometheus exporter. 128 | 129 | e.g. 130 | 131 | ``` 132 | [prometheus://java-client-1] 133 | URI = http://myhost:1234/metrics 134 | index = prometheus 135 | sourcetype = prometheus:metric 136 | host = myhost 137 | interval = 60 138 | disabled = 0 139 | ``` 140 | 141 | The index should be a "metrics" type index. The sourcetype should be prometheus:metric, which is configured in the app to recognize the data format and convert it to Splunk metrics. 142 | 143 | ### Federate server 144 | 145 | This configuration is to gather all metrics from a Prometheus server. At least one valid "match" must be supplied in order to get any data from a Prometheus federation endpoint. Eatch "match" is entered with semicolon separation in the Splunk configuration. The example "match" string given here matches all metrics. You can learn more about how to configure metrics matching at: https://prometheus.io/docs/prometheus/latest/querying/basics/#instant-vector-selectors 146 | 147 | ``` 148 | [prometheus://prom-server-1] 149 | URI = http://myhost:9090/federate 150 | match = {__name__=~"..*"} 151 | index = prometheus 152 | sourcetype = prometheus:metric 153 | host = myhost 154 | interval = 60 155 | disabled = 0 156 | ``` 157 | 158 | ### Prometheus remote-write 159 | 160 | Only one HTTP server is ever run, which is configured by the `[prometheusrw]` input stanza. The individual inputs are then distinguished by bearer tokens. At least one of the individual inputs must be configured, and a matching bearer token must be supplied from Prometheus in order to direct the received metrics to that input. 161 | 162 | e.g. 163 | 164 | ``` 165 | [prometheusrw] 166 | port = 8098 167 | maxClients = 10 168 | disabled = 0 169 | 170 | [prometheusrw://testing] 171 | bearerToken = ABC123 172 | index = prometheus 173 | whitelist = * 174 | sourcetype = prometheus:metric 175 | disabled = 0 176 | 177 | [prometheusrw://another] 178 | bearerToken = DEF456 179 | index = another_metrics_index 180 | whitelist = net* 181 | sourcetype = prometheus:metric 182 | disabled = 0 183 | 184 | [prometheusrw://parsed] 185 | bearerToken = PAR042 186 | index = another_metrics_index 187 | whitelist = * 188 | sourcetype = prometheus:metric 189 | metricNamePrefix = DEV. 190 | metricNameParse = true 191 | disabled = 0 192 | ``` 193 | 194 | This starts the HTTP listener on port 8098, and any metrics coming in with a bearer token of "ABC123" will be directed to the "testing" input, wheras any received with a bearer token of "DEF456" will be directed to the "another" input. Not including a bearer token, or a non-matching token, will result in a HTTP 401 (Unauthorized). 195 | 196 | At least one whitelist should be supplied, and a blacklist is also available. Whitelist and blacklist are comma-separated globs that match against an incoming metric name. 197 | 198 | Although the input does allow some basic whitelist and blacklist behaviour against the metric name before ingesting in Splunk, it will be more efficient and flexible to do this on the Prometheus server using write_relabel_configs if that is possible. An example of dropping metrics withis way is shown in the configuration below. 199 | 200 | In your Prometheus runtime YML file, ensure the following is set to start sending metrics to the prometheusrw Splunk input: 201 | 202 | ```yaml 203 | remote_write: 204 | - url: "http://:8098" 205 | bearer_token: "ABC123" 206 | write_relabel_configs: 207 | - source_labels: [__name__] 208 | regex: expensive.* 209 | action: drop 210 | ``` 211 | 212 | Full details of available Prometheus options are at: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#%3Cremote_write%3E 213 | 214 | ## Known Limitations 215 | 216 | - Only Linux on x86_64 is tested for now. 217 | - Validation of configuration is not very advanced -- incorrect configuration will not work with little indication as to why. 218 | - Only some basic HTTP options are supported, which should be fine for basic Prometheus endpoints but may not work with various proxying methods etc. 219 | - No user interface for configuring inputs at this stage, you'll have to do it all via inputs.conf. 220 | - This add-on does not make much sense to run in Splunk Cloud, so no compatibility there. You should run this on a local Heavy Forwarder and forward the generated metrics to Splunk Cloud. 221 | 222 | # Binary File Declaration 223 | 224 | All binaries built from source code at: https://github.com/lukemonahan/splunk_modinput_prometheus 225 | -------------------------------------------------------------------------------- /arch_federate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lukemonahan/splunk_modinput_prometheus/5162b7eda3b3450e53888aba0f9a8ebc5abefbaa/arch_federate.png -------------------------------------------------------------------------------- /arch_hybrid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lukemonahan/splunk_modinput_prometheus/5162b7eda3b3450e53888aba0f9a8ebc5abefbaa/arch_hybrid.png -------------------------------------------------------------------------------- /arch_remotewrite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lukemonahan/splunk_modinput_prometheus/5162b7eda3b3450e53888aba0f9a8ebc5abefbaa/arch_remotewrite.png -------------------------------------------------------------------------------- /arch_simple.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lukemonahan/splunk_modinput_prometheus/5162b7eda3b3450e53888aba0f9a8ebc5abefbaa/arch_simple.png -------------------------------------------------------------------------------- /modinput_prometheus/README.md: -------------------------------------------------------------------------------- 1 | # Splunk Modular Input for Prometheus 2 | 3 | A modular input which allows you to gather metrics from Prometheus servers and exporters, either through polling or acting as a remote write target. 4 | 5 | For full documentation see: https://github.com/lukemonahan/splunk_modinput_prometheus 6 | -------------------------------------------------------------------------------- /modinput_prometheus/README/inputs.conf.spec: -------------------------------------------------------------------------------- 1 | [prometheusrw] 2 | * The global configuration of the Prometheus remote-write reciving port. 3 | * Setting this input up is required, however you also need to set up a specific input for each bearer token that is sent from a Prometheus server. 4 | 5 | port = 6 | * The TCP port we will listen to for connections from the Prometheus remote-write client. It will listen on all interfaces (":"). The listener is on the root path (i.e. http://:/) 7 | 8 | maxClients = 9 | * This dictates the number of requests that will be processed and converted to metrics in parallel. Any incoming remote write requests will be queued. 10 | 11 | enableTLS = <0|1> 12 | * Enable listening on HTTPS. If set, you need to set a certFile and keyFile. 13 | 14 | certFile = 15 | * The server certificate. If a CA is required, it should be concatenated in this file. $SPLUNK_HOME environment substitution is supported. 16 | 17 | keyFile = 18 | * The server certificate key. It must have no password. $SPLUNK_HOME environment substitution is supported. 19 | 20 | [prometheusrw://] 21 | * A specific Promethes remote-write input. This requires the global [prometheusrw] input to also be enabled. 22 | 23 | bearerToken = 24 | * Incoming Prometheus remote-write requests will be sent to this input if they have a matching bearer_token. Requests with no bearer token are treated as 401 Unauthorized. 25 | 26 | whitelist = ,,... 27 | * A basic whitelist of metrics to ingest from the incoming stream. 28 | * Comma-separated globs of matching metric names 29 | * Must have at least one of "*" to match all metrics 30 | * It is recommended to configure suppression of metrics in Prometheus itself using write_relabel_configs, however this configuration provides a way for the Splunk administrator to whitelist specific metrics also. 31 | 32 | blacklist = ,,... 33 | * A basic whitelist of metrics to discared from the incoming stream. 34 | * Comma-separated globs of matching metric names 35 | * Applied after the whitelist 36 | * It is recommended to configure suppression of metrics in Prometheus itself using write_relabel_configs, however this configuration provides a way for the Splunk administrator to blacklist specific metrics also. 37 | 38 | metricNamePrefix = 39 | * A prefix for all metric names from the present stanza 40 | * Use a ending "." in order to have an extra level on metric name tree display (eg: DEV.) 41 | 42 | metricNameParse = 43 | * A parser from prometheus default metric name separated by a "_" to a Splunk metric name separated by a "." 44 | * After activation your prometheus metrics should display in a tree folded manner inside Splunk metric dashboard. 45 | 46 | [prometheus://] 47 | * An outgoing connection to a Prometheus server (e.g. federated) or to a Prometheus exporter 48 | * Any metrics series found at this endpoint will be converted to Splunk metrics and indexed 49 | 50 | URI = 51 | * The full URI of the exporter to connect to 52 | 53 | match = ,,... 54 | * Match expressions, semicolon separated 55 | * At least one valid match expression must be included if you are connecting to a Prometheus federate endpoint 56 | * They are usually ignored for most exporters, however 57 | 58 | insecureSkipVerify = <0|1> 59 | * If the URI is HTTPS, this controls whether the server certificate must be verified in order to continue 60 | 61 | httpTimeout = 62 | * Number of milliseconds to wait for HTTP timeout. Defaults to 500 if not set. 63 | -------------------------------------------------------------------------------- /modinput_prometheus/default/app.conf: -------------------------------------------------------------------------------- 1 | [install] 2 | state = enabled 3 | build = 11 4 | 5 | [ui] 6 | is_visible = false 7 | label = Prometheus Metrics for Splunk 8 | 9 | [launcher] 10 | author=Luke Monahan 11 | description=This is a modular input that enables Prometheus servers to remote-write to Splunk, creating Splunk metrics, and for Splunk to poll a Prometheus exporter or federate server in order to scrape metrics. 12 | version=1.0.1 13 | 14 | [package] 15 | id = modinput_prometheus 16 | -------------------------------------------------------------------------------- /modinput_prometheus/default/inputs.conf: -------------------------------------------------------------------------------- 1 | [prometheusrw] 2 | port = 8098 3 | maxClients = 10 4 | disabled = 1 5 | 6 | [prometheusrw://example-rw] 7 | bearerToken = ABC123 8 | index = prometheus 9 | whitelist = * 10 | sourcetype = prometheus:metric 11 | disabled = 1 12 | 13 | [prometheus://example] 14 | URI = http://localhost:9323/metrics 15 | index = prometheus 16 | sourcetype = prometheus:metric 17 | interval = 30 18 | disabled = 1 19 | 20 | [prometheus://example-federate] 21 | URI = http://localhost:9090/federate 22 | match = {__name__=~"..*"} 23 | index = prometheus 24 | sourcetype = prometheus:metric 25 | interval = 60 26 | disabled = 1 27 | -------------------------------------------------------------------------------- /modinput_prometheus/default/props.conf: -------------------------------------------------------------------------------- 1 | [prometheus:metric] 2 | TIME_FORMAT = %s%3N 3 | TIME_PREFIX = }\s[\d\-\.]+\s 4 | TRANSFORMS-prometheus_to_metric = prometheus_metric_name_value, prometheus_metric_dims 5 | NO_BINARY_CHECK = true 6 | SHOULD_LINEMERGE = false 7 | pulldown_type = 1 8 | category = Metrics 9 | -------------------------------------------------------------------------------- /modinput_prometheus/default/transforms.conf: -------------------------------------------------------------------------------- 1 | [prometheus_metric_name_value] 2 | REGEX = ^([^\s{]+)({[^}]+})? ([\d\.\-]+) 3 | FORMAT = metric_name::$1 ::$2 _value::$3 4 | WRITE_META = true 5 | 6 | [prometheus_metric_dims] 7 | REGEX = ([^{=]+)="([^"]+)",? 8 | FORMAT = $1::$2 9 | REPEAT_MATCH = true 10 | WRITE_META = true 11 | -------------------------------------------------------------------------------- /modinput_prometheus/linux_x86_64/bin/README: -------------------------------------------------------------------------------- 1 | Binaries will go here 2 | -------------------------------------------------------------------------------- /modinput_prometheus/metadata/default.meta: -------------------------------------------------------------------------------- 1 | [] 2 | access = read : [ * ], write : [ admin ] 3 | export = system 4 | -------------------------------------------------------------------------------- /modinput_prometheus/static/appIcon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lukemonahan/splunk_modinput_prometheus/5162b7eda3b3450e53888aba0f9a8ebc5abefbaa/modinput_prometheus/static/appIcon.png -------------------------------------------------------------------------------- /modinput_prometheus/static/appIcon_2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lukemonahan/splunk_modinput_prometheus/5162b7eda3b3450e53888aba0f9a8ebc5abefbaa/modinput_prometheus/static/appIcon_2x.png -------------------------------------------------------------------------------- /prometheus/prometheus.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "crypto/tls" 6 | "encoding/xml" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "log" 11 | "math" 12 | "net/http" 13 | "os" 14 | "strconv" 15 | "strings" 16 | "time" 17 | 18 | "github.com/prometheus/prometheus/model/textparse" 19 | ) 20 | 21 | // Structs to hold XML parsing of input from Splunk 22 | type input struct { 23 | XMLName xml.Name `xml:"input"` 24 | ServerHost string `xml:"server_host"` 25 | ServerURI string `xml:"server_uri"` 26 | SessionKey string `xml:"session_key"` 27 | CheckpointDir string `xml:"checkpoint_dir"` 28 | Configuration configuration `xml:"configuration"` 29 | } 30 | 31 | type configuration struct { 32 | XMLName xml.Name `xml:"configuration"` 33 | Stanzas []stanza `xml:"stanza"` 34 | } 35 | 36 | type stanza struct { 37 | XMLName xml.Name `xml:"stanza"` 38 | Params []param `xml:"param"` 39 | Name string `xml:"name,attr"` 40 | } 41 | 42 | type param struct { 43 | XMLName xml.Name `xml:"param"` 44 | Name string `xml:"name,attr"` 45 | Value string `xml:",chardata"` 46 | } 47 | 48 | // End XML structs 49 | 50 | // Struct to store final config 51 | type inputConfig struct { 52 | URI string 53 | Match []string 54 | InsecureSkipVerify bool 55 | Index string 56 | Sourcetype string 57 | Host string 58 | HttpTimeout int64 59 | } 60 | 61 | var ( 62 | // http://docs.splunk.com/Documentation/Splunk/7.2.1/AdvancedDev/ModInputsLog 63 | logInfo = log.New(os.Stderr, "INFO (prometheus) ", 0) 64 | logDebug = log.New(os.Stderr, "DEBUG (prometheus) ", 0) 65 | logError = log.New(os.Stderr, "ERROR (prometheus) ", 0) 66 | 67 | /* 68 | We use match separator ";" for matchSeparator as "," (coma) is reserved to match a label value, 69 | or to match label values against regular expressions. 70 | 71 | The following label matching operators exist: 72 | 73 | =: Select labels that are exactly equal to the provided string. 74 | !=: Select labels that are not equal to the provided string. 75 | =~: Select labels that regex-match the provided string (or substring). 76 | !~: Select labels that do not regex-match the provided string (or substring). 77 | 78 | Please visit: 79 | https://prometheus.io/docs/prometheus/latest/querying/basics/#operators 80 | for more information on label matching. 81 | */ 82 | 83 | matchSeparator = ";" 84 | ) 85 | 86 | func main() { 87 | 88 | /* 89 | 90 | TESTING MODULAR INPUTS IN DEV ENVIRONMENT 91 | 92 | In order to local test modular inputs please do the following: 93 | http://docs.splunk.com/Documentation/Splunk/7.2.1/AdvancedDev/ModInputsDevTools 94 | 95 | 1. Grab the output from you local stanza 96 | 97 | Example for local inputs.conf: 98 | 99 | [prometheus://example-federate] 100 | URI = http://localhost:9090/federate 101 | match = {__name__=~"..*"} 102 | index = prometheus 103 | sourcetype = prometheus:metric 104 | interval = 60 105 | disabled = 1 106 | 107 | # Extract stdin for local testing: 108 | $./bin/splunk cmd splunkd print-modinput-config prometheus prometheus://example-federate 109 | 110 | 111 | 112 | localhost 113 | https://127.0.0.1:8089 114 | ^gbQv^_I4sBVjoMX6Lo7K6sxK0rpKsdMIZmTecbHdm2L0tKJ1gwl8ctZxI5V^ZX6qTUyAyFEg3FKf3iuZjZoy9KGQ7nmS5WDemZyo_VHVKA7q9q9ecHMrXr 115 | /opt/splunk/var/lib/splunk/modinputs/prometheus 116 | 117 | 118 | http://localhost:9090/federate 119 | 0 120 | localhost 121 | prometheus 122 | 60 123 | {__name__=~"..*"} 124 | prometheus:metric 125 | 500/param> 126 | 127 | 128 | 129 | 130 | 2. Write your stanza into a xml file eg: tester.xml 131 | 132 | 3. Use your stanza tester.xml file as stdin for you modular input: 133 | 134 | 4. cat tester.xml | go run prometheus.go 135 | 136 | */ 137 | 138 | if len(os.Args) > 1 { 139 | if os.Args[1] == "--scheme" { 140 | fmt.Println(doScheme()) 141 | } else if os.Args[1] == "--validate-arguments" { 142 | validateArguments() 143 | } 144 | } else { 145 | run() 146 | } 147 | 148 | return 149 | } 150 | 151 | func doScheme() string { 152 | 153 | scheme := ` 154 | Prometheus 155 | Scrapes a Prometheus endpoint, either directly or via Prometheus federation 156 | false 157 | simple 158 | false 159 | 160 | 161 | Metrics URI 162 | A Prometheus exporter endpoint 163 | true 164 | true 165 | 166 | 167 | Match filter 168 | A comma-delimited list of Prometheus "match" expressions: only functional and required for /federate endpoints 169 | false 170 | false 171 | 172 | 173 | Skip certificate verification 174 | If the endpoint is HTTPS, this setting controls whether to skip verification of the server certificate or not 175 | false 176 | false 177 | 178 | 179 | HTTP timeout/title> 180 | <description>Number of milliseconds to wait for HTTP timeout. Defaults to 500 if not set or <= 0./description> 181 | <required_on_edit>false</required_on_edit> 182 | <required_on_create>false</required_on_create> 183 | </arg> 184 | </endpoint> 185 | </scheme>` 186 | 187 | return scheme 188 | } 189 | 190 | func validateArguments() { 191 | // Currently unused 192 | // Will be used to properly validate in future 193 | return 194 | } 195 | 196 | func config() inputConfig { 197 | 198 | data, _ := ioutil.ReadAll(os.Stdin) 199 | var input input 200 | xml.Unmarshal(data, &input) 201 | 202 | var inputConfig inputConfig 203 | 204 | for _, s := range input.Configuration.Stanzas { 205 | for _, p := range s.Params { 206 | if p.Name == "URI" { 207 | inputConfig.URI = p.Value 208 | } 209 | if p.Name == "insecureSkipVerify" { 210 | inputConfig.InsecureSkipVerify, _ = strconv.ParseBool(p.Value) 211 | } 212 | if p.Name == "index" { 213 | inputConfig.Index = p.Value 214 | } 215 | if p.Name == "sourcetype" { 216 | inputConfig.Sourcetype = p.Value 217 | } 218 | if p.Name == "host" { 219 | inputConfig.Host = p.Value 220 | } 221 | if p.Name == "match" { 222 | for _, m := range strings.Split(p.Value, matchSeparator) { 223 | inputConfig.Match = append(inputConfig.Match, m) 224 | } 225 | } 226 | if p.Name == "httpTimeout" { 227 | inputConfig.HttpTimeout, _ = strconv.ParseInt(p.Value, 10, 64) 228 | } 229 | } 230 | } 231 | 232 | if (inputConfig.HttpTimeout <= 0) { 233 | inputConfig.HttpTimeout = 500 234 | } 235 | 236 | return inputConfig 237 | } 238 | 239 | func run() { 240 | 241 | var inputConfig = config() 242 | 243 | tr := &http.Transport{ 244 | TLSClientConfig: &tls.Config{InsecureSkipVerify: inputConfig.InsecureSkipVerify}, 245 | } 246 | 247 | client := &http.Client{Transport: tr, Timeout: time.Duration(time.Millisecond * time.Duration(inputConfig.HttpTimeout))} 248 | 249 | req, err := http.NewRequest("GET", inputConfig.URI, nil) 250 | 251 | if err != nil { 252 | log.Fatal("Request error", err) 253 | } 254 | 255 | req.Header.Add("Accept", "application/openmetrics-text;version=1.0.0,application/openmetrics-text;version=0.0.1;q=0.75,text/plain;version=0.0.4;q=0.5,*/*;q=0.1") 256 | req.Header.Add("Accept-Encoding", "gzip") 257 | req.TransferEncoding = []string{"identity"} 258 | 259 | q := req.URL.Query() 260 | for _, m := range inputConfig.Match { 261 | q.Add("match[]", m) 262 | } 263 | req.URL.RawQuery = q.Encode() 264 | 265 | // Debug request req.URL 266 | logDebug.Print(req.URL) 267 | 268 | // Current timestamp in millis, used if response has no timestamps 269 | now := time.Now().UnixNano() / int64(time.Millisecond) 270 | 271 | resp, err := client.Do(req) 272 | 273 | if err != nil { 274 | log.Fatal(err) 275 | } 276 | 277 | defer resp.Body.Close() 278 | body, err := ioutil.ReadAll(resp.Body) 279 | 280 | if err != nil { 281 | log.Fatal(err) 282 | } 283 | 284 | // Output buffer 285 | output := bufio.NewWriter(os.Stdout) 286 | defer output.Flush() 287 | 288 | // Need to parse metrics out of body individually to convert from scientific to decimal etc. before handing to Splunk 289 | contentType := resp.Header.Get("Content-Type") 290 | p, err := textparse.New(body, contentType) 291 | 292 | if err != nil { 293 | log.Fatal(err) 294 | } 295 | 296 | for { 297 | et, err := p.Next() 298 | 299 | if err != nil { 300 | if err == io.EOF { 301 | break 302 | } else { 303 | continue 304 | } 305 | } 306 | 307 | // Only care about the actual metric series in Splunk for now 308 | if et == textparse.EntrySeries { 309 | b, ts, val := p.Series() 310 | 311 | if ts != nil { 312 | now = *ts 313 | } 314 | 315 | if math.IsNaN(val) || math.IsInf(val, 0) { 316 | continue 317 | } // Splunk won't accept NaN metrics etc. 318 | output.WriteString(fmt.Sprintf("%s %f %d\n", b, val, now)) 319 | } 320 | } 321 | 322 | return 323 | } 324 | -------------------------------------------------------------------------------- /prometheusrw/prometheusrw.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "crypto/tls" 6 | "encoding/xml" 7 | "fmt" 8 | "io/ioutil" 9 | "log" 10 | "math" 11 | "net/http" 12 | "os" 13 | "regexp" 14 | "strconv" 15 | "strings" 16 | 17 | "github.com/gobwas/glob" 18 | "github.com/gogo/protobuf/proto" 19 | "github.com/golang/snappy" 20 | "github.com/prometheus/common/model" 21 | "github.com/prometheus/prometheus/prompb" 22 | ) 23 | 24 | // Structs to hold XML parsing of input from Splunk 25 | type input struct { 26 | XMLName xml.Name `xml:"input"` 27 | ServerHost string `xml:"server_host"` 28 | ServerURI string `xml:"server_uri"` 29 | SessionKey string `xml:"session_key"` 30 | CheckpointDir string `xml:"checkpoint_dir"` 31 | Configuration configuration `xml:"configuration"` 32 | } 33 | 34 | type configuration struct { 35 | XMLName xml.Name `xml:"configuration"` 36 | Stanzas []stanza `xml:"stanza"` 37 | } 38 | 39 | type stanza struct { 40 | XMLName xml.Name `xml:"stanza"` 41 | Params []param `xml:"param"` 42 | Name string `xml:"name,attr"` 43 | } 44 | 45 | type param struct { 46 | XMLName xml.Name `xml:"param"` 47 | Name string `xml:"name,attr"` 48 | Value string `xml:",chardata"` 49 | } 50 | 51 | type feed struct { 52 | XMLName xml.Name `xml:"feed"` 53 | Keys []key `xml:"entry>content>dict>key"` 54 | } 55 | type key struct { 56 | XMLName xml.Name `xml:"key"` 57 | Name string `xml:"name,attr"` 58 | Value string `xml:",chardata"` 59 | } 60 | 61 | // End XML structs 62 | 63 | // Structs store final config 64 | type inputConfig struct { 65 | BearerToken string 66 | Whitelist []glob.Glob 67 | Blacklist []glob.Glob 68 | Index string 69 | Sourcetype string 70 | Host string 71 | MetricNamePrefix string // Add a custom prefix to metric name 72 | MetricNameParse bool // Parse metric according to splunk prefix 73 | } 74 | 75 | type globalConfig struct { 76 | ListenAddr string 77 | MaxClients int 78 | Disabled bool 79 | EnableTLS bool 80 | CertFile string 81 | KeyFile string 82 | } 83 | 84 | // End config structs 85 | 86 | var ( 87 | defaultMetricNamePrefix = "" 88 | defaultMetricNameParse = false 89 | ) 90 | 91 | func main() { 92 | 93 | if len(os.Args) > 1 { 94 | if os.Args[1] == "--scheme" { 95 | fmt.Println(doScheme()) 96 | } else if os.Args[1] == "--validate-arguments" { 97 | validateArguments() 98 | } 99 | } else { 100 | log.Fatal(run()) 101 | } 102 | 103 | return 104 | } 105 | 106 | func doScheme() string { 107 | 108 | scheme := `<scheme> 109 | <title>Prometheus Remote Write 110 | Listen on a TCP port as a remote write endpoint for the Prometheus metrics server 111 | false 112 | simple 113 | true 114 | 115 | 116 | Bearer token 117 | A token configured in Prometheus to send via the Authorization header 118 | true 119 | true 120 | 121 | 122 | Whitelist 123 | A comma-separated list of glob patterns to match metric names and index (default *) 124 | false 125 | false 126 | 127 | 128 | Blacklist 129 | A comma-separated list of glob patterns to match metric names and prevent indexing (default empty). Applied after whitelisting. 130 | false 131 | false 132 | 133 | 134 | ` 135 | 136 | return scheme 137 | } 138 | 139 | func validateArguments() { 140 | // Currently unused 141 | // Will be used to properly validate in future 142 | return 143 | } 144 | 145 | func config() (globalConfig, map[string]inputConfig) { 146 | 147 | data, err := ioutil.ReadAll(os.Stdin) 148 | if err != nil { 149 | log.Fatal(err) 150 | } 151 | 152 | var input input 153 | err = xml.Unmarshal(data, &input) 154 | 155 | if err != nil { 156 | log.Fatal(err) 157 | } 158 | 159 | configMap := make(map[string]inputConfig) 160 | 161 | for _, s := range input.Configuration.Stanzas { 162 | 163 | var inputConfig inputConfig 164 | 165 | // Defaults 166 | inputConfig.MetricNamePrefix = defaultMetricNamePrefix 167 | inputConfig.MetricNameParse = defaultMetricNameParse 168 | 169 | for _, p := range s.Params { 170 | if p.Name == "whitelist" { 171 | for _, w := range strings.Split(p.Value, ",") { 172 | inputConfig.Whitelist = append(inputConfig.Whitelist, glob.MustCompile(w)) 173 | } 174 | } 175 | if p.Name == "blacklist" { 176 | for _, b := range strings.Split(p.Value, ",") { 177 | inputConfig.Blacklist = append(inputConfig.Blacklist, glob.MustCompile(b)) 178 | } 179 | } 180 | if p.Name == "bearerToken" { 181 | inputConfig.BearerToken = p.Value 182 | } 183 | if p.Name == "index" { 184 | inputConfig.Index = p.Value 185 | } 186 | if p.Name == "sourcetype" { 187 | inputConfig.Sourcetype = p.Value 188 | } 189 | if p.Name == "host" { 190 | inputConfig.Host = p.Value 191 | } 192 | if p.Name == "metricNamePrefix" { 193 | inputConfig.MetricNamePrefix = p.Value 194 | } 195 | if p.Name == "metricNameParse" { 196 | inputConfig.MetricNameParse = (p.Value == "true") 197 | } 198 | } 199 | 200 | configMap[inputConfig.BearerToken] = inputConfig 201 | } 202 | // Default global config 203 | var globalConfig globalConfig 204 | globalConfig.ListenAddr = ":8098" 205 | globalConfig.MaxClients = 10 206 | globalConfig.Disabled = true 207 | globalConfig.EnableTLS = false 208 | 209 | // Get the global configuration 210 | tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} 211 | client := &http.Client{Transport: tr} 212 | req, err := http.NewRequest("GET", input.ServerURI+"/services/configs/inputs/prometheusrw", nil) 213 | if err != nil { 214 | log.Fatal(err) 215 | } 216 | 217 | req.Header.Add("Authorization", "Splunk "+input.SessionKey) 218 | response, err := client.Do(req) 219 | if err != nil { 220 | log.Fatal(err) 221 | } 222 | 223 | body, err := ioutil.ReadAll(response.Body) 224 | if err != nil { 225 | log.Fatal(err) 226 | } 227 | 228 | // Parse the global configuration 229 | var feed feed 230 | xml.Unmarshal(body, &feed) 231 | for _, k := range feed.Keys { 232 | if k.Name == "disabled" { 233 | globalConfig.Disabled, _ = strconv.ParseBool(k.Value) 234 | } 235 | if k.Name == "port" { 236 | port, _ := strconv.Atoi(k.Value) 237 | globalConfig.ListenAddr = fmt.Sprintf(":%d", port) 238 | } 239 | if k.Name == "maxClients" { 240 | maxClients, error := strconv.Atoi(k.Value) 241 | if error != nil || maxClients <= 0 { 242 | globalConfig.MaxClients = 10 243 | } else { 244 | globalConfig.MaxClients = maxClients 245 | } 246 | } 247 | if k.Name == "enableTLS" { 248 | globalConfig.EnableTLS, _ = strconv.ParseBool(k.Value) 249 | } 250 | if k.Name == "certFile" { 251 | globalConfig.CertFile = strings.Replace(k.Value, "$SPLUNK_HOME", os.Getenv("SPLUNK_HOME"), -1) 252 | } 253 | if k.Name == "keyFile" { 254 | globalConfig.KeyFile = strings.Replace(k.Value, "$SPLUNK_HOME", os.Getenv("SPLUNK_HOME"), -1) 255 | } 256 | } 257 | response.Body.Close() 258 | 259 | return globalConfig, configMap 260 | } 261 | 262 | func run() error { 263 | 264 | // Output of metrics are sent to Splunk via log interface 265 | // This ensures parallel requests don't interleave, which can happen using stdout directly 266 | output := log.New(os.Stdout, "", 0) 267 | 268 | // Actual logging (goes to splunkd.log) 269 | //infoLog := log.New(os.Stderr, "INFO ", 0) 270 | //debugLog := log.New(os.Stderr, "DEBUG ", 0) 271 | //errLog := log.New(os.Stderr, "ERROR ", 0) 272 | 273 | globalConfig, configMap := config() 274 | 275 | if globalConfig.Disabled == true { 276 | log.Fatal("Prometheus input globally disabled") 277 | } 278 | 279 | // Semaphore to limit to maxClients concurrency 280 | sema := make(chan struct{}, globalConfig.MaxClients) 281 | 282 | http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 283 | 284 | // Get the bearer token and corresponding config 285 | bearerToken := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") 286 | 287 | if _, ok := configMap[bearerToken]; !ok { 288 | http.Error(w, "Bearer token not recognized. Please contact your Splunk admin.", http.StatusUnauthorized) 289 | return 290 | } 291 | 292 | inputConfig := configMap[bearerToken] 293 | 294 | // This will queue a client if > maxClients are processing 295 | sema <- struct{}{} 296 | defer func() { <-sema }() 297 | 298 | // A buffer to build out metrics in for this request 299 | // We dump it all at once, as we may have index/sourcetype etc. directives and we can't have them separated from the metrics they effect by another request 300 | var buffer bytes.Buffer 301 | 302 | buffer.WriteString(fmt.Sprintf("***SPLUNK*** index=%s sourcetype=%s host=%s\n", inputConfig.Index, inputConfig.Sourcetype, inputConfig.Host)) 303 | 304 | compressed, err := ioutil.ReadAll(r.Body) 305 | if err != nil { 306 | http.Error(w, err.Error(), http.StatusInternalServerError) 307 | return 308 | } 309 | 310 | reqBuf, err := snappy.Decode(nil, compressed) 311 | if err != nil { 312 | http.Error(w, err.Error(), http.StatusBadRequest) 313 | return 314 | } 315 | 316 | var req prompb.WriteRequest 317 | if err := proto.Unmarshal(reqBuf, &req); err != nil { 318 | http.Error(w, err.Error(), http.StatusBadRequest) 319 | return 320 | } 321 | for _, ts := range req.Timeseries { 322 | 323 | m := make(model.Metric, len(ts.Labels)) 324 | 325 | for _, l := range ts.Labels { 326 | m[model.LabelName(l.Name)] = model.LabelValue(l.Value) 327 | } 328 | 329 | whitelisted := false 330 | for _, w := range inputConfig.Whitelist { 331 | if w.Match(string(m["__name__"])) { 332 | whitelisted = true 333 | } 334 | } 335 | 336 | if !whitelisted { 337 | continue 338 | } 339 | 340 | blacklisted := false 341 | for _, b := range inputConfig.Blacklist { 342 | if b.Match(string(m["__name__"])) { 343 | blacklisted = true 344 | } 345 | } 346 | 347 | if blacklisted { 348 | continue 349 | } 350 | 351 | if inputConfig.MetricNameParse { 352 | m["__name__"] = formatMetricLabelValue(string(m["__name__"]), inputConfig.MetricNamePrefix) 353 | } 354 | 355 | for _, s := range ts.Samples { 356 | if math.IsNaN(s.Value) || math.IsInf(s.Value, 0) { 357 | continue 358 | } // Splunk won't accept NaN metrics etc. 359 | buffer.WriteString(fmt.Sprintf("%s %f %d\n", m, s.Value, s.Timestamp)) 360 | } 361 | } 362 | 363 | output.Print(buffer.String()) 364 | buffer.Truncate(0) 365 | }) 366 | 367 | if globalConfig.EnableTLS == true { 368 | return http.ListenAndServeTLS(globalConfig.ListenAddr, globalConfig.CertFile, globalConfig.KeyFile, nil) 369 | } else { 370 | return http.ListenAndServe(globalConfig.ListenAddr, nil) 371 | } 372 | } 373 | 374 | func formatMetricLabelValue(value string, prefix string) model.LabelValue { 375 | s := []string{} 376 | s = append(s, prefix) 377 | s = append(s, regexp.MustCompile("_").ReplaceAllString(value, ".")) 378 | return model.LabelValue(strings.Join(s, "")) 379 | } 380 | -------------------------------------------------------------------------------- /prometheusrw/prometheusrw_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Prometheus Authors 2 | // Licensed under the Apache License, Version 2.0 (the "License"); 3 | // you may not use this file except in compliance with the License. 4 | // You may obtain a copy of the License at 5 | // 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | 14 | // Example: go test -run '' 15 | 16 | package main 17 | 18 | import ( 19 | "testing" 20 | 21 | "github.com/prometheus/common/model" 22 | "github.com/prometheus/prometheus/util/testutil" 23 | ) 24 | 25 | func TestFormatMetricLabelValue(t *testing.T) { 26 | tests := []struct { 27 | value string 28 | parse bool 29 | prefix string 30 | output model.LabelValue 31 | }{ 32 | { 33 | value: "container_network_receive_errors_total", 34 | output: "container.network.receive.errors.total", 35 | prefix: "", 36 | }, 37 | { 38 | value: "container_network_receive_errors_total", 39 | output: "DEV.container.network.receive.errors.total", 40 | prefix: "DEV.", 41 | }, 42 | } 43 | 44 | for _, test := range tests { 45 | res := formatMetricLabelValue(test.value, test.prefix) 46 | testutil.Equals(t, res, test.output) 47 | } 48 | } 49 | --------------------------------------------------------------------------------