├── .github ├── actionlint-matcher.json ├── renovate.json5 └── workflows │ └── ci.yml ├── .gitignore ├── .pylintrc ├── .style.yapf ├── CHANGELOG.md ├── LICENSE ├── README.md ├── example-generator.py ├── example-prometheus-stub.yml ├── mypy.ini ├── promgen_k8s ├── __init__.py ├── cadvisor_job.py ├── cluster.py ├── generator.py ├── ingresses_job.py ├── nodes_job.py ├── pods_job.py ├── prom_dsl.py ├── service_endpoints_job.py └── services_job.py ├── pyproject.toml └── tests ├── __init__.py ├── promgen_k8s_test.py └── snapshots ├── __init__.py └── snap_promgen_k8s_test.py /.github/actionlint-matcher.json: -------------------------------------------------------------------------------- 1 | { 2 | "problemMatcher": [ 3 | { 4 | "owner": "actionlint", 5 | "pattern": [ 6 | { 7 | "regexp": "^(?:\\x1b\\[\\d+m)?(.+?)(?:\\x1b\\[\\d+m)*:(?:\\x1b\\[\\d+m)*(\\d+)(?:\\x1b\\[\\d+m)*:(?:\\x1b\\[\\d+m)*(\\d+)(?:\\x1b\\[\\d+m)*: (?:\\x1b\\[\\d+m)*(.+?)(?:\\x1b\\[\\d+m)* \\[(.+?)\\]$", 8 | "file": 1, 9 | "line": 2, 10 | "column": 3, 11 | "message": 4, 12 | "code": 5 13 | } 14 | ] 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /.github/renovate.json5: -------------------------------------------------------------------------------- 1 | { 2 | extends: [ 3 | "config:base", 4 | ":dependencyDashboard", 5 | ":prHourlyLimitNone", 6 | ":prConcurrentLimitNone", 7 | ":label(dependency-upgrade)", 8 | ], 9 | schedule: ["before 8am on thursday"], 10 | branchPrefix: "renovate-", 11 | dependencyDashboardHeader: "View repository job log [here](https://app.renovatebot.com/dashboard#github/cmur2/promgen-k8s).", 12 | separateMinorPatch: true, 13 | commitMessagePrefix: "module: ", 14 | commitMessageAction: "update", 15 | commitMessageTopic: "{{depName}}", 16 | commitMessageExtra: "to {{#if isSingleVersion}}v{{{newVersion}}}{{else}}{{{newValue}}}{{/if}}", 17 | packageRules: [ 18 | // Commit message formats 19 | { 20 | matchManagers: ["github-actions"], 21 | commitMessagePrefix: "ci: ", 22 | }, 23 | ], 24 | regexManagers: [], 25 | } 26 | 27 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | name: ci 3 | 4 | on: 5 | push: 6 | branches: [master] 7 | pull_request: 8 | branches: [master] 9 | workflow_dispatch: 10 | schedule: 11 | - cron: '45 4 * * 4' # weekly on thursday morning 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-latest 16 | strategy: 17 | fail-fast: false 18 | matrix: 19 | python-version: 20 | - '3.7' 21 | - '3.8' 22 | - '3.9' 23 | - '3.10' 24 | steps: 25 | - uses: actions/checkout@v4 26 | - name: Set up Python ${{ matrix.python-version }} 27 | uses: actions/setup-python@v4 28 | with: 29 | python-version: ${{ matrix.python-version }} 30 | - name: Install dependencies 31 | run: | 32 | pip install poetry 33 | poetry install 34 | - name: Lint 35 | run: | 36 | poetry run yapf -q -r promgen_k8s 37 | poetry run pylint --ignore=snapshots promgen_k8s tests 38 | poetry run mypy promgen_k8s 39 | - name: Test 40 | run: | 41 | poetry run pytest 42 | 43 | actionlint: 44 | runs-on: ubuntu-latest 45 | steps: 46 | - uses: actions/checkout@v4 47 | - name: Check workflow files 48 | run: | 49 | echo "::add-matcher::.github/actionlint-matcher.json" 50 | bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) 51 | ./actionlint 52 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | .mypy_cache 3 | .pytest_cache 4 | *.egg-info/ 5 | *.py[cod] 6 | build/ 7 | dist/ 8 | example-prometheus.yml 9 | MANIFEST 10 | poetry.lock 11 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [FORMAT] 2 | indent-string=' ' 3 | indent-after-paren=2 4 | max-line-length=121 5 | 6 | [MESSAGES CONTROL] 7 | disable=missing-docstring, 8 | wildcard-import, 9 | unused-wildcard-import, 10 | bad-continuation, 11 | 12 | [SIMILARITIES] 13 | min-similarity-lines=15 14 | 15 | [DESIGN] 16 | max-args=6 17 | max-attributes=8 18 | min-public-methods=1 19 | -------------------------------------------------------------------------------- /.style.yapf: -------------------------------------------------------------------------------- 1 | [style] 2 | based_on_style = pep8 3 | indent_width = 2 4 | continuation_indent_width = 2 5 | column_limit = 120 6 | allow_split_before_dict_value = false 7 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 0.3.0 (June 17, 2021) 4 | 5 | CHANGES: 6 | 7 | - Require Python 3.7 or above 8 | 9 | FIXES: 10 | 11 | - Re-add previously removed `rc_list()` DSL function 12 | 13 | ## 0.2.2 (December 10, 2020) 14 | 15 | CHANGES: 16 | 17 | - Support Python 3 only 18 | - Use [Poetry](https://python-poetry.org/docs/) dependency manager instead of [pipenv](https://pipenv.pypa.io/) for the project now as Poetry is more stable, supports [pyproject.toml](pyproject.toml]) file format and compatible Python version ranges 19 | 20 | ## 0.2.1 (October 21, 2019) 21 | 22 | IMPROVEMENTS: 23 | 24 | - Force usage of HTTPS for Kubernetes SD apiserver connections since newer Prometheus versions changed the default 25 | 26 | ## 0.2.0 (March 27, 2019) 27 | 28 | CHANGES: 29 | 30 | - Switch to kubelet HTTP port 10255 for scraping cadvisor and nodes metrics since that also works with kubelet `--authorization-mode=Webhook` 31 | 32 | ## 0.1.7 (December 17, 2018) 33 | 34 | IMPROVEMENTS: 35 | 36 | - Allow specifying `prometheus.io/module` annotation for blackbox ingress monitoring 37 | 38 | ## 0.1.6 (June 11, 2018) 39 | 40 | IMPROVEMENTS: 41 | 42 | - Add `kubernetes_container_name` label for all endpoints of pods where `prometheus.io/filterport: true` 43 | 44 | ## 0.1.5 (June 11, 2018) 45 | 46 | IMPROVEMENTS: 47 | 48 | - Add snapshot tests for generated YAML 49 | - Allow specifying `prometheus.io/filterport: true` on pods to automatically discover all endpoints who's pod port name ends with `metrics` instead of using `prometheus.io/port` 50 | 51 | ## 0.1.4 (February 13, 2018) 52 | 53 | IMPROVEMENTS: 54 | 55 | - Allow specifying `prometheus.io/module` and `prometheus.io/path` annotations for blackbox service monitoring 56 | 57 | ## 0.1.3 (December 22, 2017) 58 | 59 | IMPROVEMENTS: 60 | 61 | - Remove some unneeded since duplicated labels from pods, nodes and cadvisor jobs 62 | - Fix typo in name of the `ServiceEndpointsJob` 63 | - Provide unique instance label for all jobs, usually copied from the `__address__` label 64 | - Add `IngressesJob` for probing ingresses via the blackbox exporter (no HTTPS support) 65 | - Allow setting the `scrape_interval` for nodes, cadvisor, services, service endpoints and ingresses jobs 66 | - Support different scrape intervals for pods using `prometheus.io/interval` annotation 67 | - Do not require proxies in remote clusters anymore for scraping pods, services, service endpoints or ingresses but blackbox-exporter might be needed instead 68 | 69 | ## 0.1.2 (November 7, 2017) 70 | 71 | IMPROVEMENTS: 72 | 73 | - Use cAdvisor metrics path compatible with Kubernetes 1.7.3 and above 74 | 75 | ## 0.1.1 (September 13, 2017) 76 | 77 | IMPROVEMENTS: 78 | 79 | - Do not use aliases in generated YAML 80 | 81 | ## 0.1.0 (September 13, 2017) 82 | 83 | NEW FEATURES: 84 | 85 | - Initial release 86 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # promgen-k8s 2 | 3 | ![ci](https://github.com/cmur2/promgen-k8s/workflows/ci/badge.svg) 4 | 5 | A modular [Prometheus 2](https://prometheus.io) configuration file generator to monitor multiple [Kubernetes](https://kubernetes.io) clusters with a single Prometheus instance. 6 | 7 | ## Scenario 8 | 9 | You have one Kubernetes cluster per environment (dev, qa, staging, production, etc.) for whichever reasons including e.g. special permission model, limiting the blast radius of problems related to Kubernetes or to test Kubernetes upgrades in staging environment first. 10 | You have a separate Kubernetes cluster for operations that runs your monitoring and logging setup, CI pipeline, etc. 11 | You want to monitor all environments/clusters with a single Prometheus instance located in the operations cluster. 12 | 13 | Writing, modifying and maintaining the [prometheus.yml](https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml) for one Kubernetes cluster by hand is tedious and error prone. Doing so for multiple clusters that should share most of the configuration except for specific rules can become a nightmare. 14 | 15 | promgen-k8s aims to provide a toolkit allowing you to easily generate a prometheus.yml tailored for monitoring multiple Kubernetes clusters with invididual scraping jobs and relabeling rules per cluster. 16 | 17 | ## Current State 18 | 19 | promgen-k8s is currently in a very early state where it manages to generate the example [prometheus-kubernetes.yml](https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml) for multiple clusters using [autodiscovery](https://prometheus.io/docs/operating/configuration/#%3Ckubernetes_sd_config%3E) including custom relabeling rules per cluster. It assumes that it can reach all Kubernetes apiservers by predictable DNS names using the provided secret files for authentication. 20 | 21 | For ingresses, pod, service and service endpoint monitoring in remote clusters promgen-k8s uses the Kubernetes apiservers as an [HTTP proxy](https://kubernetes.io/docs/tasks/administer-cluster/access-cluster-services/). 22 | 23 | As ingresses and services are checked using the [blackbox-exporter](https://github.com/prometheus/blackbox_exporter/) promgen-k8s also assumes a `blackbox-exporter` pod and service in `monitoring` namespace running in each cluster (including remote clusters). 24 | 25 | promgen-k8s is successfully used with Kubernetes 1.8 clusters on [AWS](http://aws.amazon.com/) created by [kops](https://github.com/kubernetes/kops). 26 | 27 | ## Example 28 | 29 | The [example](example-generator.py) uses a [stub file](example-prometheus-stub.yml) for manual configuration and can be tested using `pip install promgen-k8s` and then `python example-generator.py`. 30 | 31 | ## Related Work 32 | 33 | - [https://github.com/line/promgen]([https://github.com/line/promgen]) 34 | 35 | ## Doing a Release 36 | 37 | You need to have the [Poetry](https://python-poetry.org/docs/) dependency manager installed and a [PyPI](https://pypi.org/) account (a [test account](https://test.pypi.org/) works as well) including an authentication token. 38 | 39 | ### Testing the Release 40 | 41 | One-time setup: 42 | 43 | ```bash 44 | poetry config repositories.testpypi https://test.pypi.org/legacy/ 45 | poetry config pypi-token.testpypi TOKEN_FROM_TESTPYPI_ACCOUNT 46 | ``` 47 | 48 | Bump version in [pyproject.toml](pyproject.toml), update the [changelog](CHANGELOG.md) with new release and date and commit. 49 | 50 | Build and release: 51 | 52 | ```bash 53 | poetry publish -r testpypi --build 54 | ``` 55 | 56 | Check installation: 57 | 58 | ```bash 59 | pip install --index-url https://test.pypi.org/simple/ promgen_k8s 60 | ``` 61 | 62 | ### Official Release 63 | 64 | One-time setup: 65 | 66 | ```bash 67 | poetry config pypi-token.pypi TOKEN_FROM_PYPI_ACCOUNT 68 | ``` 69 | 70 | Bump version in [pyproject.toml](pyproject.toml), update the [changelog](CHANGELOG.md) with new release and date and commit. 71 | 72 | Build and release: 73 | 74 | ```bash 75 | poetry publish --build 76 | ``` 77 | 78 | Check installation: 79 | 80 | ```bash 81 | pip install promgen_k8s 82 | ``` 83 | 84 | ## License 85 | 86 | promgen-k8s is licensed under the Apache License, Version 2.0. See LICENSE for more information. 87 | -------------------------------------------------------------------------------- /example-generator.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import yaml 4 | 5 | import promgen_k8s as g 6 | from promgen_k8s.prom_dsl import * 7 | 8 | 9 | CLUSTERS = [ 10 | g.Cluster('staging', jobs=[ 11 | g.NodesJob(), 12 | g.IngressesJob(), 13 | g.CadvisorJob(), 14 | g.ServiceEndpointsJob(), 15 | g.ServicesJob(additional_relabel_configs=[ 16 | # reverse.default.svc:443 should prompt for authentication in staging environment 17 | replace(source_labels=['__param_target'], 18 | regex='reverse.default.svc:443', replacement='https_401', 19 | target_label='__param_module') 20 | ]), 21 | g.PodsJob() 22 | ]), 23 | g.Cluster('production', jobs=[ 24 | g.NodesJob(), 25 | g.IngressesJob(), 26 | g.CadvisorJob(), 27 | g.ServiceEndpointsJob(), 28 | g.ServicesJob(), 29 | g.PodsJob() 30 | ]), 31 | # The incluster property marks the cluster in which Prometheus in running 32 | g.Cluster('operations', incluster=True, jobs=[ 33 | g.NodesJob(), 34 | g.IngressesJob(), 35 | g.CadvisorJob(), 36 | g.ServiceEndpointsJob(), 37 | g.ServicesJob(), 38 | g.PodsJob(interval_map={'long': '1h'}) 39 | ]) 40 | ] # yapf: disable 41 | 42 | 43 | def main() -> None: 44 | # read in the stub config 45 | with open('example-prometheus-stub.yml', 'r') as stub_file: 46 | stub_prom_conf = yaml.load(stub_file, Loader=yaml.SafeLoader) 47 | 48 | generator = g.Generator(CLUSTERS, initial_prom_conf=stub_prom_conf) 49 | 50 | # write out merged result 51 | with open('example-prometheus.yml', 'w') as final_file: 52 | generator.dump(final_file) 53 | 54 | 55 | if __name__ == "__main__": 56 | main() 57 | -------------------------------------------------------------------------------- /example-prometheus-stub.yml: -------------------------------------------------------------------------------- 1 | # This configuration file is only a stub used by the example-generator.py 2 | # to allow manual modifications or additions to the final prometheus.yml 3 | 4 | --- 5 | # Global settings 6 | global: 7 | scrape_interval: 1m 8 | evaluation_interval: 1m 9 | 10 | # Load rules once and periodically evaluate them according to the global 'evaluation_interval'. 11 | rule_files: 12 | - "/etc/alert.rules" 13 | 14 | scrape_configs: 15 | # This is only a stub - all Kubernetes related jobs will be generated! 16 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | ignore_missing_imports=true 3 | follow_imports=silent 4 | show_column_numbers=true 5 | -------------------------------------------------------------------------------- /promgen_k8s/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = [ 2 | 'cadvisor_job', 'cluster', 'generator', 'ingresses_job', 'nodes_job', 'pods_job', 'prom_dsl', 'service_endpoints_job', 3 | 'services_job' 4 | ] 5 | 6 | from .cadvisor_job import CadvisorJob 7 | from .cluster import Cluster 8 | from .generator import Generator 9 | from .ingresses_job import IngressesJob 10 | from .nodes_job import NodesJob 11 | from .pods_job import PodsJob 12 | from .prom_dsl import * 13 | from .service_endpoints_job import ServiceEndpointsJob 14 | from .service_endpoints_job import ServiceEndpoitsJob 15 | from .services_job import ServicesJob 16 | -------------------------------------------------------------------------------- /promgen_k8s/cadvisor_job.py: -------------------------------------------------------------------------------- 1 | from typing import (Optional) 2 | 3 | from .cluster import * 4 | from .prom_dsl import * 5 | 6 | 7 | class CadvisorJob(GeneratorJob): 8 | 9 | def __init__(self, 10 | scrape_interval: Optional[str] = None, 11 | additional_relabel_configs: Optional[List[Any]] = None, 12 | additional_metric_relabel_configs: Optional[List[Any]] = None): 13 | self.type = 'cadvisor' 14 | self.scrape_interval = scrape_interval 15 | self.additional_relabel_configs = additional_relabel_configs or [] 16 | self.additional_metric_relabel_configs = additional_metric_relabel_configs or [] 17 | 18 | # Scrape config for Kubelet cAdvisor. 19 | # 20 | # This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics 21 | # (those whose names begin with 'container_') have been removed from the 22 | # Kubelet metrics endpoint. This job scrapes the cAdvisor endpoint to 23 | # retrieve those metrics. 24 | # 25 | # In Kubernetes 1.7.0-1.7.2, these metrics are only exposed on the cAdvisor 26 | # HTTP endpoint; use "replacement: /api/v1/nodes/${1}:4194/proxy/metrics" 27 | # in that case (and ensure cAdvisor's HTTP server hasn't been disabled with 28 | # the --cadvisor-port=0 Kubelet flag). 29 | # 30 | # This job is not necessary and should be removed in Kubernetes 1.6 and 31 | # earlier versions, or it will cause the metrics to be scraped twice. 32 | def generate(self, prom_conf: Dict[str, Any], cluster: Cluster) -> None: 33 | prom_conf['scrape_configs'].append({ 34 | 'job_name': f'{cluster.name}-kubernetes-cadvisor', 35 | 'scheme': 'https', 36 | 'kubernetes_sd_configs': [ 37 | cluster.get_kubernetes_sd_config('node') 38 | ], 39 | 40 | # This TLS & bearer token file config is used to connect to the actual scrape 41 | # endpoints for cluster components. This is separate to discovery auth 42 | # configuration because discovery & scraping are two separate concerns in 43 | # Prometheus. The discovery auth config is automatic if Prometheus runs inside 44 | # the cluster. Otherwise, more config options have to be provided within the 45 | # . 46 | 'tls_config': { 47 | 'ca_file': cluster.ca_file 48 | }, 49 | 'bearer_token_file': cluster.bearer_token_file, 50 | 51 | 'relabel_configs': [ 52 | remove_label('__meta_kubernetes_node_label_node_role_kubernetes_io_node'), 53 | remove_label('__meta_kubernetes_node_label_node_role_kubernetes_io_master'), 54 | labelmap(regex='__meta_kubernetes_node_label_(.+)'), 55 | copy_value('__address__', 'instance'), 56 | set_value('__address__', f'{cluster.api_server}:443'), 57 | replace(source_labels=['__meta_kubernetes_node_name'], 58 | regex='(.+)', replacement='/api/v1/nodes/${1}:10255/proxy/metrics/cadvisor', 59 | target_label='__metrics_path__') 60 | ], 61 | 62 | 'metric_relabel_configs': [ 63 | replace(source_labels=['id'], 64 | regex=r'^/machine\.slice/machine-rkt\\x2d([^\\]+)\\.+/([^/]+)\.service$', replacement='${2}-${1}', 65 | target_label='rkt_container_name'), 66 | replace(source_labels=['id'], 67 | regex=r'^/system\.slice/(.+)\.service$', replacement='${1}', 68 | target_label='systemd_service_name'), 69 | drop(source_labels=['__name__'], regex='go_.*') 70 | ] 71 | }) # yapf: disable 72 | 73 | # set job's scrape_interval if defined 74 | if not self.scrape_interval is None: 75 | prom_conf['scrape_configs'][-1]['scrape_interval'] = self.scrape_interval 76 | 77 | # add additional configs 78 | prom_conf['scrape_configs'][-1]['relabel_configs'].extend(self.additional_relabel_configs) 79 | prom_conf['scrape_configs'][-1]['metric_relabel_configs'].extend(self.additional_metric_relabel_configs) 80 | -------------------------------------------------------------------------------- /promgen_k8s/cluster.py: -------------------------------------------------------------------------------- 1 | from typing import (Any, Dict, List, Optional) 2 | 3 | import abc 4 | 5 | 6 | class GeneratorJob(abc.ABC): 7 | 8 | @abc.abstractmethod 9 | def generate(self, prom_conf: Dict[str, Any], cluster: 'Cluster') -> None: 10 | pass 11 | 12 | 13 | class Cluster(): 14 | 15 | def __init__(self, 16 | name: str, 17 | public_domain='example.com', 18 | private_domain='example.loc', 19 | incluster=False, 20 | jobs: Optional[List[GeneratorJob]] = None): 21 | self.name = name 22 | self.public_domain = public_domain 23 | self.private_domain = private_domain 24 | self.incluster = incluster 25 | self.jobs = jobs or [] 26 | 27 | if self.incluster: 28 | self.bearer_token_file = '/var/run/secrets/kubernetes.io/serviceaccount/token' 29 | self.ca_file = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' 30 | self.api_server = 'kubernetes.default.svc' 31 | else: 32 | self.bearer_token_file = f'/var/run/kube_secrets/{self.name}_bearer_token' 33 | self.ca_file = f'/var/run/kube_secrets/{self.name}_ca_crt' 34 | self.api_server = f'api.internal.{self.name}.{self.public_domain}' 35 | 36 | def get_kubernetes_sd_config(self, role: str): 37 | if self.incluster: 38 | return {'role': role} 39 | 40 | return { 41 | 'role': role, 42 | 'api_server': f'https://{self.api_server}', 43 | 'tls_config': { 44 | 'ca_file': self.ca_file 45 | }, 46 | 'bearer_token_file': self.bearer_token_file 47 | } 48 | -------------------------------------------------------------------------------- /promgen_k8s/generator.py: -------------------------------------------------------------------------------- 1 | from typing import (Any, Dict, IO, List, Optional) 2 | 3 | import yaml 4 | 5 | from .cadvisor_job import * 6 | from .cluster import * 7 | from .nodes_job import * 8 | from .pods_job import * 9 | from .prom_dsl import * 10 | from .service_endpoints_job import * 11 | from .services_job import * 12 | 13 | 14 | # via http://pyyaml.org/ticket/64 and http://signal0.com/2013/02/06/disabling_aliases_in_pyyaml.html 15 | # pylint: disable=too-many-ancestors 16 | class ListIndentingDumper(yaml.Dumper): 17 | 18 | def increase_indent(self, flow=False, indentless=False): 19 | return super().increase_indent(flow, False) 20 | 21 | def ignore_aliases(self, data): 22 | return True 23 | 24 | 25 | class Generator(): 26 | 27 | def __init__(self, clusters: List[Cluster], initial_prom_conf: Optional[Dict[str, Any]] = None): 28 | self.clusters = clusters 29 | self.initial_prom_conf = initial_prom_conf or {} 30 | 31 | def dump(self, yaml_file: IO) -> None: 32 | prom_conf = {} 33 | prom_conf.update(self.initial_prom_conf) 34 | 35 | if prom_conf['scrape_configs'] is None: 36 | prom_conf['scrape_configs'] = [] 37 | 38 | # add jobs for k8s clusters 39 | for cluster in self.clusters: 40 | for job in cluster.jobs: 41 | job.generate(prom_conf, cluster) 42 | 43 | yaml.dump(prom_conf, 44 | yaml_file, 45 | encoding=('utf-8'), 46 | Dumper=ListIndentingDumper, 47 | default_flow_style=False, 48 | explicit_start=True) 49 | -------------------------------------------------------------------------------- /promgen_k8s/ingresses_job.py: -------------------------------------------------------------------------------- 1 | from typing import (Optional) 2 | 3 | from .cluster import * 4 | from .prom_dsl import * 5 | 6 | 7 | class IngressesJob(GeneratorJob): 8 | 9 | def __init__(self, 10 | scrape_interval: Optional[str] = None, 11 | additional_relabel_configs: Optional[List[Any]] = None, 12 | additional_metric_relabel_configs: Optional[List[Any]] = None): 13 | self.type = 'ingresses' 14 | self.scrape_interval = scrape_interval 15 | self.additional_relabel_configs = additional_relabel_configs or [] 16 | self.additional_metric_relabel_configs = additional_metric_relabel_configs or [] 17 | 18 | # Example scrape config for probing ingresses via the Blackbox Exporter. 19 | # 20 | # The relabeling allows the actual ingress scrape endpoint to be configured 21 | # via the following annotations: 22 | # 23 | # * `prometheus.io/probe`: Only probe services that have a value of `true` 24 | # * `prometheus.io/module`: If the Blackbox exporter module used is not named `http_2xx` override this. 25 | def generate(self, prom_conf: Dict[str, Any], cluster: Cluster) -> None: 26 | prom_conf['scrape_configs'].append({ 27 | 'job_name': f'{cluster.name}-kubernetes-ingresses', 28 | 'scheme': 'https', 29 | 'kubernetes_sd_configs': [ 30 | cluster.get_kubernetes_sd_config('ingress') 31 | ], 32 | 33 | # This TLS & bearer token file config is used to connect to the actual scrape 34 | # endpoints for cluster components. This is separate to discovery auth 35 | # configuration because discovery & scraping are two separate concerns in 36 | # Prometheus. The discovery auth config is automatic if Prometheus runs inside 37 | # the cluster. Otherwise, more config options have to be provided within the 38 | # . 39 | 'tls_config': { 40 | 'ca_file': cluster.ca_file 41 | }, 42 | 'bearer_token_file': cluster.bearer_token_file, 43 | 44 | 'metrics_path': '/api/v1/namespaces/monitoring/services/blackbox-exporter/proxy/probe', 45 | 'params': { 46 | 'module': ['http_2xx'] 47 | }, 48 | 49 | 'relabel_configs': [ 50 | keep(source_labels=['__meta_kubernetes_ingress_annotation_prometheus_io_probe'], regex='true'), 51 | # Note: does not support any __meta_kubernetes_ingress_scheme except HTTP 52 | copy_value('__meta_kubernetes_ingress_annotation_prometheus_io_module', '__param_module'), 53 | replace(source_labels=['__address__', '__meta_kubernetes_ingress_path'], 54 | separator=';', regex='(.+);(.+)', replacement='$1$2', 55 | target_label='__address__'), 56 | copy_value('__address__', '__param_target'), 57 | copy_value('__address__', 'instance'), 58 | set_value('__address__', f'{cluster.api_server}:443'), 59 | labelmap(regex='__meta_kubernetes_ingress_label_(.+)'), 60 | copy_value('__meta_kubernetes_namespace', 'kubernetes_namespace'), 61 | copy_value('__meta_kubernetes_ingress_name', 'kubernetes_ingress_name') 62 | ] 63 | }) # yapf: disable 64 | 65 | # set job's scrape_interval if defined 66 | if not self.scrape_interval is None: 67 | prom_conf['scrape_configs'][-1]['scrape_interval'] = self.scrape_interval 68 | 69 | # add additional configs 70 | prom_conf['scrape_configs'][-1]['relabel_configs'].extend(self.additional_relabel_configs) 71 | prom_conf['scrape_configs'][-1]['metric_relabel_configs'] = self.additional_metric_relabel_configs 72 | -------------------------------------------------------------------------------- /promgen_k8s/nodes_job.py: -------------------------------------------------------------------------------- 1 | from typing import (Optional) 2 | 3 | from .cluster import * 4 | from .prom_dsl import * 5 | 6 | 7 | class NodesJob(GeneratorJob): 8 | 9 | def __init__(self, 10 | scrape_interval: Optional[str] = None, 11 | additional_relabel_configs: Optional[List[Any]] = None, 12 | additional_metric_relabel_configs: Optional[List[Any]] = None): 13 | self.type = 'nodes' 14 | self.scrape_interval = scrape_interval 15 | self.additional_relabel_configs = additional_relabel_configs or [] 16 | self.additional_metric_relabel_configs = additional_metric_relabel_configs or [] 17 | 18 | # Scrape config for nodes (kubelet). 19 | # 20 | # Rather than connecting directly to the node, the scrape is proxied though the 21 | # Kubernetes apiserver. This means it will work if Prometheus is running out of 22 | # cluster, or can't connect to nodes for some other reason (e.g. because of 23 | # firewalling). 24 | def generate(self, prom_conf: Dict[str, Any], cluster: Cluster) -> None: 25 | prom_conf['scrape_configs'].append({ 26 | 'job_name': f'{cluster.name}-kubernetes-nodes', 27 | 'scheme': 'https', 28 | 'kubernetes_sd_configs': [ 29 | cluster.get_kubernetes_sd_config('node') 30 | ], 31 | 32 | # This TLS & bearer token file config is used to connect to the actual scrape 33 | # endpoints for cluster components. This is separate to discovery auth 34 | # configuration because discovery & scraping are two separate concerns in 35 | # Prometheus. The discovery auth config is automatic if Prometheus runs inside 36 | # the cluster. Otherwise, more config options have to be provided within the 37 | # . 38 | 'tls_config': { 39 | 'ca_file': cluster.ca_file 40 | }, 41 | 'bearer_token_file': cluster.bearer_token_file, 42 | 43 | # Keep all node labels without their prefix, proxy against apiserver with 44 | # HTTPS by default 45 | 'relabel_configs': [ 46 | remove_label('__meta_kubernetes_node_label_node_role_kubernetes_io_node'), 47 | remove_label('__meta_kubernetes_node_label_node_role_kubernetes_io_master'), 48 | labelmap(regex='__meta_kubernetes_node_label_(.+)'), 49 | copy_value('__address__', 'instance'), 50 | set_value('__address__', f'{cluster.api_server}:443'), 51 | replace(source_labels=['__meta_kubernetes_node_name'], 52 | regex='(.+)', replacement='/api/v1/nodes/${1}:10255/proxy/metrics', 53 | target_label='__metrics_path__') 54 | ], 55 | 56 | 'metric_relabel_configs': [ 57 | drop(source_labels=['__name__'], regex='go_.*') 58 | ] 59 | }) # yapf: disable 60 | 61 | # set job's scrape_interval if defined 62 | if not self.scrape_interval is None: 63 | prom_conf['scrape_configs'][-1]['scrape_interval'] = self.scrape_interval 64 | 65 | # add additional configs 66 | prom_conf['scrape_configs'][-1]['relabel_configs'].extend(self.additional_relabel_configs) 67 | prom_conf['scrape_configs'][-1]['metric_relabel_configs'].extend(self.additional_metric_relabel_configs) 68 | -------------------------------------------------------------------------------- /promgen_k8s/pods_job.py: -------------------------------------------------------------------------------- 1 | from typing import (Optional) 2 | 3 | from .cluster import * 4 | from .prom_dsl import * 5 | 6 | 7 | class PodsJob(GeneratorJob): 8 | 9 | def __init__(self, 10 | interval_map: Optional[Dict[str, Optional[str]]] = None, 11 | additional_relabel_configs: Optional[List[Any]] = None, 12 | additional_metric_relabel_configs: Optional[List[Any]] = None): 13 | self.type = 'pods' 14 | self.interval_map = interval_map or {} 15 | self.additional_relabel_configs = additional_relabel_configs or [] 16 | self.additional_metric_relabel_configs = additional_metric_relabel_configs or [] 17 | 18 | # Example scrape config for pods 19 | # 20 | # The relabeling allows the actual pod scrape endpoint to be configured via the 21 | # following annotations: 22 | # 23 | # * `prometheus.io/scrape`: Only scrape pods that have a value of `true` 24 | # * `prometheus.io/path`: If the metrics path is not `/metrics` override this. 25 | # * `prometheus.io/port`: Scrape the pod on the indicated port instead of the 26 | # pod's declared ports (default is a port-free target if none are declared). 27 | # * `prometheus.io/filterport`: If `true` and no `prometheus.io/port` given 28 | # then filter the pod's declared ports for those with name ending in 'metrics'. 29 | # * `prometheus.io/interval`: If present, use the given level instead 30 | # of the global default (must be configured appropriately) 31 | def generate(self, prom_conf: Dict[str, Any], cluster: Cluster) -> None: 32 | self.generate_interval(prom_conf, cluster, 'default', None) 33 | for name, value in self.interval_map.items(): 34 | self.generate_interval(prom_conf, cluster, name, value) 35 | 36 | def generate_interval(self, prom_conf: Dict[str, Any], cluster: Cluster, interval_name, interval_value) -> None: 37 | prom_conf['scrape_configs'].append({ 38 | 'job_name': f'{cluster.name}-kubernetes-pods-{interval_name}', 39 | 'scheme': 'https', 40 | 'kubernetes_sd_configs': [ 41 | cluster.get_kubernetes_sd_config('pod') 42 | ], 43 | 44 | # This TLS & bearer token file config is used to connect to the actual scrape 45 | # endpoints for cluster components. This is separate to discovery auth 46 | # configuration because discovery & scraping are two separate concerns in 47 | # Prometheus. The discovery auth config is automatic if Prometheus runs inside 48 | # the cluster. Otherwise, more config options have to be provided within the 49 | # . 50 | 'tls_config': { 51 | 'ca_file': cluster.ca_file 52 | }, 53 | 'bearer_token_file': cluster.bearer_token_file, 54 | 55 | 'relabel_configs': [ 56 | keep(source_labels=['__meta_kubernetes_pod_annotation_prometheus_io_scrape'], regex='true'), 57 | None, 58 | # set prometheus.io/filterport to false if prometheus.io/port is given 59 | replace(source_labels=['__meta_kubernetes_pod_annotation_prometheus_io_port'], 60 | regex='(.+)', replacement='false', 61 | target_label='__meta_kubernetes_pod_annotation_prometheus_io_filterport'), 62 | # keep all if prometheus.io/filterport is false or only matching else 63 | keep(source_labels=[ 64 | '__meta_kubernetes_pod_annotation_prometheus_io_filterport', 65 | '__meta_kubernetes_pod_container_port_name' 66 | ], 67 | separator=';', regex='(false;.*)|(true;.*metrics)'), 68 | # set prometheus.io/port to container port number if prometheus.io/filterport is true 69 | replace(source_labels=[ 70 | '__meta_kubernetes_pod_annotation_prometheus_io_filterport', 71 | '__meta_kubernetes_pod_container_port_number' 72 | ], 73 | separator=';', regex='true;(.+)', 74 | target_label='__meta_kubernetes_pod_annotation_prometheus_io_port'), 75 | # set container name label for distinction if prometheus.io/filterport is true 76 | replace(source_labels=[ 77 | '__meta_kubernetes_pod_annotation_prometheus_io_filterport', 78 | '__meta_kubernetes_pod_container_name' 79 | ], 80 | separator=';', regex='true;(.+)', 81 | target_label='kubernetes_container_name'), 82 | # allow overwriting scrape path via prometheus.io/path 83 | replace(source_labels=['__meta_kubernetes_pod_annotation_prometheus_io_path'], 84 | regex='(.+)', 85 | target_label='__metrics_path__'), 86 | # update address if prometheus.io/port is given 87 | replace(source_labels=['__address__', '__meta_kubernetes_pod_annotation_prometheus_io_port'], 88 | separator=';', regex='([^:]+)(?::\\d+)?;(\\d+)', replacement='$1:$2', 89 | target_label='__address__'), 90 | # rewrite scrape path to use Kubernetes apiserver proxy 91 | replace(source_labels=[ 92 | '__meta_kubernetes_namespace', '__meta_kubernetes_pod_name', 93 | '__meta_kubernetes_pod_annotation_prometheus_io_port', '__metrics_path__' 94 | ], 95 | separator=';', regex='(.+);(.+);(.+);(.+)', replacement='/api/v1/namespaces/$1/pods/$2:$3/proxy$4', 96 | target_label='__metrics_path__'), 97 | copy_value('__address__', 'instance'), 98 | set_value('__address__', f'{cluster.api_server}:443'), 99 | labelmap(regex='__meta_kubernetes_pod_label_(.+)'), 100 | copy_value('__meta_kubernetes_namespace', 'kubernetes_namespace'), 101 | copy_value('__meta_kubernetes_pod_name', 'kubernetes_pod_name'), 102 | remove_label('pod_template_hash'), 103 | remove_label('controller_revision_hash'), 104 | remove_label('pod_template_generation') 105 | ], 106 | 107 | 'metric_relabel_configs': [ 108 | drop(source_labels=['__name__'], regex='go_.*') 109 | ] 110 | }) # yapf: disable 111 | 112 | if interval_name == 'default': 113 | prom_conf['scrape_configs'][-1]['relabel_configs'][1] = \ 114 | drop(source_labels=['__meta_kubernetes_pod_annotation_prometheus_io_interval'], regex='.+') 115 | else: 116 | prom_conf['scrape_configs'][-1]['relabel_configs'][1] = \ 117 | keep(source_labels=['__meta_kubernetes_pod_annotation_prometheus_io_interval'], regex=interval_name) 118 | 119 | # set job's scrape_interval if defined 120 | if not interval_value is None: 121 | prom_conf['scrape_configs'][-1]['scrape_interval'] = interval_value 122 | 123 | # add additional configs 124 | prom_conf['scrape_configs'][-1]['relabel_configs'].extend(self.additional_relabel_configs) 125 | prom_conf['scrape_configs'][-1]['metric_relabel_configs'].extend(self.additional_metric_relabel_configs) 126 | -------------------------------------------------------------------------------- /promgen_k8s/prom_dsl.py: -------------------------------------------------------------------------------- 1 | import itertools 2 | from typing import (Dict) 3 | 4 | RelabelingStep = Dict # pylint: disable=invalid-name 5 | 6 | # Basic DSL 7 | # def rc_entry(**kwargs): 8 | # return kwargs 9 | 10 | 11 | # Intermediate DSL 12 | def drop(**kwargs) -> RelabelingStep: 13 | """ 14 | Combines all keyword arguments into a Prometheus relabeling step with action set to 'drop'. 15 | """ 16 | kwargs['action'] = 'drop' 17 | return kwargs 18 | 19 | 20 | def keep(**kwargs) -> RelabelingStep: 21 | """ 22 | Combines all keyword arguments into a Prometheus relabeling step with action set to 'keep'. 23 | """ 24 | kwargs['action'] = 'keep' 25 | return kwargs 26 | 27 | 28 | def labelmap(**kwargs) -> RelabelingStep: 29 | """ 30 | Combines all keyword arguments into a Prometheus relabeling step with action set to 'labelmap'. 31 | """ 32 | kwargs['action'] = 'labelmap' 33 | return kwargs 34 | 35 | 36 | def replace(**kwargs) -> RelabelingStep: 37 | """ 38 | Combines all keyword arguments into a Prometheus relabeling step with action set to 'replace'. 39 | """ 40 | kwargs['action'] = 'replace' 41 | return kwargs 42 | 43 | 44 | def rc_list(*things): 45 | """ 46 | Returns an iterable list 47 | """ 48 | iterables = [thing if isinstance(thing, list) else [thing] for thing in things] 49 | return list(itertools.chain(*iterables)) 50 | 51 | 52 | # Advanced DSL 53 | def set_value(label_name, value) -> RelabelingStep: 54 | # replace target with given value unconditionally 55 | return replace(replacement=value, target_label=label_name) 56 | 57 | 58 | def copy_value(src_label_name, tgt_label_name) -> RelabelingStep: 59 | # replace target with source unconditionally 60 | return replace(source_labels=[src_label_name], target_label=tgt_label_name) 61 | 62 | 63 | def remove_label(label_name) -> RelabelingStep: 64 | # set empty value 65 | return set_value(label_name, '') 66 | -------------------------------------------------------------------------------- /promgen_k8s/service_endpoints_job.py: -------------------------------------------------------------------------------- 1 | from typing import (Optional) 2 | 3 | from .cluster import * 4 | from .prom_dsl import * 5 | 6 | 7 | class ServiceEndpointsJob(GeneratorJob): 8 | 9 | def __init__(self, 10 | scrape_interval: Optional[str] = None, 11 | additional_relabel_configs: Optional[List[Any]] = None, 12 | additional_metric_relabel_configs: Optional[List[Any]] = None): 13 | self.type = 'service-endpoints' 14 | self.scrape_interval = scrape_interval 15 | self.additional_relabel_configs = additional_relabel_configs or [] 16 | self.additional_metric_relabel_configs = additional_metric_relabel_configs or [] 17 | 18 | # Scrape config for service endpoints. 19 | # 20 | # The relabeling allows the actual service scrape endpoint to be configured 21 | # via the following annotations: 22 | # 23 | # * `prometheus.io/scrape`: Only scrape services that have a value of `true` 24 | # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need 25 | # to set this to `https` & most likely set the `tls_config` of the scrape config. 26 | # * `prometheus.io/path`: If the metrics path is not `/metrics` override this. 27 | # * `prometheus.io/port`: If the metrics are exposed on a different port to the 28 | # service then set this appropriately. 29 | def generate(self, prom_conf: Dict[str, Any], cluster: Cluster) -> None: 30 | prom_conf['scrape_configs'].append({ 31 | 'job_name': f'{cluster.name}-kubernetes-service-endpoints', 32 | 'scheme': 'https', 33 | 'kubernetes_sd_configs': [ 34 | cluster.get_kubernetes_sd_config('endpoints') 35 | ], 36 | 37 | # This TLS & bearer token file config is used to connect to the actual scrape 38 | # endpoints for cluster components. This is separate to discovery auth 39 | # configuration because discovery & scraping are two separate concerns in 40 | # Prometheus. The discovery auth config is automatic if Prometheus runs inside 41 | # the cluster. Otherwise, more config options have to be provided within the 42 | # . 43 | 'tls_config': { 44 | 'ca_file': cluster.ca_file 45 | }, 46 | 'bearer_token_file': cluster.bearer_token_file, 47 | 48 | 'relabel_configs': [ 49 | keep(source_labels=['__meta_kubernetes_service_annotation_prometheus_io_scrape'], regex='true'), 50 | # Note: does not support any __meta_kubernetes_service_annotation_prometheus_io_scheme except HTTP 51 | replace(source_labels=['__meta_kubernetes_service_annotation_prometheus_io_path'], 52 | regex='(.+)', 53 | target_label='__metrics_path__'), 54 | replace(source_labels=['__address__', '__meta_kubernetes_service_annotation_prometheus_io_port'], 55 | regex='([^:]+)(?::\\d+)?;(\\d+)', replacement='$1:$2', 56 | target_label='__address__'), 57 | replace(source_labels=[ 58 | '__meta_kubernetes_namespace', '__meta_kubernetes_pod_name', 59 | '__meta_kubernetes_service_annotation_prometheus_io_port', '__metrics_path__' 60 | ], 61 | separator=';', regex='(.+);(.+);(.+);(.+)', replacement='/api/v1/namespaces/$1/pods/$2:$3/proxy$4', 62 | target_label='__metrics_path__'), 63 | copy_value('__address__', 'instance'), 64 | set_value('__address__', f'{cluster.api_server}:443'), 65 | labelmap(regex='__meta_kubernetes_service_label_(.+)'), 66 | copy_value('__meta_kubernetes_namespace', 'kubernetes_namespace'), 67 | copy_value('__meta_kubernetes_service_name', 'kubernetes_service_name') 68 | ] 69 | }) # yapf: disable 70 | 71 | # set job's scrape_interval if defined 72 | if not self.scrape_interval is None: 73 | prom_conf['scrape_configs'][-1]['scrape_interval'] = self.scrape_interval 74 | 75 | # add additional configs 76 | prom_conf['scrape_configs'][-1]['relabel_configs'].extend(self.additional_relabel_configs) 77 | prom_conf['scrape_configs'][-1]['metric_relabel_configs'] = self.additional_metric_relabel_configs 78 | 79 | 80 | # backwards-compatibiliy for typo 81 | class ServiceEndpoitsJob(ServiceEndpointsJob): 82 | pass 83 | -------------------------------------------------------------------------------- /promgen_k8s/services_job.py: -------------------------------------------------------------------------------- 1 | from typing import (Optional) 2 | 3 | from .cluster import * 4 | from .prom_dsl import * 5 | 6 | 7 | class ServicesJob(GeneratorJob): 8 | 9 | def __init__(self, 10 | scrape_interval: Optional[str] = None, 11 | additional_relabel_configs: Optional[List[Any]] = None, 12 | additional_metric_relabel_configs: Optional[List[Any]] = None): 13 | self.type = 'services' 14 | self.scrape_interval = scrape_interval 15 | self.additional_relabel_configs = additional_relabel_configs or [] 16 | self.additional_metric_relabel_configs = additional_metric_relabel_configs or [] 17 | 18 | # Example scrape config for probing services via the Blackbox Exporter. 19 | # 20 | # The relabeling allows the actual service scrape endpoint to be configured 21 | # via the following annotations: 22 | # 23 | # * `prometheus.io/probe`: Only probe services that have a value of `true` 24 | # * `prometheus.io/path`: If the probe path is not `/` override this. 25 | # * `prometheus.io/module`: If the Blackbox exporter module used is not named `http_2xx` override this. 26 | def generate(self, prom_conf: Dict[str, Any], cluster: Cluster) -> None: 27 | prom_conf['scrape_configs'].append({ 28 | 'job_name': f'{cluster.name}-kubernetes-services', 29 | 'scheme': 'https', 30 | 'kubernetes_sd_configs': [ 31 | cluster.get_kubernetes_sd_config('service') 32 | ], 33 | 34 | # This TLS & bearer token file config is used to connect to the actual scrape 35 | # endpoints for cluster components. This is separate to discovery auth 36 | # configuration because discovery & scraping are two separate concerns in 37 | # Prometheus. The discovery auth config is automatic if Prometheus runs inside 38 | # the cluster. Otherwise, more config options have to be provided within the 39 | # . 40 | 'tls_config': { 41 | 'ca_file': cluster.ca_file 42 | }, 43 | 'bearer_token_file': cluster.bearer_token_file, 44 | 45 | 'metrics_path': '/api/v1/namespaces/monitoring/services/blackbox-exporter/proxy/probe', 46 | 'params': { 47 | 'module': ['http_2xx'] 48 | }, 49 | 50 | 'relabel_configs': [ 51 | keep(source_labels=['__meta_kubernetes_service_annotation_prometheus_io_probe'], regex='true'), 52 | copy_value('__meta_kubernetes_service_annotation_prometheus_io_module', '__param_module'), 53 | copy_value('__address__', '__param_target'), 54 | replace(source_labels=['__param_target', '__meta_kubernetes_service_annotation_prometheus_io_path'], 55 | separator=';', regex='(.+);(.+)', replacement='$1$2', 56 | target_label='__param_target'), 57 | copy_value('__address__', 'instance'), 58 | set_value('__address__', f'{cluster.api_server}:443'), 59 | labelmap(regex='__meta_kubernetes_service_label_(.+)'), 60 | copy_value('__meta_kubernetes_namespace', 'kubernetes_namespace'), 61 | copy_value('__meta_kubernetes_service_name', 'kubernetes_service_name') 62 | ] 63 | }) # yapf: disable 64 | 65 | # set job's scrape_interval if defined 66 | if not self.scrape_interval is None: 67 | prom_conf['scrape_configs'][-1]['scrape_interval'] = self.scrape_interval 68 | 69 | # add additional_relabel_configs 70 | prom_conf['scrape_configs'][-1]['relabel_configs'].extend(self.additional_relabel_configs) 71 | prom_conf['scrape_configs'][-1]['metric_relabel_configs'] = self.additional_metric_relabel_configs 72 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["poetry>=1.0"] 3 | build-backend = "poetry.masonry.api" 4 | 5 | [tool.poetry] 6 | name = "promgen-k8s" 7 | version = "0.3.0" 8 | description = "A modular Prometheus 2 configuration file generator to monitor multiple Kubernetes clusters with a single Prometheus instance." 9 | authors = ["Christian Nicolai"] 10 | license = "Apache-2.0" 11 | homepage = "https://github.com/cmur2/promgen-k8s" 12 | include = ["CHANGELOG.md", "README.md"] 13 | 14 | [tool.poetry.dependencies] 15 | python = "^3.7" 16 | pyyaml = "*" 17 | 18 | [tool.poetry.dev-dependencies] 19 | pylint = "*" 20 | pytest = "*" 21 | snapshottest = "*" 22 | rope = "*" 23 | toml = "*" 24 | yapf = "*" 25 | mypy = "*" 26 | types-pyyaml = "*" 27 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cmur2/promgen-k8s/f2d0f80f39bf8c637ec2de7dbac216eefad1a089/tests/__init__.py -------------------------------------------------------------------------------- /tests/promgen_k8s_test.py: -------------------------------------------------------------------------------- 1 | # import unittest 2 | import io 3 | 4 | import snapshottest 5 | 6 | import promgen_k8s as g 7 | from promgen_k8s.prom_dsl import * 8 | 9 | 10 | class TestPromgenK8s(snapshottest.TestCase): 11 | def test_generate(self): 12 | stub_prom_conf = { 13 | 'global': { 14 | 'scrape_interval': '1m', 15 | 'evaluation_interval': '1m' 16 | }, 17 | 'rule_files': ['/etc/alert.rules'], 18 | 'scrape_configs': [] 19 | } 20 | clusters = [ 21 | g.Cluster('staging', jobs=[ 22 | g.NodesJob(), 23 | g.IngressesJob(), 24 | g.CadvisorJob(), 25 | g.ServiceEndpointsJob(), 26 | g.ServicesJob(additional_relabel_configs=[ 27 | # reverse.default.svc:443 should prompt for authentication in staging environment 28 | replace(source_labels=['__param_target'], 29 | regex='reverse.default.svc:443', replacement='https_401', 30 | target_label='__param_module') 31 | ]), 32 | g.PodsJob() 33 | ]), 34 | g.Cluster('production', jobs=[ 35 | g.NodesJob(), 36 | g.IngressesJob(), 37 | g.CadvisorJob(), 38 | g.ServiceEndpointsJob(), 39 | g.ServicesJob(), 40 | g.PodsJob() 41 | ]), 42 | # The incluster property marks the cluster in which Prometheus in running 43 | g.Cluster('operations', incluster=True, jobs=[ 44 | g.NodesJob(), 45 | g.IngressesJob(), 46 | g.CadvisorJob(), 47 | g.ServiceEndpointsJob(), 48 | g.ServicesJob(), 49 | g.PodsJob(interval_map={'long': '1h'}) 50 | ]) 51 | ] # yapf: disable 52 | generator = g.Generator(clusters, initial_prom_conf=stub_prom_conf) 53 | self.assertIsNotNone(generator) 54 | tmp_file = io.BytesIO() 55 | generator.dump(tmp_file) 56 | self.assertMatchSnapshot(tmp_file.getvalue().decode('utf-8').split('\n')) 57 | tmp_file.close() 58 | -------------------------------------------------------------------------------- /tests/snapshots/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cmur2/promgen-k8s/f2d0f80f39bf8c637ec2de7dbac216eefad1a089/tests/snapshots/__init__.py -------------------------------------------------------------------------------- /tests/snapshots/snap_promgen_k8s_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # snapshottest: v1 - https://goo.gl/zC4yUc 3 | from __future__ import unicode_literals 4 | 5 | from snapshottest import Snapshot 6 | 7 | 8 | snapshots = Snapshot() 9 | 10 | snapshots['TestPromgenK8s::test_generate 1'] = [ 11 | '---', 12 | 'global:', 13 | ' evaluation_interval: 1m', 14 | ' scrape_interval: 1m', 15 | 'rule_files:', 16 | ' - /etc/alert.rules', 17 | 'scrape_configs:', 18 | ' - bearer_token_file: /var/run/kube_secrets/staging_bearer_token', 19 | ' job_name: staging-kubernetes-nodes', 20 | ' kubernetes_sd_configs:', 21 | ' - api_server: https://api.internal.staging.example.com', 22 | ' bearer_token_file: /var/run/kube_secrets/staging_bearer_token', 23 | ' role: node', 24 | ' tls_config:', 25 | ' ca_file: /var/run/kube_secrets/staging_ca_crt', 26 | ' metric_relabel_configs:', 27 | ' - action: drop', 28 | ' regex: go_.*', 29 | ' source_labels:', 30 | ' - __name__', 31 | ' relabel_configs:', 32 | ' - action: replace', 33 | " replacement: ''", 34 | ' target_label: __meta_kubernetes_node_label_node_role_kubernetes_io_node', 35 | ' - action: replace', 36 | " replacement: ''", 37 | ' target_label: __meta_kubernetes_node_label_node_role_kubernetes_io_master', 38 | ' - action: labelmap', 39 | ' regex: __meta_kubernetes_node_label_(.+)', 40 | ' - action: replace', 41 | ' source_labels:', 42 | ' - __address__', 43 | ' target_label: instance', 44 | ' - action: replace', 45 | ' replacement: api.internal.staging.example.com:443', 46 | ' target_label: __address__', 47 | ' - action: replace', 48 | ' regex: (.+)', 49 | ' replacement: /api/v1/nodes/${1}:10255/proxy/metrics', 50 | ' source_labels:', 51 | ' - __meta_kubernetes_node_name', 52 | ' target_label: __metrics_path__', 53 | ' scheme: https', 54 | ' tls_config:', 55 | ' ca_file: /var/run/kube_secrets/staging_ca_crt', 56 | ' - bearer_token_file: /var/run/kube_secrets/staging_bearer_token', 57 | ' job_name: staging-kubernetes-ingresses', 58 | ' kubernetes_sd_configs:', 59 | ' - api_server: https://api.internal.staging.example.com', 60 | ' bearer_token_file: /var/run/kube_secrets/staging_bearer_token', 61 | ' role: ingress', 62 | ' tls_config:', 63 | ' ca_file: /var/run/kube_secrets/staging_ca_crt', 64 | ' metric_relabel_configs: []', 65 | ' metrics_path: /api/v1/namespaces/monitoring/services/blackbox-exporter/proxy/probe', 66 | ' params:', 67 | ' module:', 68 | ' - http_2xx', 69 | ' relabel_configs:', 70 | ' - action: keep', 71 | " regex: 'true'", 72 | ' source_labels:', 73 | ' - __meta_kubernetes_ingress_annotation_prometheus_io_probe', 74 | ' - action: replace', 75 | ' source_labels:', 76 | ' - __meta_kubernetes_ingress_annotation_prometheus_io_module', 77 | ' target_label: __param_module', 78 | ' - action: replace', 79 | ' regex: (.+);(.+)', 80 | ' replacement: $1$2', 81 | ' separator: ;', 82 | ' source_labels:', 83 | ' - __address__', 84 | ' - __meta_kubernetes_ingress_path', 85 | ' target_label: __address__', 86 | ' - action: replace', 87 | ' source_labels:', 88 | ' - __address__', 89 | ' target_label: __param_target', 90 | ' - action: replace', 91 | ' source_labels:', 92 | ' - __address__', 93 | ' target_label: instance', 94 | ' - action: replace', 95 | ' replacement: api.internal.staging.example.com:443', 96 | ' target_label: __address__', 97 | ' - action: labelmap', 98 | ' regex: __meta_kubernetes_ingress_label_(.+)', 99 | ' - action: replace', 100 | ' source_labels:', 101 | ' - __meta_kubernetes_namespace', 102 | ' target_label: kubernetes_namespace', 103 | ' - action: replace', 104 | ' source_labels:', 105 | ' - __meta_kubernetes_ingress_name', 106 | ' target_label: kubernetes_ingress_name', 107 | ' scheme: https', 108 | ' tls_config:', 109 | ' ca_file: /var/run/kube_secrets/staging_ca_crt', 110 | ' - bearer_token_file: /var/run/kube_secrets/staging_bearer_token', 111 | ' job_name: staging-kubernetes-cadvisor', 112 | ' kubernetes_sd_configs:', 113 | ' - api_server: https://api.internal.staging.example.com', 114 | ' bearer_token_file: /var/run/kube_secrets/staging_bearer_token', 115 | ' role: node', 116 | ' tls_config:', 117 | ' ca_file: /var/run/kube_secrets/staging_ca_crt', 118 | ' metric_relabel_configs:', 119 | ' - action: replace', 120 | ' regex: ^/machine\\.slice/machine-rkt\\\\x2d([^\\\\]+)\\\\.+/([^/]+)\\.service$', 121 | ' replacement: ${2}-${1}', 122 | ' source_labels:', 123 | ' - id', 124 | ' target_label: rkt_container_name', 125 | ' - action: replace', 126 | ' regex: ^/system\\.slice/(.+)\\.service$', 127 | ' replacement: ${1}', 128 | ' source_labels:', 129 | ' - id', 130 | ' target_label: systemd_service_name', 131 | ' - action: drop', 132 | ' regex: go_.*', 133 | ' source_labels:', 134 | ' - __name__', 135 | ' relabel_configs:', 136 | ' - action: replace', 137 | " replacement: ''", 138 | ' target_label: __meta_kubernetes_node_label_node_role_kubernetes_io_node', 139 | ' - action: replace', 140 | " replacement: ''", 141 | ' target_label: __meta_kubernetes_node_label_node_role_kubernetes_io_master', 142 | ' - action: labelmap', 143 | ' regex: __meta_kubernetes_node_label_(.+)', 144 | ' - action: replace', 145 | ' source_labels:', 146 | ' - __address__', 147 | ' target_label: instance', 148 | ' - action: replace', 149 | ' replacement: api.internal.staging.example.com:443', 150 | ' target_label: __address__', 151 | ' - action: replace', 152 | ' regex: (.+)', 153 | ' replacement: /api/v1/nodes/${1}:10255/proxy/metrics/cadvisor', 154 | ' source_labels:', 155 | ' - __meta_kubernetes_node_name', 156 | ' target_label: __metrics_path__', 157 | ' scheme: https', 158 | ' tls_config:', 159 | ' ca_file: /var/run/kube_secrets/staging_ca_crt', 160 | ' - bearer_token_file: /var/run/kube_secrets/staging_bearer_token', 161 | ' job_name: staging-kubernetes-service-endpoints', 162 | ' kubernetes_sd_configs:', 163 | ' - api_server: https://api.internal.staging.example.com', 164 | ' bearer_token_file: /var/run/kube_secrets/staging_bearer_token', 165 | ' role: endpoints', 166 | ' tls_config:', 167 | ' ca_file: /var/run/kube_secrets/staging_ca_crt', 168 | ' metric_relabel_configs: []', 169 | ' relabel_configs:', 170 | ' - action: keep', 171 | " regex: 'true'", 172 | ' source_labels:', 173 | ' - __meta_kubernetes_service_annotation_prometheus_io_scrape', 174 | ' - action: replace', 175 | ' regex: (.+)', 176 | ' source_labels:', 177 | ' - __meta_kubernetes_service_annotation_prometheus_io_path', 178 | ' target_label: __metrics_path__', 179 | ' - action: replace', 180 | ' regex: ([^:]+)(?::\\d+)?;(\\d+)', 181 | ' replacement: $1:$2', 182 | ' source_labels:', 183 | ' - __address__', 184 | ' - __meta_kubernetes_service_annotation_prometheus_io_port', 185 | ' target_label: __address__', 186 | ' - action: replace', 187 | ' regex: (.+);(.+);(.+);(.+)', 188 | ' replacement: /api/v1/namespaces/$1/pods/$2:$3/proxy$4', 189 | ' separator: ;', 190 | ' source_labels:', 191 | ' - __meta_kubernetes_namespace', 192 | ' - __meta_kubernetes_pod_name', 193 | ' - __meta_kubernetes_service_annotation_prometheus_io_port', 194 | ' - __metrics_path__', 195 | ' target_label: __metrics_path__', 196 | ' - action: replace', 197 | ' source_labels:', 198 | ' - __address__', 199 | ' target_label: instance', 200 | ' - action: replace', 201 | ' replacement: api.internal.staging.example.com:443', 202 | ' target_label: __address__', 203 | ' - action: labelmap', 204 | ' regex: __meta_kubernetes_service_label_(.+)', 205 | ' - action: replace', 206 | ' source_labels:', 207 | ' - __meta_kubernetes_namespace', 208 | ' target_label: kubernetes_namespace', 209 | ' - action: replace', 210 | ' source_labels:', 211 | ' - __meta_kubernetes_service_name', 212 | ' target_label: kubernetes_service_name', 213 | ' scheme: https', 214 | ' tls_config:', 215 | ' ca_file: /var/run/kube_secrets/staging_ca_crt', 216 | ' - bearer_token_file: /var/run/kube_secrets/staging_bearer_token', 217 | ' job_name: staging-kubernetes-services', 218 | ' kubernetes_sd_configs:', 219 | ' - api_server: https://api.internal.staging.example.com', 220 | ' bearer_token_file: /var/run/kube_secrets/staging_bearer_token', 221 | ' role: service', 222 | ' tls_config:', 223 | ' ca_file: /var/run/kube_secrets/staging_ca_crt', 224 | ' metric_relabel_configs: []', 225 | ' metrics_path: /api/v1/namespaces/monitoring/services/blackbox-exporter/proxy/probe', 226 | ' params:', 227 | ' module:', 228 | ' - http_2xx', 229 | ' relabel_configs:', 230 | ' - action: keep', 231 | " regex: 'true'", 232 | ' source_labels:', 233 | ' - __meta_kubernetes_service_annotation_prometheus_io_probe', 234 | ' - action: replace', 235 | ' source_labels:', 236 | ' - __meta_kubernetes_service_annotation_prometheus_io_module', 237 | ' target_label: __param_module', 238 | ' - action: replace', 239 | ' source_labels:', 240 | ' - __address__', 241 | ' target_label: __param_target', 242 | ' - action: replace', 243 | ' regex: (.+);(.+)', 244 | ' replacement: $1$2', 245 | ' separator: ;', 246 | ' source_labels:', 247 | ' - __param_target', 248 | ' - __meta_kubernetes_service_annotation_prometheus_io_path', 249 | ' target_label: __param_target', 250 | ' - action: replace', 251 | ' source_labels:', 252 | ' - __address__', 253 | ' target_label: instance', 254 | ' - action: replace', 255 | ' replacement: api.internal.staging.example.com:443', 256 | ' target_label: __address__', 257 | ' - action: labelmap', 258 | ' regex: __meta_kubernetes_service_label_(.+)', 259 | ' - action: replace', 260 | ' source_labels:', 261 | ' - __meta_kubernetes_namespace', 262 | ' target_label: kubernetes_namespace', 263 | ' - action: replace', 264 | ' source_labels:', 265 | ' - __meta_kubernetes_service_name', 266 | ' target_label: kubernetes_service_name', 267 | ' - action: replace', 268 | ' regex: reverse.default.svc:443', 269 | ' replacement: https_401', 270 | ' source_labels:', 271 | ' - __param_target', 272 | ' target_label: __param_module', 273 | ' scheme: https', 274 | ' tls_config:', 275 | ' ca_file: /var/run/kube_secrets/staging_ca_crt', 276 | ' - bearer_token_file: /var/run/kube_secrets/staging_bearer_token', 277 | ' job_name: staging-kubernetes-pods-default', 278 | ' kubernetes_sd_configs:', 279 | ' - api_server: https://api.internal.staging.example.com', 280 | ' bearer_token_file: /var/run/kube_secrets/staging_bearer_token', 281 | ' role: pod', 282 | ' tls_config:', 283 | ' ca_file: /var/run/kube_secrets/staging_ca_crt', 284 | ' metric_relabel_configs:', 285 | ' - action: drop', 286 | ' regex: go_.*', 287 | ' source_labels:', 288 | ' - __name__', 289 | ' relabel_configs:', 290 | ' - action: keep', 291 | " regex: 'true'", 292 | ' source_labels:', 293 | ' - __meta_kubernetes_pod_annotation_prometheus_io_scrape', 294 | ' - action: drop', 295 | ' regex: .+', 296 | ' source_labels:', 297 | ' - __meta_kubernetes_pod_annotation_prometheus_io_interval', 298 | ' - action: replace', 299 | ' regex: (.+)', 300 | " replacement: 'false'", 301 | ' source_labels:', 302 | ' - __meta_kubernetes_pod_annotation_prometheus_io_port', 303 | ' target_label: __meta_kubernetes_pod_annotation_prometheus_io_filterport', 304 | ' - action: keep', 305 | ' regex: (false;.*)|(true;.*metrics)', 306 | ' separator: ;', 307 | ' source_labels:', 308 | ' - __meta_kubernetes_pod_annotation_prometheus_io_filterport', 309 | ' - __meta_kubernetes_pod_container_port_name', 310 | ' - action: replace', 311 | ' regex: true;(.+)', 312 | ' separator: ;', 313 | ' source_labels:', 314 | ' - __meta_kubernetes_pod_annotation_prometheus_io_filterport', 315 | ' - __meta_kubernetes_pod_container_port_number', 316 | ' target_label: __meta_kubernetes_pod_annotation_prometheus_io_port', 317 | ' - action: replace', 318 | ' regex: true;(.+)', 319 | ' separator: ;', 320 | ' source_labels:', 321 | ' - __meta_kubernetes_pod_annotation_prometheus_io_filterport', 322 | ' - __meta_kubernetes_pod_container_name', 323 | ' target_label: kubernetes_container_name', 324 | ' - action: replace', 325 | ' regex: (.+)', 326 | ' source_labels:', 327 | ' - __meta_kubernetes_pod_annotation_prometheus_io_path', 328 | ' target_label: __metrics_path__', 329 | ' - action: replace', 330 | ' regex: ([^:]+)(?::\\d+)?;(\\d+)', 331 | ' replacement: $1:$2', 332 | ' separator: ;', 333 | ' source_labels:', 334 | ' - __address__', 335 | ' - __meta_kubernetes_pod_annotation_prometheus_io_port', 336 | ' target_label: __address__', 337 | ' - action: replace', 338 | ' regex: (.+);(.+);(.+);(.+)', 339 | ' replacement: /api/v1/namespaces/$1/pods/$2:$3/proxy$4', 340 | ' separator: ;', 341 | ' source_labels:', 342 | ' - __meta_kubernetes_namespace', 343 | ' - __meta_kubernetes_pod_name', 344 | ' - __meta_kubernetes_pod_annotation_prometheus_io_port', 345 | ' - __metrics_path__', 346 | ' target_label: __metrics_path__', 347 | ' - action: replace', 348 | ' source_labels:', 349 | ' - __address__', 350 | ' target_label: instance', 351 | ' - action: replace', 352 | ' replacement: api.internal.staging.example.com:443', 353 | ' target_label: __address__', 354 | ' - action: labelmap', 355 | ' regex: __meta_kubernetes_pod_label_(.+)', 356 | ' - action: replace', 357 | ' source_labels:', 358 | ' - __meta_kubernetes_namespace', 359 | ' target_label: kubernetes_namespace', 360 | ' - action: replace', 361 | ' source_labels:', 362 | ' - __meta_kubernetes_pod_name', 363 | ' target_label: kubernetes_pod_name', 364 | ' - action: replace', 365 | " replacement: ''", 366 | ' target_label: pod_template_hash', 367 | ' - action: replace', 368 | " replacement: ''", 369 | ' target_label: controller_revision_hash', 370 | ' - action: replace', 371 | " replacement: ''", 372 | ' target_label: pod_template_generation', 373 | ' scheme: https', 374 | ' tls_config:', 375 | ' ca_file: /var/run/kube_secrets/staging_ca_crt', 376 | ' - bearer_token_file: /var/run/kube_secrets/production_bearer_token', 377 | ' job_name: production-kubernetes-nodes', 378 | ' kubernetes_sd_configs:', 379 | ' - api_server: https://api.internal.production.example.com', 380 | ' bearer_token_file: /var/run/kube_secrets/production_bearer_token', 381 | ' role: node', 382 | ' tls_config:', 383 | ' ca_file: /var/run/kube_secrets/production_ca_crt', 384 | ' metric_relabel_configs:', 385 | ' - action: drop', 386 | ' regex: go_.*', 387 | ' source_labels:', 388 | ' - __name__', 389 | ' relabel_configs:', 390 | ' - action: replace', 391 | " replacement: ''", 392 | ' target_label: __meta_kubernetes_node_label_node_role_kubernetes_io_node', 393 | ' - action: replace', 394 | " replacement: ''", 395 | ' target_label: __meta_kubernetes_node_label_node_role_kubernetes_io_master', 396 | ' - action: labelmap', 397 | ' regex: __meta_kubernetes_node_label_(.+)', 398 | ' - action: replace', 399 | ' source_labels:', 400 | ' - __address__', 401 | ' target_label: instance', 402 | ' - action: replace', 403 | ' replacement: api.internal.production.example.com:443', 404 | ' target_label: __address__', 405 | ' - action: replace', 406 | ' regex: (.+)', 407 | ' replacement: /api/v1/nodes/${1}:10255/proxy/metrics', 408 | ' source_labels:', 409 | ' - __meta_kubernetes_node_name', 410 | ' target_label: __metrics_path__', 411 | ' scheme: https', 412 | ' tls_config:', 413 | ' ca_file: /var/run/kube_secrets/production_ca_crt', 414 | ' - bearer_token_file: /var/run/kube_secrets/production_bearer_token', 415 | ' job_name: production-kubernetes-ingresses', 416 | ' kubernetes_sd_configs:', 417 | ' - api_server: https://api.internal.production.example.com', 418 | ' bearer_token_file: /var/run/kube_secrets/production_bearer_token', 419 | ' role: ingress', 420 | ' tls_config:', 421 | ' ca_file: /var/run/kube_secrets/production_ca_crt', 422 | ' metric_relabel_configs: []', 423 | ' metrics_path: /api/v1/namespaces/monitoring/services/blackbox-exporter/proxy/probe', 424 | ' params:', 425 | ' module:', 426 | ' - http_2xx', 427 | ' relabel_configs:', 428 | ' - action: keep', 429 | " regex: 'true'", 430 | ' source_labels:', 431 | ' - __meta_kubernetes_ingress_annotation_prometheus_io_probe', 432 | ' - action: replace', 433 | ' source_labels:', 434 | ' - __meta_kubernetes_ingress_annotation_prometheus_io_module', 435 | ' target_label: __param_module', 436 | ' - action: replace', 437 | ' regex: (.+);(.+)', 438 | ' replacement: $1$2', 439 | ' separator: ;', 440 | ' source_labels:', 441 | ' - __address__', 442 | ' - __meta_kubernetes_ingress_path', 443 | ' target_label: __address__', 444 | ' - action: replace', 445 | ' source_labels:', 446 | ' - __address__', 447 | ' target_label: __param_target', 448 | ' - action: replace', 449 | ' source_labels:', 450 | ' - __address__', 451 | ' target_label: instance', 452 | ' - action: replace', 453 | ' replacement: api.internal.production.example.com:443', 454 | ' target_label: __address__', 455 | ' - action: labelmap', 456 | ' regex: __meta_kubernetes_ingress_label_(.+)', 457 | ' - action: replace', 458 | ' source_labels:', 459 | ' - __meta_kubernetes_namespace', 460 | ' target_label: kubernetes_namespace', 461 | ' - action: replace', 462 | ' source_labels:', 463 | ' - __meta_kubernetes_ingress_name', 464 | ' target_label: kubernetes_ingress_name', 465 | ' scheme: https', 466 | ' tls_config:', 467 | ' ca_file: /var/run/kube_secrets/production_ca_crt', 468 | ' - bearer_token_file: /var/run/kube_secrets/production_bearer_token', 469 | ' job_name: production-kubernetes-cadvisor', 470 | ' kubernetes_sd_configs:', 471 | ' - api_server: https://api.internal.production.example.com', 472 | ' bearer_token_file: /var/run/kube_secrets/production_bearer_token', 473 | ' role: node', 474 | ' tls_config:', 475 | ' ca_file: /var/run/kube_secrets/production_ca_crt', 476 | ' metric_relabel_configs:', 477 | ' - action: replace', 478 | ' regex: ^/machine\\.slice/machine-rkt\\\\x2d([^\\\\]+)\\\\.+/([^/]+)\\.service$', 479 | ' replacement: ${2}-${1}', 480 | ' source_labels:', 481 | ' - id', 482 | ' target_label: rkt_container_name', 483 | ' - action: replace', 484 | ' regex: ^/system\\.slice/(.+)\\.service$', 485 | ' replacement: ${1}', 486 | ' source_labels:', 487 | ' - id', 488 | ' target_label: systemd_service_name', 489 | ' - action: drop', 490 | ' regex: go_.*', 491 | ' source_labels:', 492 | ' - __name__', 493 | ' relabel_configs:', 494 | ' - action: replace', 495 | " replacement: ''", 496 | ' target_label: __meta_kubernetes_node_label_node_role_kubernetes_io_node', 497 | ' - action: replace', 498 | " replacement: ''", 499 | ' target_label: __meta_kubernetes_node_label_node_role_kubernetes_io_master', 500 | ' - action: labelmap', 501 | ' regex: __meta_kubernetes_node_label_(.+)', 502 | ' - action: replace', 503 | ' source_labels:', 504 | ' - __address__', 505 | ' target_label: instance', 506 | ' - action: replace', 507 | ' replacement: api.internal.production.example.com:443', 508 | ' target_label: __address__', 509 | ' - action: replace', 510 | ' regex: (.+)', 511 | ' replacement: /api/v1/nodes/${1}:10255/proxy/metrics/cadvisor', 512 | ' source_labels:', 513 | ' - __meta_kubernetes_node_name', 514 | ' target_label: __metrics_path__', 515 | ' scheme: https', 516 | ' tls_config:', 517 | ' ca_file: /var/run/kube_secrets/production_ca_crt', 518 | ' - bearer_token_file: /var/run/kube_secrets/production_bearer_token', 519 | ' job_name: production-kubernetes-service-endpoints', 520 | ' kubernetes_sd_configs:', 521 | ' - api_server: https://api.internal.production.example.com', 522 | ' bearer_token_file: /var/run/kube_secrets/production_bearer_token', 523 | ' role: endpoints', 524 | ' tls_config:', 525 | ' ca_file: /var/run/kube_secrets/production_ca_crt', 526 | ' metric_relabel_configs: []', 527 | ' relabel_configs:', 528 | ' - action: keep', 529 | " regex: 'true'", 530 | ' source_labels:', 531 | ' - __meta_kubernetes_service_annotation_prometheus_io_scrape', 532 | ' - action: replace', 533 | ' regex: (.+)', 534 | ' source_labels:', 535 | ' - __meta_kubernetes_service_annotation_prometheus_io_path', 536 | ' target_label: __metrics_path__', 537 | ' - action: replace', 538 | ' regex: ([^:]+)(?::\\d+)?;(\\d+)', 539 | ' replacement: $1:$2', 540 | ' source_labels:', 541 | ' - __address__', 542 | ' - __meta_kubernetes_service_annotation_prometheus_io_port', 543 | ' target_label: __address__', 544 | ' - action: replace', 545 | ' regex: (.+);(.+);(.+);(.+)', 546 | ' replacement: /api/v1/namespaces/$1/pods/$2:$3/proxy$4', 547 | ' separator: ;', 548 | ' source_labels:', 549 | ' - __meta_kubernetes_namespace', 550 | ' - __meta_kubernetes_pod_name', 551 | ' - __meta_kubernetes_service_annotation_prometheus_io_port', 552 | ' - __metrics_path__', 553 | ' target_label: __metrics_path__', 554 | ' - action: replace', 555 | ' source_labels:', 556 | ' - __address__', 557 | ' target_label: instance', 558 | ' - action: replace', 559 | ' replacement: api.internal.production.example.com:443', 560 | ' target_label: __address__', 561 | ' - action: labelmap', 562 | ' regex: __meta_kubernetes_service_label_(.+)', 563 | ' - action: replace', 564 | ' source_labels:', 565 | ' - __meta_kubernetes_namespace', 566 | ' target_label: kubernetes_namespace', 567 | ' - action: replace', 568 | ' source_labels:', 569 | ' - __meta_kubernetes_service_name', 570 | ' target_label: kubernetes_service_name', 571 | ' scheme: https', 572 | ' tls_config:', 573 | ' ca_file: /var/run/kube_secrets/production_ca_crt', 574 | ' - bearer_token_file: /var/run/kube_secrets/production_bearer_token', 575 | ' job_name: production-kubernetes-services', 576 | ' kubernetes_sd_configs:', 577 | ' - api_server: https://api.internal.production.example.com', 578 | ' bearer_token_file: /var/run/kube_secrets/production_bearer_token', 579 | ' role: service', 580 | ' tls_config:', 581 | ' ca_file: /var/run/kube_secrets/production_ca_crt', 582 | ' metric_relabel_configs: []', 583 | ' metrics_path: /api/v1/namespaces/monitoring/services/blackbox-exporter/proxy/probe', 584 | ' params:', 585 | ' module:', 586 | ' - http_2xx', 587 | ' relabel_configs:', 588 | ' - action: keep', 589 | " regex: 'true'", 590 | ' source_labels:', 591 | ' - __meta_kubernetes_service_annotation_prometheus_io_probe', 592 | ' - action: replace', 593 | ' source_labels:', 594 | ' - __meta_kubernetes_service_annotation_prometheus_io_module', 595 | ' target_label: __param_module', 596 | ' - action: replace', 597 | ' source_labels:', 598 | ' - __address__', 599 | ' target_label: __param_target', 600 | ' - action: replace', 601 | ' regex: (.+);(.+)', 602 | ' replacement: $1$2', 603 | ' separator: ;', 604 | ' source_labels:', 605 | ' - __param_target', 606 | ' - __meta_kubernetes_service_annotation_prometheus_io_path', 607 | ' target_label: __param_target', 608 | ' - action: replace', 609 | ' source_labels:', 610 | ' - __address__', 611 | ' target_label: instance', 612 | ' - action: replace', 613 | ' replacement: api.internal.production.example.com:443', 614 | ' target_label: __address__', 615 | ' - action: labelmap', 616 | ' regex: __meta_kubernetes_service_label_(.+)', 617 | ' - action: replace', 618 | ' source_labels:', 619 | ' - __meta_kubernetes_namespace', 620 | ' target_label: kubernetes_namespace', 621 | ' - action: replace', 622 | ' source_labels:', 623 | ' - __meta_kubernetes_service_name', 624 | ' target_label: kubernetes_service_name', 625 | ' scheme: https', 626 | ' tls_config:', 627 | ' ca_file: /var/run/kube_secrets/production_ca_crt', 628 | ' - bearer_token_file: /var/run/kube_secrets/production_bearer_token', 629 | ' job_name: production-kubernetes-pods-default', 630 | ' kubernetes_sd_configs:', 631 | ' - api_server: https://api.internal.production.example.com', 632 | ' bearer_token_file: /var/run/kube_secrets/production_bearer_token', 633 | ' role: pod', 634 | ' tls_config:', 635 | ' ca_file: /var/run/kube_secrets/production_ca_crt', 636 | ' metric_relabel_configs:', 637 | ' - action: drop', 638 | ' regex: go_.*', 639 | ' source_labels:', 640 | ' - __name__', 641 | ' relabel_configs:', 642 | ' - action: keep', 643 | " regex: 'true'", 644 | ' source_labels:', 645 | ' - __meta_kubernetes_pod_annotation_prometheus_io_scrape', 646 | ' - action: drop', 647 | ' regex: .+', 648 | ' source_labels:', 649 | ' - __meta_kubernetes_pod_annotation_prometheus_io_interval', 650 | ' - action: replace', 651 | ' regex: (.+)', 652 | " replacement: 'false'", 653 | ' source_labels:', 654 | ' - __meta_kubernetes_pod_annotation_prometheus_io_port', 655 | ' target_label: __meta_kubernetes_pod_annotation_prometheus_io_filterport', 656 | ' - action: keep', 657 | ' regex: (false;.*)|(true;.*metrics)', 658 | ' separator: ;', 659 | ' source_labels:', 660 | ' - __meta_kubernetes_pod_annotation_prometheus_io_filterport', 661 | ' - __meta_kubernetes_pod_container_port_name', 662 | ' - action: replace', 663 | ' regex: true;(.+)', 664 | ' separator: ;', 665 | ' source_labels:', 666 | ' - __meta_kubernetes_pod_annotation_prometheus_io_filterport', 667 | ' - __meta_kubernetes_pod_container_port_number', 668 | ' target_label: __meta_kubernetes_pod_annotation_prometheus_io_port', 669 | ' - action: replace', 670 | ' regex: true;(.+)', 671 | ' separator: ;', 672 | ' source_labels:', 673 | ' - __meta_kubernetes_pod_annotation_prometheus_io_filterport', 674 | ' - __meta_kubernetes_pod_container_name', 675 | ' target_label: kubernetes_container_name', 676 | ' - action: replace', 677 | ' regex: (.+)', 678 | ' source_labels:', 679 | ' - __meta_kubernetes_pod_annotation_prometheus_io_path', 680 | ' target_label: __metrics_path__', 681 | ' - action: replace', 682 | ' regex: ([^:]+)(?::\\d+)?;(\\d+)', 683 | ' replacement: $1:$2', 684 | ' separator: ;', 685 | ' source_labels:', 686 | ' - __address__', 687 | ' - __meta_kubernetes_pod_annotation_prometheus_io_port', 688 | ' target_label: __address__', 689 | ' - action: replace', 690 | ' regex: (.+);(.+);(.+);(.+)', 691 | ' replacement: /api/v1/namespaces/$1/pods/$2:$3/proxy$4', 692 | ' separator: ;', 693 | ' source_labels:', 694 | ' - __meta_kubernetes_namespace', 695 | ' - __meta_kubernetes_pod_name', 696 | ' - __meta_kubernetes_pod_annotation_prometheus_io_port', 697 | ' - __metrics_path__', 698 | ' target_label: __metrics_path__', 699 | ' - action: replace', 700 | ' source_labels:', 701 | ' - __address__', 702 | ' target_label: instance', 703 | ' - action: replace', 704 | ' replacement: api.internal.production.example.com:443', 705 | ' target_label: __address__', 706 | ' - action: labelmap', 707 | ' regex: __meta_kubernetes_pod_label_(.+)', 708 | ' - action: replace', 709 | ' source_labels:', 710 | ' - __meta_kubernetes_namespace', 711 | ' target_label: kubernetes_namespace', 712 | ' - action: replace', 713 | ' source_labels:', 714 | ' - __meta_kubernetes_pod_name', 715 | ' target_label: kubernetes_pod_name', 716 | ' - action: replace', 717 | " replacement: ''", 718 | ' target_label: pod_template_hash', 719 | ' - action: replace', 720 | " replacement: ''", 721 | ' target_label: controller_revision_hash', 722 | ' - action: replace', 723 | " replacement: ''", 724 | ' target_label: pod_template_generation', 725 | ' scheme: https', 726 | ' tls_config:', 727 | ' ca_file: /var/run/kube_secrets/production_ca_crt', 728 | ' - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token', 729 | ' job_name: operations-kubernetes-nodes', 730 | ' kubernetes_sd_configs:', 731 | ' - role: node', 732 | ' metric_relabel_configs:', 733 | ' - action: drop', 734 | ' regex: go_.*', 735 | ' source_labels:', 736 | ' - __name__', 737 | ' relabel_configs:', 738 | ' - action: replace', 739 | " replacement: ''", 740 | ' target_label: __meta_kubernetes_node_label_node_role_kubernetes_io_node', 741 | ' - action: replace', 742 | " replacement: ''", 743 | ' target_label: __meta_kubernetes_node_label_node_role_kubernetes_io_master', 744 | ' - action: labelmap', 745 | ' regex: __meta_kubernetes_node_label_(.+)', 746 | ' - action: replace', 747 | ' source_labels:', 748 | ' - __address__', 749 | ' target_label: instance', 750 | ' - action: replace', 751 | ' replacement: kubernetes.default.svc:443', 752 | ' target_label: __address__', 753 | ' - action: replace', 754 | ' regex: (.+)', 755 | ' replacement: /api/v1/nodes/${1}:10255/proxy/metrics', 756 | ' source_labels:', 757 | ' - __meta_kubernetes_node_name', 758 | ' target_label: __metrics_path__', 759 | ' scheme: https', 760 | ' tls_config:', 761 | ' ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt', 762 | ' - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token', 763 | ' job_name: operations-kubernetes-ingresses', 764 | ' kubernetes_sd_configs:', 765 | ' - role: ingress', 766 | ' metric_relabel_configs: []', 767 | ' metrics_path: /api/v1/namespaces/monitoring/services/blackbox-exporter/proxy/probe', 768 | ' params:', 769 | ' module:', 770 | ' - http_2xx', 771 | ' relabel_configs:', 772 | ' - action: keep', 773 | " regex: 'true'", 774 | ' source_labels:', 775 | ' - __meta_kubernetes_ingress_annotation_prometheus_io_probe', 776 | ' - action: replace', 777 | ' source_labels:', 778 | ' - __meta_kubernetes_ingress_annotation_prometheus_io_module', 779 | ' target_label: __param_module', 780 | ' - action: replace', 781 | ' regex: (.+);(.+)', 782 | ' replacement: $1$2', 783 | ' separator: ;', 784 | ' source_labels:', 785 | ' - __address__', 786 | ' - __meta_kubernetes_ingress_path', 787 | ' target_label: __address__', 788 | ' - action: replace', 789 | ' source_labels:', 790 | ' - __address__', 791 | ' target_label: __param_target', 792 | ' - action: replace', 793 | ' source_labels:', 794 | ' - __address__', 795 | ' target_label: instance', 796 | ' - action: replace', 797 | ' replacement: kubernetes.default.svc:443', 798 | ' target_label: __address__', 799 | ' - action: labelmap', 800 | ' regex: __meta_kubernetes_ingress_label_(.+)', 801 | ' - action: replace', 802 | ' source_labels:', 803 | ' - __meta_kubernetes_namespace', 804 | ' target_label: kubernetes_namespace', 805 | ' - action: replace', 806 | ' source_labels:', 807 | ' - __meta_kubernetes_ingress_name', 808 | ' target_label: kubernetes_ingress_name', 809 | ' scheme: https', 810 | ' tls_config:', 811 | ' ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt', 812 | ' - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token', 813 | ' job_name: operations-kubernetes-cadvisor', 814 | ' kubernetes_sd_configs:', 815 | ' - role: node', 816 | ' metric_relabel_configs:', 817 | ' - action: replace', 818 | ' regex: ^/machine\\.slice/machine-rkt\\\\x2d([^\\\\]+)\\\\.+/([^/]+)\\.service$', 819 | ' replacement: ${2}-${1}', 820 | ' source_labels:', 821 | ' - id', 822 | ' target_label: rkt_container_name', 823 | ' - action: replace', 824 | ' regex: ^/system\\.slice/(.+)\\.service$', 825 | ' replacement: ${1}', 826 | ' source_labels:', 827 | ' - id', 828 | ' target_label: systemd_service_name', 829 | ' - action: drop', 830 | ' regex: go_.*', 831 | ' source_labels:', 832 | ' - __name__', 833 | ' relabel_configs:', 834 | ' - action: replace', 835 | " replacement: ''", 836 | ' target_label: __meta_kubernetes_node_label_node_role_kubernetes_io_node', 837 | ' - action: replace', 838 | " replacement: ''", 839 | ' target_label: __meta_kubernetes_node_label_node_role_kubernetes_io_master', 840 | ' - action: labelmap', 841 | ' regex: __meta_kubernetes_node_label_(.+)', 842 | ' - action: replace', 843 | ' source_labels:', 844 | ' - __address__', 845 | ' target_label: instance', 846 | ' - action: replace', 847 | ' replacement: kubernetes.default.svc:443', 848 | ' target_label: __address__', 849 | ' - action: replace', 850 | ' regex: (.+)', 851 | ' replacement: /api/v1/nodes/${1}:10255/proxy/metrics/cadvisor', 852 | ' source_labels:', 853 | ' - __meta_kubernetes_node_name', 854 | ' target_label: __metrics_path__', 855 | ' scheme: https', 856 | ' tls_config:', 857 | ' ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt', 858 | ' - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token', 859 | ' job_name: operations-kubernetes-service-endpoints', 860 | ' kubernetes_sd_configs:', 861 | ' - role: endpoints', 862 | ' metric_relabel_configs: []', 863 | ' relabel_configs:', 864 | ' - action: keep', 865 | " regex: 'true'", 866 | ' source_labels:', 867 | ' - __meta_kubernetes_service_annotation_prometheus_io_scrape', 868 | ' - action: replace', 869 | ' regex: (.+)', 870 | ' source_labels:', 871 | ' - __meta_kubernetes_service_annotation_prometheus_io_path', 872 | ' target_label: __metrics_path__', 873 | ' - action: replace', 874 | ' regex: ([^:]+)(?::\\d+)?;(\\d+)', 875 | ' replacement: $1:$2', 876 | ' source_labels:', 877 | ' - __address__', 878 | ' - __meta_kubernetes_service_annotation_prometheus_io_port', 879 | ' target_label: __address__', 880 | ' - action: replace', 881 | ' regex: (.+);(.+);(.+);(.+)', 882 | ' replacement: /api/v1/namespaces/$1/pods/$2:$3/proxy$4', 883 | ' separator: ;', 884 | ' source_labels:', 885 | ' - __meta_kubernetes_namespace', 886 | ' - __meta_kubernetes_pod_name', 887 | ' - __meta_kubernetes_service_annotation_prometheus_io_port', 888 | ' - __metrics_path__', 889 | ' target_label: __metrics_path__', 890 | ' - action: replace', 891 | ' source_labels:', 892 | ' - __address__', 893 | ' target_label: instance', 894 | ' - action: replace', 895 | ' replacement: kubernetes.default.svc:443', 896 | ' target_label: __address__', 897 | ' - action: labelmap', 898 | ' regex: __meta_kubernetes_service_label_(.+)', 899 | ' - action: replace', 900 | ' source_labels:', 901 | ' - __meta_kubernetes_namespace', 902 | ' target_label: kubernetes_namespace', 903 | ' - action: replace', 904 | ' source_labels:', 905 | ' - __meta_kubernetes_service_name', 906 | ' target_label: kubernetes_service_name', 907 | ' scheme: https', 908 | ' tls_config:', 909 | ' ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt', 910 | ' - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token', 911 | ' job_name: operations-kubernetes-services', 912 | ' kubernetes_sd_configs:', 913 | ' - role: service', 914 | ' metric_relabel_configs: []', 915 | ' metrics_path: /api/v1/namespaces/monitoring/services/blackbox-exporter/proxy/probe', 916 | ' params:', 917 | ' module:', 918 | ' - http_2xx', 919 | ' relabel_configs:', 920 | ' - action: keep', 921 | " regex: 'true'", 922 | ' source_labels:', 923 | ' - __meta_kubernetes_service_annotation_prometheus_io_probe', 924 | ' - action: replace', 925 | ' source_labels:', 926 | ' - __meta_kubernetes_service_annotation_prometheus_io_module', 927 | ' target_label: __param_module', 928 | ' - action: replace', 929 | ' source_labels:', 930 | ' - __address__', 931 | ' target_label: __param_target', 932 | ' - action: replace', 933 | ' regex: (.+);(.+)', 934 | ' replacement: $1$2', 935 | ' separator: ;', 936 | ' source_labels:', 937 | ' - __param_target', 938 | ' - __meta_kubernetes_service_annotation_prometheus_io_path', 939 | ' target_label: __param_target', 940 | ' - action: replace', 941 | ' source_labels:', 942 | ' - __address__', 943 | ' target_label: instance', 944 | ' - action: replace', 945 | ' replacement: kubernetes.default.svc:443', 946 | ' target_label: __address__', 947 | ' - action: labelmap', 948 | ' regex: __meta_kubernetes_service_label_(.+)', 949 | ' - action: replace', 950 | ' source_labels:', 951 | ' - __meta_kubernetes_namespace', 952 | ' target_label: kubernetes_namespace', 953 | ' - action: replace', 954 | ' source_labels:', 955 | ' - __meta_kubernetes_service_name', 956 | ' target_label: kubernetes_service_name', 957 | ' scheme: https', 958 | ' tls_config:', 959 | ' ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt', 960 | ' - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token', 961 | ' job_name: operations-kubernetes-pods-default', 962 | ' kubernetes_sd_configs:', 963 | ' - role: pod', 964 | ' metric_relabel_configs:', 965 | ' - action: drop', 966 | ' regex: go_.*', 967 | ' source_labels:', 968 | ' - __name__', 969 | ' relabel_configs:', 970 | ' - action: keep', 971 | " regex: 'true'", 972 | ' source_labels:', 973 | ' - __meta_kubernetes_pod_annotation_prometheus_io_scrape', 974 | ' - action: drop', 975 | ' regex: .+', 976 | ' source_labels:', 977 | ' - __meta_kubernetes_pod_annotation_prometheus_io_interval', 978 | ' - action: replace', 979 | ' regex: (.+)', 980 | " replacement: 'false'", 981 | ' source_labels:', 982 | ' - __meta_kubernetes_pod_annotation_prometheus_io_port', 983 | ' target_label: __meta_kubernetes_pod_annotation_prometheus_io_filterport', 984 | ' - action: keep', 985 | ' regex: (false;.*)|(true;.*metrics)', 986 | ' separator: ;', 987 | ' source_labels:', 988 | ' - __meta_kubernetes_pod_annotation_prometheus_io_filterport', 989 | ' - __meta_kubernetes_pod_container_port_name', 990 | ' - action: replace', 991 | ' regex: true;(.+)', 992 | ' separator: ;', 993 | ' source_labels:', 994 | ' - __meta_kubernetes_pod_annotation_prometheus_io_filterport', 995 | ' - __meta_kubernetes_pod_container_port_number', 996 | ' target_label: __meta_kubernetes_pod_annotation_prometheus_io_port', 997 | ' - action: replace', 998 | ' regex: true;(.+)', 999 | ' separator: ;', 1000 | ' source_labels:', 1001 | ' - __meta_kubernetes_pod_annotation_prometheus_io_filterport', 1002 | ' - __meta_kubernetes_pod_container_name', 1003 | ' target_label: kubernetes_container_name', 1004 | ' - action: replace', 1005 | ' regex: (.+)', 1006 | ' source_labels:', 1007 | ' - __meta_kubernetes_pod_annotation_prometheus_io_path', 1008 | ' target_label: __metrics_path__', 1009 | ' - action: replace', 1010 | ' regex: ([^:]+)(?::\\d+)?;(\\d+)', 1011 | ' replacement: $1:$2', 1012 | ' separator: ;', 1013 | ' source_labels:', 1014 | ' - __address__', 1015 | ' - __meta_kubernetes_pod_annotation_prometheus_io_port', 1016 | ' target_label: __address__', 1017 | ' - action: replace', 1018 | ' regex: (.+);(.+);(.+);(.+)', 1019 | ' replacement: /api/v1/namespaces/$1/pods/$2:$3/proxy$4', 1020 | ' separator: ;', 1021 | ' source_labels:', 1022 | ' - __meta_kubernetes_namespace', 1023 | ' - __meta_kubernetes_pod_name', 1024 | ' - __meta_kubernetes_pod_annotation_prometheus_io_port', 1025 | ' - __metrics_path__', 1026 | ' target_label: __metrics_path__', 1027 | ' - action: replace', 1028 | ' source_labels:', 1029 | ' - __address__', 1030 | ' target_label: instance', 1031 | ' - action: replace', 1032 | ' replacement: kubernetes.default.svc:443', 1033 | ' target_label: __address__', 1034 | ' - action: labelmap', 1035 | ' regex: __meta_kubernetes_pod_label_(.+)', 1036 | ' - action: replace', 1037 | ' source_labels:', 1038 | ' - __meta_kubernetes_namespace', 1039 | ' target_label: kubernetes_namespace', 1040 | ' - action: replace', 1041 | ' source_labels:', 1042 | ' - __meta_kubernetes_pod_name', 1043 | ' target_label: kubernetes_pod_name', 1044 | ' - action: replace', 1045 | " replacement: ''", 1046 | ' target_label: pod_template_hash', 1047 | ' - action: replace', 1048 | " replacement: ''", 1049 | ' target_label: controller_revision_hash', 1050 | ' - action: replace', 1051 | " replacement: ''", 1052 | ' target_label: pod_template_generation', 1053 | ' scheme: https', 1054 | ' tls_config:', 1055 | ' ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt', 1056 | ' - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token', 1057 | ' job_name: operations-kubernetes-pods-long', 1058 | ' kubernetes_sd_configs:', 1059 | ' - role: pod', 1060 | ' metric_relabel_configs:', 1061 | ' - action: drop', 1062 | ' regex: go_.*', 1063 | ' source_labels:', 1064 | ' - __name__', 1065 | ' relabel_configs:', 1066 | ' - action: keep', 1067 | " regex: 'true'", 1068 | ' source_labels:', 1069 | ' - __meta_kubernetes_pod_annotation_prometheus_io_scrape', 1070 | ' - action: keep', 1071 | ' regex: long', 1072 | ' source_labels:', 1073 | ' - __meta_kubernetes_pod_annotation_prometheus_io_interval', 1074 | ' - action: replace', 1075 | ' regex: (.+)', 1076 | " replacement: 'false'", 1077 | ' source_labels:', 1078 | ' - __meta_kubernetes_pod_annotation_prometheus_io_port', 1079 | ' target_label: __meta_kubernetes_pod_annotation_prometheus_io_filterport', 1080 | ' - action: keep', 1081 | ' regex: (false;.*)|(true;.*metrics)', 1082 | ' separator: ;', 1083 | ' source_labels:', 1084 | ' - __meta_kubernetes_pod_annotation_prometheus_io_filterport', 1085 | ' - __meta_kubernetes_pod_container_port_name', 1086 | ' - action: replace', 1087 | ' regex: true;(.+)', 1088 | ' separator: ;', 1089 | ' source_labels:', 1090 | ' - __meta_kubernetes_pod_annotation_prometheus_io_filterport', 1091 | ' - __meta_kubernetes_pod_container_port_number', 1092 | ' target_label: __meta_kubernetes_pod_annotation_prometheus_io_port', 1093 | ' - action: replace', 1094 | ' regex: true;(.+)', 1095 | ' separator: ;', 1096 | ' source_labels:', 1097 | ' - __meta_kubernetes_pod_annotation_prometheus_io_filterport', 1098 | ' - __meta_kubernetes_pod_container_name', 1099 | ' target_label: kubernetes_container_name', 1100 | ' - action: replace', 1101 | ' regex: (.+)', 1102 | ' source_labels:', 1103 | ' - __meta_kubernetes_pod_annotation_prometheus_io_path', 1104 | ' target_label: __metrics_path__', 1105 | ' - action: replace', 1106 | ' regex: ([^:]+)(?::\\d+)?;(\\d+)', 1107 | ' replacement: $1:$2', 1108 | ' separator: ;', 1109 | ' source_labels:', 1110 | ' - __address__', 1111 | ' - __meta_kubernetes_pod_annotation_prometheus_io_port', 1112 | ' target_label: __address__', 1113 | ' - action: replace', 1114 | ' regex: (.+);(.+);(.+);(.+)', 1115 | ' replacement: /api/v1/namespaces/$1/pods/$2:$3/proxy$4', 1116 | ' separator: ;', 1117 | ' source_labels:', 1118 | ' - __meta_kubernetes_namespace', 1119 | ' - __meta_kubernetes_pod_name', 1120 | ' - __meta_kubernetes_pod_annotation_prometheus_io_port', 1121 | ' - __metrics_path__', 1122 | ' target_label: __metrics_path__', 1123 | ' - action: replace', 1124 | ' source_labels:', 1125 | ' - __address__', 1126 | ' target_label: instance', 1127 | ' - action: replace', 1128 | ' replacement: kubernetes.default.svc:443', 1129 | ' target_label: __address__', 1130 | ' - action: labelmap', 1131 | ' regex: __meta_kubernetes_pod_label_(.+)', 1132 | ' - action: replace', 1133 | ' source_labels:', 1134 | ' - __meta_kubernetes_namespace', 1135 | ' target_label: kubernetes_namespace', 1136 | ' - action: replace', 1137 | ' source_labels:', 1138 | ' - __meta_kubernetes_pod_name', 1139 | ' target_label: kubernetes_pod_name', 1140 | ' - action: replace', 1141 | " replacement: ''", 1142 | ' target_label: pod_template_hash', 1143 | ' - action: replace', 1144 | " replacement: ''", 1145 | ' target_label: controller_revision_hash', 1146 | ' - action: replace', 1147 | " replacement: ''", 1148 | ' target_label: pod_template_generation', 1149 | ' scheme: https', 1150 | ' scrape_interval: 1h', 1151 | ' tls_config:', 1152 | ' ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt', 1153 | '' 1154 | ] 1155 | --------------------------------------------------------------------------------