├── .github └── workflows │ ├── build.yml │ └── release.yml ├── .gitignore ├── .goreleaser.yaml ├── LICENSE ├── README.md ├── errors.go ├── go.mod ├── go.sum ├── helpers.go ├── main.go ├── models.go ├── vip_test.go └── webserver.go /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | push: 5 | branches: ["master"] 6 | pull_request: 7 | branches: ["master"] 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v3 17 | 18 | - name: Set up Go 19 | uses: actions/setup-go@v4 20 | with: 21 | go-version: 1.22 22 | cache: true 23 | 24 | - name: Build 25 | run: go build -v ./... 26 | 27 | - name: Test 28 | run: go test -v ./... 29 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Releaser workflow setup 3 | # https://goreleaser.com/ci/actions/ 4 | # 5 | name: release 6 | 7 | # run only on tags 8 | on: 9 | push: 10 | tags: 11 | - "v*" 12 | 13 | permissions: 14 | contents: write # needed to write releases 15 | id-token: write # needed for keyless signing 16 | packages: write # needed for ghcr access 17 | 18 | jobs: 19 | release: 20 | runs-on: ubuntu-latest 21 | steps: 22 | - uses: actions/checkout@v4 23 | with: 24 | fetch-depth: 0 # this is important, otherwise it won't checkout the full tree (i.e. no previous tags) 25 | - uses: actions/setup-go@v4 26 | with: 27 | go-version: 1.22 28 | cache: true 29 | - uses: sigstore/cosign-installer@v3.1.2 # installs cosign 30 | - uses: anchore/sbom-action/download-syft@v0.14.3 # installs syft 31 | - uses: goreleaser/goreleaser-action@v5 # run goreleaser 32 | with: 33 | version: latest 34 | args: release --clean 35 | env: 36 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | project_name: prometheus-net-discovery 2 | 3 | # setups builds for linux and darwin on amd64 and arm64 4 | # https://goreleaser.com/customization/build 5 | builds: 6 | - env: 7 | - CGO_ENABLED=0 8 | goos: 9 | - linux 10 | goarch: 11 | - amd64 12 | # ensures mod timestamp to be the commit timestamp 13 | mod_timestamp: "{{ .CommitTimestamp }}" 14 | flags: 15 | # trims path 16 | - -trimpath 17 | ldflags: 18 | # use commit date instead of current date as main.date 19 | # only needed if you actually use those things in your main package, otherwise can be ignored. 20 | - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{ .CommitDate }} 21 | 22 | # proxies from the go mod proxy before building 23 | # https://goreleaser.com/customization/gomod 24 | gomod: 25 | proxy: true 26 | 27 | # config the checksum filename 28 | # https://goreleaser.com/customization/checksum 29 | checksum: 30 | name_template: "checksums.txt" 31 | 32 | # create a source tarball 33 | # https://goreleaser.com/customization/source/ 34 | source: 35 | enabled: true 36 | 37 | # creates SBOMs of all archives and the source tarball using syft 38 | # https://goreleaser.com/customization/sbom 39 | sboms: 40 | - artifacts: archive 41 | - id: source # Two different sbom configurations need two different IDs 42 | artifacts: source 43 | 44 | # signs the checksum file 45 | # all files (including the sboms) are included in the checksum, so we don't need to sign each one if we don't want to 46 | # https://goreleaser.com/customization/sign 47 | signs: 48 | - cmd: cosign 49 | env: 50 | - COSIGN_EXPERIMENTAL=1 51 | certificate: "${artifact}.pem" 52 | args: 53 | - sign-blob 54 | - "--output-certificate=${certificate}" 55 | - "--output-signature=${signature}" 56 | - "${artifact}" 57 | - "--yes" # needed on cosign 2.0.0+ 58 | artifacts: checksum 59 | output: true 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # prometheus-net-discovery 2 | Scan entire networks for open ports with known prometheus exporters and generate files with targets to be used with `file_sd_config` file based discovery 3 | 4 | ### install 5 | 6 | ``` 7 | go get -u github.com/fortnoxab/prometheus-net-discovery 8 | ``` 9 | 10 | ### example run 11 | 12 | ``` 13 | prometheus-net-discovery -networks "192.168.1.0/24" --filesdpath /tmp/ 14 | ``` 15 | 16 | ### usage 17 | 18 | ``` 19 | $ prometheus-net-discovery --help 20 | Usage of prometheus-net-discovery: 21 | -filesdpath 22 | Change value of FileSdPath. 23 | -interval 24 | Change value of Interval. (default 60m) 25 | -log-format 26 | Change value of Log-Format. (default text) 27 | -log-formatter 28 | Change value of Log-Formatter. (default ) 29 | -log-level 30 | Change value of Log-Level. 31 | -networks 32 | Change value of Networks. 33 | 34 | Generated environment variables: 35 | CONFIG_FILESDPATH 36 | CONFIG_INTERVAL 37 | CONFIG_LOG_FORMAT 38 | CONFIG_LOG_FORMATTER 39 | CONFIG_LOG_LEVEL 40 | CONFIG_NETWORKS 41 | 42 | ``` 43 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | type timeout interface { 8 | Timeout() bool 9 | } 10 | 11 | func IsTimeout(err error) bool { 12 | var t timeout 13 | if errors.As(err, &t) && t.Timeout() { 14 | return true 15 | } 16 | return false 17 | } 18 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/fortnoxab/prometheus-net-discovery/v2 2 | 3 | go 1.22 4 | 5 | require ( 6 | github.com/fortnoxab/fnxlogrus v0.0.0-20220823093317-6a580c56b8fd 7 | github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7 8 | github.com/prometheus/client_golang v1.20.1 9 | github.com/sirupsen/logrus v1.9.3 10 | github.com/stretchr/testify v1.9.0 11 | ) 12 | 13 | require ( 14 | github.com/BurntSushi/toml v1.4.0 // indirect 15 | github.com/beorn7/perks v1.0.1 // indirect 16 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 17 | github.com/davecgh/go-spew v1.1.1 // indirect 18 | github.com/fatih/camelcase v1.0.0 // indirect 19 | github.com/fatih/structs v1.1.0 // indirect 20 | github.com/klauspost/compress v1.17.9 // indirect 21 | github.com/kr/text v0.2.0 // indirect 22 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 23 | github.com/pkg/errors v0.9.1 // indirect 24 | github.com/pmezard/go-difflib v1.0.0 // indirect 25 | github.com/prometheus/client_model v0.6.1 // indirect 26 | github.com/prometheus/common v0.55.0 // indirect 27 | github.com/prometheus/procfs v0.15.1 // indirect 28 | golang.org/x/sys v0.24.0 // indirect 29 | google.golang.org/protobuf v1.34.2 // indirect 30 | gopkg.in/yaml.v2 v2.4.0 // indirect 31 | gopkg.in/yaml.v3 v3.0.1 // indirect 32 | ) 33 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= 2 | github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= 3 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 4 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 5 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 6 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 7 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 10 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= 12 | github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= 13 | github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= 14 | github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= 15 | github.com/fortnoxab/fnxlogrus v0.0.0-20220823093317-6a580c56b8fd h1:vjKVmkcuAhOGIJnbCGfm1MvKvJgE7jpIZTHS6pqdV+s= 16 | github.com/fortnoxab/fnxlogrus v0.0.0-20220823093317-6a580c56b8fd/go.mod h1:R6mab3Zy+NZ3qi4krgbsMSPFVh2AFVSBG2xnus0fl0Q= 17 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 18 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 19 | github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= 20 | github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= 21 | github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7 h1:SWlt7BoQNASbhTUD0Oy5yysI2seJ7vWuGUp///OM4TM= 22 | github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7/go.mod h1:Y2SaZf2Rzd0pXkLVhLlCiAXFCLSXAIbTKDivVgff/AM= 23 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 24 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 25 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 26 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 27 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 28 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 29 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 30 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 31 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 32 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 33 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 34 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 35 | github.com/prometheus/client_golang v1.20.1 h1:IMJXHOD6eARkQpxo8KkhgEVFlBNm+nkrFUyGlIu7Na8= 36 | github.com/prometheus/client_golang v1.20.1/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= 37 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 38 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 39 | github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= 40 | github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= 41 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 42 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 43 | github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= 44 | github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= 45 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 46 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 47 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 48 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 49 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 50 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 51 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 52 | golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= 53 | golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 54 | google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= 55 | google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= 56 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 57 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 58 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 59 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 60 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 61 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 62 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 63 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 64 | -------------------------------------------------------------------------------- /helpers.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "context" 7 | "encoding/json" 8 | "fmt" 9 | "net" 10 | "net/http" 11 | "time" 12 | 13 | "github.com/sirupsen/logrus" 14 | ) 15 | 16 | func inc(ip net.IP) { 17 | for j := len(ip) - 1; j >= 0; j-- { 18 | ip[j]++ 19 | if ip[j] > 0 { 20 | break 21 | } 22 | } 23 | } 24 | 25 | func getFirst(s []string) string { 26 | for _, v := range s { 27 | return v 28 | } 29 | return "" 30 | } 31 | 32 | var dialer = &net.Dialer{ 33 | Timeout: 3 * time.Second, 34 | KeepAlive: 30 * time.Second, 35 | } 36 | var client = &http.Client{ 37 | Transport: &http.Transport{ 38 | // TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // #nosec only cockroach at the moment 39 | DialContext: dialer.DialContext, 40 | ForceAttemptHTTP2: true, 41 | MaxIdleConns: 100, 42 | IdleConnTimeout: 10 * time.Second, 43 | TLSHandshakeTimeout: 10 * time.Second, 44 | ExpectContinueTimeout: 1 * time.Second, 45 | }, 46 | } 47 | 48 | func checkExporterExporter(parentCtx context.Context, host, port string) ([]string, error) { 49 | u := fmt.Sprintf("http://%s", net.JoinHostPort(host, port)) 50 | ctx, cancel := context.WithTimeout(parentCtx, time.Second*3) 51 | defer cancel() 52 | 53 | req, err := http.NewRequestWithContext(ctx, "GET", u, nil) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | req.Header.Set("Accept", "application/json") 59 | resp, err := client.Do(req) 60 | if err != nil { 61 | return nil, fmt.Errorf("error http get %s: %w", u, err) 62 | } 63 | 64 | defer resp.Body.Close() 65 | var anyJSON map[string]interface{} 66 | err = json.NewDecoder(resp.Body).Decode(&anyJSON) 67 | if err != nil { 68 | return nil, fmt.Errorf("error decoding json body: %w", err) 69 | } 70 | 71 | exporters := make([]string, len(anyJSON)) 72 | i := 0 73 | for k := range anyJSON { 74 | exporters[i] = k 75 | i++ 76 | } 77 | 78 | return exporters, nil 79 | } 80 | 81 | func alive(parentCtx context.Context, host, port, path string) bool { 82 | if path != "" { 83 | u := fmt.Sprintf(path, net.JoinHostPort(host, port)) 84 | 85 | ctx, cancel := context.WithTimeout(parentCtx, time.Second*3) 86 | defer cancel() 87 | req, err := http.NewRequestWithContext(ctx, "GET", u, nil) 88 | if err != nil { 89 | logrus.Errorf("error creating request: %s", err) 90 | return false 91 | } 92 | resp, err := client.Do(req) 93 | if err != nil { 94 | return false 95 | } 96 | defer resp.Body.Close() 97 | 98 | r := bufio.NewReader(resp.Body) 99 | for i := 0; i < 10; i++ { 100 | line, _, err := r.ReadLine() 101 | if err != nil { 102 | return false 103 | } 104 | if bytes.Contains(line, []byte("# TYPE")) { 105 | return true 106 | } 107 | } 108 | return false 109 | } 110 | 111 | conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, port), 200*time.Millisecond) 112 | if err != nil { 113 | return false 114 | } 115 | 116 | if conn != nil { 117 | conn.Close() 118 | return true 119 | } 120 | return false 121 | } 122 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "io" 8 | "log" 9 | "net" 10 | "os" 11 | "os/signal" 12 | "path/filepath" 13 | "regexp" 14 | "strings" 15 | "sync" 16 | "syscall" 17 | "time" 18 | 19 | "github.com/fortnoxab/fnxlogrus" 20 | "github.com/koding/multiconfig" 21 | "github.com/sirupsen/logrus" 22 | ) 23 | 24 | // ExporterConfig configures ports to scan to what filename to save it to. 25 | // if path is set we will try to make a HTTP get and find # TYPE in the first 10 rows of the response to make sure we know its prometheus metrics. 26 | type ExporterConfig []struct { 27 | port string 28 | filename string 29 | path string 30 | } 31 | 32 | var exporterConfig = ExporterConfig{ 33 | // We only support exporter_exporter at the moment 34 | { 35 | port: "", 36 | filename: "", 37 | }, 38 | } 39 | 40 | var mutex sync.Mutex 41 | 42 | func main() { 43 | config := &Config{} 44 | multiconfig.MustLoad(&config) 45 | exporterConfig[0].port = config.ExpoterExporterPort 46 | 47 | fnxlogrus.Init(config.Log, logrus.StandardLogger()) 48 | 49 | networks := strings.Split(config.Networks, ",") 50 | 51 | interval, err := time.ParseDuration(config.Interval) 52 | if err != nil { 53 | logrus.Errorf("invalid duration %s: %s", config.Interval, err.Error()) 54 | return 55 | } 56 | 57 | logrus.Infof("Running with interval %s", interval) 58 | 59 | ticker := time.NewTicker(interval) 60 | ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGQUIT, syscall.SIGTERM) 61 | defer cancel() 62 | 63 | go func() { 64 | startWs(config, ctx) 65 | ticker.Stop() 66 | log.Println("Shutting down") 67 | }() 68 | 69 | runDiscovery(ctx, config, networks) 70 | for { 71 | select { 72 | case <-ticker.C: 73 | runDiscovery(ctx, config, networks) 74 | case <-ctx.Done(): 75 | return 76 | } 77 | } 78 | } 79 | 80 | func runDiscovery(parentCtx context.Context, config *Config, networks []string) { 81 | logrus.Info("Running discovery") 82 | 83 | job := make(chan func(context.Context)) 84 | exporter := make(chan *Address) 85 | ctx, cancel := context.WithCancel(parentCtx) 86 | defer cancel() 87 | var wg sync.WaitGroup 88 | for i := 0; i < 128; i++ { 89 | wg.Add(1) 90 | i := i 91 | go func() { 92 | defer wg.Done() 93 | for { 94 | select { 95 | case fn, ok := <-job: 96 | if !ok { 97 | logrus.Debugf("worker %d finished", i) 98 | return 99 | } 100 | fn(ctx) 101 | case <-ctx.Done(): 102 | return 103 | } 104 | } 105 | }() 106 | } 107 | 108 | go func() { 109 | for _, v := range networks { 110 | if ctx.Err() != nil { 111 | return 112 | } 113 | network := strings.TrimSpace(v) 114 | if network == "" { 115 | continue 116 | } 117 | discoverNetwork(network, job, exporter) 118 | } 119 | close(job) 120 | }() 121 | 122 | exporters := make(Exporters) 123 | 124 | go func() { 125 | for { 126 | select { 127 | case address, ok := <-exporter: 128 | if !ok { 129 | return 130 | } 131 | exporters[address.Exporter] = append(exporters[address.Exporter], *address) 132 | case <-ctx.Done(): 133 | return 134 | } 135 | } 136 | }() 137 | 138 | wg.Wait() 139 | 140 | saveConfigs(ctx, config, exporters) 141 | logrus.Info("discovery done") 142 | } 143 | 144 | func saveConfigs(ctx context.Context, config *Config, exporters Exporters) { 145 | mutex.Lock() 146 | defer mutex.Unlock() 147 | for name, addresses := range exporters { 148 | if ctx.Err() != nil { 149 | return 150 | } 151 | err := writeFileSDConfig(config, name, addresses) 152 | if err != nil { 153 | logrus.Error(err) 154 | continue 155 | } 156 | } 157 | } 158 | 159 | var vipRegexp = regexp.MustCompile(`^.+-vip(\d+)?\.`) 160 | 161 | func isVip(name string) bool { 162 | return vipRegexp.MatchString(name) 163 | } 164 | 165 | func discoverNetwork(network string, queue chan func(context.Context), exporter chan *Address) { 166 | networkip, ipnet, err := net.ParseCIDR(network) 167 | if err != nil { 168 | log.Fatal("network CIDR could not be parsed:", err) 169 | } 170 | for ip := networkip.Mask(ipnet.Mask); ipnet.Contains(ip); inc(ip) { 171 | network := network 172 | ip := ip.String() 173 | queue <- func(ctx context.Context) { 174 | for _, data := range exporterConfig { 175 | if ctx.Err() != nil { 176 | return 177 | } 178 | port := data.port 179 | logrus.Debugf("scanning port: %s:%s", ip, port) 180 | var exporters []string 181 | exporters, err = checkExporterExporter(ctx, ip, port) 182 | if err != nil { 183 | if !errors.Is(err, context.DeadlineExceeded) && !IsTimeout(err) { 184 | logrus.Debugf("error fetching from exporter_exporter: %s", err) 185 | } 186 | logrus.Debugf("%s:%s was not open", ip, port) 187 | continue 188 | } 189 | 190 | logrus.Info(net.JoinHostPort(ip, port), " is alive") 191 | addr, _ := net.LookupAddr(ip) // #nosec 192 | hostname := strings.TrimRight(getFirst(addr), ".") 193 | if hostname == "" { 194 | logrus.Error("missing reverse record for ", ip) 195 | continue 196 | } 197 | if isVip(hostname) && !strings.HasPrefix(hostname, "k8s-") { 198 | logrus.Info("skipping vip ", hostname, ip) 199 | continue 200 | } 201 | 202 | if len(exporters) > 0 { 203 | for _, filename := range exporters { 204 | a := Address{ 205 | IP: strings.TrimSpace(ip), 206 | Hostname: strings.TrimSpace(hostname), 207 | Subnet: strings.TrimSpace(network), 208 | Exporter: filename, 209 | Port: port, 210 | } 211 | exporter <- &a 212 | } 213 | } 214 | } 215 | } 216 | } 217 | } 218 | 219 | func getOldGroups(path string) ([]Group, error) { 220 | file, err := os.Open(path) // #nosec 221 | if os.IsNotExist(err) { 222 | return nil, nil // ignore if files not found 223 | } 224 | if err != nil { 225 | return nil, err 226 | } 227 | defer file.Close() 228 | 229 | oldGroups := []Group{} 230 | err = json.NewDecoder(file).Decode(&oldGroups) 231 | if errors.Is(err, io.EOF) { // Ignore empty files 232 | return nil, nil 233 | } 234 | return oldGroups, err 235 | } 236 | 237 | func writeFileSDConfig(config *Config, exporterName string, addresses []Address) error { 238 | path := filepath.Join(config.FileSdPath, exporterName+".json") 239 | 240 | if _, err := os.Stat(config.FileSdPath); os.IsNotExist(err) { 241 | os.MkdirAll(config.FileSdPath, 0755) 242 | } 243 | 244 | groups := []Group{} 245 | 246 | for _, v := range addresses { 247 | group := Group{ 248 | Targets: []string{net.JoinHostPort(v.IP, v.Port)}, 249 | Labels: map[string]string{ 250 | "subnet": v.Subnet, 251 | "host": v.Hostname, 252 | }, 253 | } 254 | if v.Port == config.ExpoterExporterPort { 255 | group.Labels["__metrics_path__"] = "/proxy" 256 | group.Labels["__param_module"] = exporterName 257 | } 258 | groups = append(groups, group) 259 | } 260 | 261 | previous, err := getOldGroups(path) 262 | if err != nil { 263 | return err 264 | } 265 | 266 | // Dont remove targets if they happened to be down at the moment 267 | for _, prev := range previous { 268 | exists := false 269 | for _, current := range groups { 270 | if getFirst(current.Targets) == getFirst(prev.Targets) { 271 | exists = true 272 | } 273 | } 274 | if !exists { 275 | logrus.Errorf("%s (%s) was removed, keeping it anyway. To remove it delete it from %s manually", prev.Targets[0], prev.Labels["host"], path) 276 | groups = append(groups, prev) 277 | } 278 | } 279 | 280 | file, err := os.Create(path) 281 | if err != nil { 282 | return err 283 | } 284 | defer file.Close() 285 | 286 | encoder := json.NewEncoder(file) 287 | encoder.SetIndent("", "\t") 288 | return encoder.Encode(groups) 289 | } 290 | -------------------------------------------------------------------------------- /models.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/fortnoxab/fnxlogrus" 4 | 5 | // Config is main application configuration. 6 | type Config struct { 7 | // Comma separated string of networks in 192.168.0.1/24 format 8 | Networks string 9 | // Interval is how often to scan. Default 60m 10 | Interval string `default:"60m"` 11 | // FileSdPath specifies where to put your generated files. Example /etc/prometheus/file_sd/ 12 | FileSdPath string 13 | Log fnxlogrus.Config 14 | Port string `default:"8080"` 15 | ExpoterExporterPort string `default:"9999"` 16 | } 17 | 18 | // Exporters is a list of addresses grouped by exporter name. 19 | type Exporters map[string][]Address 20 | 21 | // Address represents a host:ip to monitor. 22 | type Address struct { 23 | IP string 24 | Hostname string 25 | Subnet string 26 | Port string 27 | Exporter string 28 | MetricsPath string 29 | } 30 | 31 | // Group is a prometheus target config. Copied struct from prometheus repo. 32 | type Group struct { 33 | Targets []string `json:"targets"` 34 | Labels map[string]string `json:"labels"` 35 | } 36 | -------------------------------------------------------------------------------- /vip_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestIsVipMatch(t *testing.T) { 10 | assert.Equal(t, true, isVip("asdf-internal-vip.asdf.com")) 11 | assert.Equal(t, true, isVip("dev-ssl-vip2.asdf.com")) 12 | assert.Equal(t, false, isVip("dev-ssl-vipp01.asdf.com")) 13 | assert.Equal(t, false, isVip("dev-ssl-vipp.asdf.com")) 14 | } 15 | -------------------------------------------------------------------------------- /webserver.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io/fs" 9 | "net/http" 10 | "os" 11 | "path/filepath" 12 | "slices" 13 | "time" 14 | 15 | "github.com/prometheus/client_golang/prometheus/promhttp" 16 | "github.com/sirupsen/logrus" 17 | ) 18 | 19 | type ErrorHandler func(w http.ResponseWriter, r *http.Request) error 20 | 21 | func errH(f ErrorHandler) http.HandlerFunc { 22 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 23 | err := f(w, r) 24 | if err != nil { 25 | logrus.Error(err) 26 | w.WriteHeader(http.StatusBadRequest) 27 | fmt.Fprintf(w, "error: %s", err) 28 | return 29 | } 30 | }) 31 | } 32 | 33 | func startWs(config *Config, ctx context.Context) { 34 | handler := http.NewServeMux() 35 | 36 | handler.HandleFunc("/api/hosts/{host}", errH(func(w http.ResponseWriter, r *http.Request) error { 37 | if r.Method != http.MethodDelete { 38 | return fmt.Errorf("only DELETE is supported") 39 | } 40 | host := r.PathValue("host") 41 | mutex.Lock() 42 | defer mutex.Unlock() 43 | 44 | files, err := fs.Glob(os.DirFS(config.FileSdPath), "*.json") 45 | if err != nil { 46 | return err 47 | } 48 | 49 | for _, file := range files { 50 | path := filepath.Join(config.FileSdPath, file) 51 | groups, err := getOldGroups(path) 52 | if err != nil { 53 | return err 54 | } 55 | 56 | newGroups := slices.DeleteFunc(groups, func(dp Group) bool { 57 | return dp.Labels["host"] == host 58 | }) 59 | 60 | if len(groups) == len(newGroups) { 61 | continue 62 | } 63 | 64 | file, err := os.Create(path) 65 | if err != nil { 66 | return err 67 | } 68 | defer file.Close() 69 | 70 | encoder := json.NewEncoder(file) 71 | encoder.SetIndent("", "\t") 72 | err = encoder.Encode(newGroups) 73 | if err != nil { 74 | return err 75 | } 76 | } 77 | 78 | return nil 79 | 80 | })) 81 | handler.Handle("/metrics", promhttp.Handler()) 82 | srv := &http.Server{ 83 | ReadTimeout: 10 * time.Second, 84 | WriteTimeout: 10 * time.Second, 85 | IdleTimeout: 30 * time.Second, 86 | ReadHeaderTimeout: 20 * time.Second, 87 | Addr: ":" + config.Port, 88 | Handler: handler, 89 | } 90 | 91 | go func() { 92 | if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { 93 | logrus.Fatalf("error starting webserver %s", err) 94 | } 95 | }() 96 | 97 | logrus.Debug("webserver started") 98 | 99 | <-ctx.Done() 100 | 101 | if os.Getenv("KUBERNETES_SERVICE_HOST") != "" && os.Getenv("KUBERNETES_SERVICE_PORT") != "" { 102 | logrus.Debug("sleeping 5 sec before shutdown") // to give k8s ingresses time to sync 103 | time.Sleep(5 * time.Second) 104 | } 105 | ctxShutDown, cancel := context.WithTimeout(context.Background(), 5*time.Second) 106 | defer cancel() 107 | 108 | if err := srv.Shutdown(ctxShutDown); !errors.Is(err, http.ErrServerClosed) && err != nil { 109 | logrus.Error(err) 110 | } 111 | } 112 | --------------------------------------------------------------------------------