├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── go.mod ├── namenode_exporter.go └── resourcemanager_exporter.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: namenode_exporter resourcemanager_exporter 2 | .PHONY: all 3 | 4 | deps: 5 | go get github.com/prometheus/client_golang/prometheus 6 | go get github.com/prometheus/log 7 | 8 | namenode_exporter: deps namenode_exporter.go 9 | go build namenode_exporter.go 10 | 11 | resourcemanager_exporter: deps resourcemanager_exporter.go 12 | go build resourcemanager_exporter.go 13 | 14 | clean: 15 | rm -rf namenode_exporter resourcemanager_exporter 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Hadoop Exporter for Prometheus 2 | Exports hadoop metrics via HTTP for Prometheus consumption. 3 | 4 | How to build 5 | ``` 6 | go get github.com/prometheus/client_golang/prometheus 7 | go get github.com/prometheus/log 8 | go build namenode_exporter.go 9 | go build resourcemanager_exporter.go 10 | ``` 11 | 12 | Help on flags of namenode_exporter: 13 | ``` 14 | -namenode.jmx.url string 15 | Hadoop JMX URL. (default "http://localhost:50070/jmx") 16 | -web.listen-address string 17 | Address on which to expose metrics and web interface. (default ":9070") 18 | -web.telemetry-path string 19 | Path under which to expose metrics. (default "/metrics") 20 | ``` 21 | 22 | Help on flags of resourcemanager_exporter: 23 | ``` 24 | -resourcemanager.url string 25 | Hadoop ResourceManager URL. (default "http://localhost:8088") 26 | -web.listen-address string 27 | Address on which to expose metrics and web interface. (default ":9088") 28 | -web.telemetry-path string 29 | Path under which to expose metrics. (default "/metrics") 30 | ``` 31 | 32 | Tested on HDP2.3 33 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/wyukawa/hadoop_exporter 2 | 3 | require ( 4 | github.com/Sirupsen/logrus v1.0.6 // indirect 5 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect 6 | github.com/golang/protobuf v1.2.0 // indirect 7 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 8 | github.com/prometheus/client_golang v0.8.0 9 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect 10 | github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e // indirect 11 | github.com/prometheus/log v0.0.0-20151026012452-9a3136781e1f 12 | github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 // indirect 13 | golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b // indirect 14 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e // indirect 15 | ) 16 | -------------------------------------------------------------------------------- /namenode_exporter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "io/ioutil" 7 | "net/http" 8 | 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/prometheus/log" 11 | ) 12 | 13 | const ( 14 | namespace = "namenode" 15 | ) 16 | 17 | var ( 18 | listenAddress = flag.String("web.listen-address", ":9070", "Address on which to expose metrics and web interface.") 19 | metricsPath = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.") 20 | namenodeJmxUrl = flag.String("namenode.jmx.url", "http://localhost:50070/jmx", "Hadoop JMX URL.") 21 | ) 22 | 23 | type Exporter struct { 24 | url string 25 | MissingBlocks prometheus.Gauge 26 | CapacityTotal prometheus.Gauge 27 | CapacityUsed prometheus.Gauge 28 | CapacityRemaining prometheus.Gauge 29 | CapacityUsedNonDFS prometheus.Gauge 30 | BlocksTotal prometheus.Gauge 31 | FilesTotal prometheus.Gauge 32 | CorruptBlocks prometheus.Gauge 33 | ExcessBlocks prometheus.Gauge 34 | StaleDataNodes prometheus.Gauge 35 | pnGcCount prometheus.Gauge 36 | pnGcTime prometheus.Gauge 37 | cmsGcCount prometheus.Gauge 38 | cmsGcTime prometheus.Gauge 39 | heapMemoryUsageCommitted prometheus.Gauge 40 | heapMemoryUsageInit prometheus.Gauge 41 | heapMemoryUsageMax prometheus.Gauge 42 | heapMemoryUsageUsed prometheus.Gauge 43 | isActive prometheus.Gauge 44 | } 45 | 46 | func NewExporter(url string) *Exporter { 47 | return &Exporter{ 48 | url: url, 49 | MissingBlocks: prometheus.NewGauge(prometheus.GaugeOpts{ 50 | Namespace: namespace, 51 | Name: "MissingBlocks", 52 | Help: "MissingBlocks", 53 | }), 54 | CapacityTotal: prometheus.NewGauge(prometheus.GaugeOpts{ 55 | Namespace: namespace, 56 | Name: "CapacityTotal", 57 | Help: "CapacityTotal", 58 | }), 59 | CapacityUsed: prometheus.NewGauge(prometheus.GaugeOpts{ 60 | Namespace: namespace, 61 | Name: "CapacityUsed", 62 | Help: "CapacityUsed", 63 | }), 64 | CapacityRemaining: prometheus.NewGauge(prometheus.GaugeOpts{ 65 | Namespace: namespace, 66 | Name: "CapacityRemaining", 67 | Help: "CapacityRemaining", 68 | }), 69 | CapacityUsedNonDFS: prometheus.NewGauge(prometheus.GaugeOpts{ 70 | Namespace: namespace, 71 | Name: "CapacityUsedNonDFS", 72 | Help: "CapacityUsedNonDFS", 73 | }), 74 | BlocksTotal: prometheus.NewGauge(prometheus.GaugeOpts{ 75 | Namespace: namespace, 76 | Name: "BlocksTotal", 77 | Help: "BlocksTotal", 78 | }), 79 | FilesTotal: prometheus.NewGauge(prometheus.GaugeOpts{ 80 | Namespace: namespace, 81 | Name: "FilesTotal", 82 | Help: "FilesTotal", 83 | }), 84 | CorruptBlocks: prometheus.NewGauge(prometheus.GaugeOpts{ 85 | Namespace: namespace, 86 | Name: "CorruptBlocks", 87 | Help: "CorruptBlocks", 88 | }), 89 | ExcessBlocks: prometheus.NewGauge(prometheus.GaugeOpts{ 90 | Namespace: namespace, 91 | Name: "ExcessBlocks", 92 | Help: "ExcessBlocks", 93 | }), 94 | StaleDataNodes: prometheus.NewGauge(prometheus.GaugeOpts{ 95 | Namespace: namespace, 96 | Name: "StaleDataNodes", 97 | Help: "StaleDataNodes", 98 | }), 99 | pnGcCount: prometheus.NewGauge(prometheus.GaugeOpts{ 100 | Namespace: namespace, 101 | Name: "ParNew_CollectionCount", 102 | Help: "ParNew GC Count", 103 | }), 104 | pnGcTime: prometheus.NewGauge(prometheus.GaugeOpts{ 105 | Namespace: namespace, 106 | Name: "ParNew_CollectionTime", 107 | Help: "ParNew GC Time", 108 | }), 109 | cmsGcCount: prometheus.NewGauge(prometheus.GaugeOpts{ 110 | Namespace: namespace, 111 | Name: "ConcurrentMarkSweep_CollectionCount", 112 | Help: "ConcurrentMarkSweep GC Count", 113 | }), 114 | cmsGcTime: prometheus.NewGauge(prometheus.GaugeOpts{ 115 | Namespace: namespace, 116 | Name: "ConcurrentMarkSweep_CollectionTime", 117 | Help: "ConcurrentMarkSweep GC Time", 118 | }), 119 | heapMemoryUsageCommitted: prometheus.NewGauge(prometheus.GaugeOpts{ 120 | Namespace: namespace, 121 | Name: "heapMemoryUsageCommitted", 122 | Help: "heapMemoryUsageCommitted", 123 | }), 124 | heapMemoryUsageInit: prometheus.NewGauge(prometheus.GaugeOpts{ 125 | Namespace: namespace, 126 | Name: "heapMemoryUsageInit", 127 | Help: "heapMemoryUsageInit", 128 | }), 129 | heapMemoryUsageMax: prometheus.NewGauge(prometheus.GaugeOpts{ 130 | Namespace: namespace, 131 | Name: "heapMemoryUsageMax", 132 | Help: "heapMemoryUsageMax", 133 | }), 134 | heapMemoryUsageUsed: prometheus.NewGauge(prometheus.GaugeOpts{ 135 | Namespace: namespace, 136 | Name: "heapMemoryUsageUsed", 137 | Help: "heapMemoryUsageUsed", 138 | }), 139 | isActive: prometheus.NewGauge(prometheus.GaugeOpts{ 140 | Namespace: namespace, 141 | Name: "isActive", 142 | Help: "isActive", 143 | }), 144 | } 145 | } 146 | 147 | // Describe implements the prometheus.Collector interface. 148 | func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { 149 | e.MissingBlocks.Describe(ch) 150 | e.CapacityTotal.Describe(ch) 151 | e.CapacityUsed.Describe(ch) 152 | e.CapacityRemaining.Describe(ch) 153 | e.CapacityUsedNonDFS.Describe(ch) 154 | e.BlocksTotal.Describe(ch) 155 | e.FilesTotal.Describe(ch) 156 | e.CorruptBlocks.Describe(ch) 157 | e.ExcessBlocks.Describe(ch) 158 | e.StaleDataNodes.Describe(ch) 159 | e.pnGcCount.Describe(ch) 160 | e.pnGcTime.Describe(ch) 161 | e.cmsGcCount.Describe(ch) 162 | e.cmsGcTime.Describe(ch) 163 | e.heapMemoryUsageCommitted.Describe(ch) 164 | e.heapMemoryUsageInit.Describe(ch) 165 | e.heapMemoryUsageMax.Describe(ch) 166 | e.heapMemoryUsageUsed.Describe(ch) 167 | e.isActive.Describe(ch) 168 | } 169 | 170 | // Collect implements the prometheus.Collector interface. 171 | func (e *Exporter) Collect(ch chan<- prometheus.Metric) { 172 | resp, err := http.Get(e.url) 173 | if err != nil { 174 | log.Error(err) 175 | } 176 | defer resp.Body.Close() 177 | data, err := ioutil.ReadAll(resp.Body) 178 | if err != nil { 179 | log.Error(err) 180 | } 181 | var f interface{} 182 | err = json.Unmarshal(data, &f) 183 | if err != nil { 184 | log.Error(err) 185 | } 186 | // {"beans":[{"name":"Hadoop:service=NameNode,name=FSNamesystem", ...}, {"name":"java.lang:type=MemoryPool,name=Code Cache", ...}, ...]} 187 | m := f.(map[string]interface{}) 188 | // [{"name":"Hadoop:service=NameNode,name=FSNamesystem", ...}, {"name":"java.lang:type=MemoryPool,name=Code Cache", ...}, ...] 189 | var nameList = m["beans"].([]interface{}) 190 | for _, nameData := range nameList { 191 | nameDataMap := nameData.(map[string]interface{}) 192 | /* 193 | { 194 | "name" : "Hadoop:service=NameNode,name=FSNamesystem", 195 | "modelerType" : "FSNamesystem", 196 | "tag.Context" : "dfs", 197 | "tag.HAState" : "active", 198 | "tag.TotalSyncTimes" : "23 6 ", 199 | "tag.Hostname" : "CNHORTO7502.line.ism", 200 | "MissingBlocks" : 0, 201 | "MissingReplOneBlocks" : 0, 202 | "ExpiredHeartbeats" : 0, 203 | "TransactionsSinceLastCheckpoint" : 2007, 204 | "TransactionsSinceLastLogRoll" : 7, 205 | "LastWrittenTransactionId" : 172706, 206 | "LastCheckpointTime" : 1456089173101, 207 | "CapacityTotal" : 307099828224, 208 | "CapacityTotalGB" : 286.0, 209 | "CapacityUsed" : 1471291392, 210 | "CapacityUsedGB" : 1.0, 211 | "CapacityRemaining" : 279994568704, 212 | "CapacityRemainingGB" : 261.0, 213 | "CapacityUsedNonDFS" : 25633968128, 214 | "TotalLoad" : 6, 215 | "SnapshottableDirectories" : 0, 216 | "Snapshots" : 0, 217 | "LockQueueLength" : 0, 218 | "BlocksTotal" : 67, 219 | "NumFilesUnderConstruction" : 0, 220 | "NumActiveClients" : 0, 221 | "FilesTotal" : 184, 222 | "PendingReplicationBlocks" : 0, 223 | "UnderReplicatedBlocks" : 0, 224 | "CorruptBlocks" : 0, 225 | "ScheduledReplicationBlocks" : 0, 226 | "PendingDeletionBlocks" : 0, 227 | "ExcessBlocks" : 0, 228 | "PostponedMisreplicatedBlocks" : 0, 229 | "PendingDataNodeMessageCount" : 0, 230 | "MillisSinceLastLoadedEdits" : 0, 231 | "BlockCapacity" : 2097152, 232 | "StaleDataNodes" : 0, 233 | "TotalFiles" : 184, 234 | "TotalSyncCount" : 7 235 | } 236 | */ 237 | if nameDataMap["name"] == "Hadoop:service=NameNode,name=FSNamesystem" { 238 | e.MissingBlocks.Set(nameDataMap["MissingBlocks"].(float64)) 239 | e.CapacityTotal.Set(nameDataMap["CapacityTotal"].(float64)) 240 | e.CapacityUsed.Set(nameDataMap["CapacityUsed"].(float64)) 241 | e.CapacityRemaining.Set(nameDataMap["CapacityRemaining"].(float64)) 242 | e.CapacityUsedNonDFS.Set(nameDataMap["CapacityUsedNonDFS"].(float64)) 243 | e.BlocksTotal.Set(nameDataMap["BlocksTotal"].(float64)) 244 | e.FilesTotal.Set(nameDataMap["FilesTotal"].(float64)) 245 | e.CorruptBlocks.Set(nameDataMap["CorruptBlocks"].(float64)) 246 | e.ExcessBlocks.Set(nameDataMap["ExcessBlocks"].(float64)) 247 | e.StaleDataNodes.Set(nameDataMap["StaleDataNodes"].(float64)) 248 | } 249 | if nameDataMap["name"] == "java.lang:type=GarbageCollector,name=ParNew" { 250 | e.pnGcCount.Set(nameDataMap["CollectionCount"].(float64)) 251 | e.pnGcTime.Set(nameDataMap["CollectionTime"].(float64)) 252 | } 253 | if nameDataMap["name"] == "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep" { 254 | e.cmsGcCount.Set(nameDataMap["CollectionCount"].(float64)) 255 | e.cmsGcTime.Set(nameDataMap["CollectionTime"].(float64)) 256 | } 257 | /* 258 | "name" : "java.lang:type=Memory", 259 | "modelerType" : "sun.management.MemoryImpl", 260 | "HeapMemoryUsage" : { 261 | "committed" : 1060372480, 262 | "init" : 1073741824, 263 | "max" : 1060372480, 264 | "used" : 124571464 265 | }, 266 | */ 267 | if nameDataMap["name"] == "java.lang:type=Memory" { 268 | heapMemoryUsage := nameDataMap["HeapMemoryUsage"].(map[string]interface{}) 269 | e.heapMemoryUsageCommitted.Set(heapMemoryUsage["committed"].(float64)) 270 | e.heapMemoryUsageInit.Set(heapMemoryUsage["init"].(float64)) 271 | e.heapMemoryUsageMax.Set(heapMemoryUsage["max"].(float64)) 272 | e.heapMemoryUsageUsed.Set(heapMemoryUsage["used"].(float64)) 273 | } 274 | 275 | if nameDataMap["name"] == "Hadoop:service=NameNode,name=FSNamesystem" { 276 | if nameDataMap["tag.HAState"] == "active" { 277 | e.isActive.Set(1) 278 | } else { 279 | e.isActive.Set(0) 280 | } 281 | } 282 | 283 | } 284 | e.MissingBlocks.Collect(ch) 285 | e.CapacityTotal.Collect(ch) 286 | e.CapacityUsed.Collect(ch) 287 | e.CapacityRemaining.Collect(ch) 288 | e.CapacityUsedNonDFS.Collect(ch) 289 | e.BlocksTotal.Collect(ch) 290 | e.FilesTotal.Collect(ch) 291 | e.CorruptBlocks.Collect(ch) 292 | e.ExcessBlocks.Collect(ch) 293 | e.StaleDataNodes.Collect(ch) 294 | e.pnGcCount.Collect(ch) 295 | e.pnGcTime.Collect(ch) 296 | e.cmsGcCount.Collect(ch) 297 | e.cmsGcTime.Collect(ch) 298 | e.heapMemoryUsageCommitted.Collect(ch) 299 | e.heapMemoryUsageInit.Collect(ch) 300 | e.heapMemoryUsageMax.Collect(ch) 301 | e.heapMemoryUsageUsed.Collect(ch) 302 | e.isActive.Collect(ch) 303 | } 304 | 305 | func main() { 306 | flag.Parse() 307 | 308 | exporter := NewExporter(*namenodeJmxUrl) 309 | prometheus.MustRegister(exporter) 310 | 311 | log.Printf("Starting Server: %s", *listenAddress) 312 | http.Handle(*metricsPath, prometheus.Handler()) 313 | http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 314 | w.Write([]byte(` 315 | NameNode Exporter 316 | 317 |

NameNode Exporter

318 |

Metrics

319 | 320 | `)) 321 | }) 322 | err := http.ListenAndServe(*listenAddress, nil) 323 | if err != nil { 324 | log.Fatal(err) 325 | } 326 | } 327 | -------------------------------------------------------------------------------- /resourcemanager_exporter.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "io/ioutil" 7 | "net/http" 8 | 9 | "github.com/prometheus/client_golang/prometheus" 10 | "github.com/prometheus/log" 11 | ) 12 | 13 | const ( 14 | namespace = "resourcemanager" 15 | ) 16 | 17 | var ( 18 | listenAddress = flag.String("web.listen-address", ":9088", "Address on which to expose metrics and web interface.") 19 | metricsPath = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.") 20 | resourceManagerUrl = flag.String("resourcemanager.url", "http://localhost:8088", "Hadoop ResourceManager URL.") 21 | ) 22 | 23 | type Exporter struct { 24 | url string 25 | activeNodes prometheus.Gauge 26 | rebootedNodes prometheus.Gauge 27 | decommissionedNodes prometheus.Gauge 28 | unhealthyNodes prometheus.Gauge 29 | lostNodes prometheus.Gauge 30 | totalNodes prometheus.Gauge 31 | totalVirtualCores prometheus.Gauge 32 | availableMB prometheus.Gauge 33 | reservedMB prometheus.Gauge 34 | appsKilled prometheus.Gauge 35 | appsFailed prometheus.Gauge 36 | appsRunning prometheus.Gauge 37 | appsPending prometheus.Gauge 38 | appsCompleted prometheus.Gauge 39 | appsSubmitted prometheus.Gauge 40 | allocatedMB prometheus.Gauge 41 | reservedVirtualCores prometheus.Gauge 42 | availableVirtualCores prometheus.Gauge 43 | allocatedVirtualCores prometheus.Gauge 44 | containersAllocated prometheus.Gauge 45 | containersReserved prometheus.Gauge 46 | containersPending prometheus.Gauge 47 | totalMB prometheus.Gauge 48 | } 49 | 50 | func NewExporter(url string) *Exporter { 51 | return &Exporter{ 52 | url: url, 53 | activeNodes: prometheus.NewGauge(prometheus.GaugeOpts{ 54 | Namespace: namespace, 55 | Name: "activeNodes", 56 | Help: "activeNodes", 57 | }), 58 | rebootedNodes: prometheus.NewGauge(prometheus.GaugeOpts{ 59 | Namespace: namespace, 60 | Name: "rebootedNodes", 61 | Help: "rebootedNodes", 62 | }), 63 | decommissionedNodes: prometheus.NewGauge(prometheus.GaugeOpts{ 64 | Namespace: namespace, 65 | Name: "decommissionedNodes", 66 | Help: "decommissionedNodes", 67 | }), 68 | unhealthyNodes: prometheus.NewGauge(prometheus.GaugeOpts{ 69 | Namespace: namespace, 70 | Name: "unhealthyNodes", 71 | Help: "unhealthyNodes", 72 | }), 73 | lostNodes: prometheus.NewGauge(prometheus.GaugeOpts{ 74 | Namespace: namespace, 75 | Name: "lostNodes", 76 | Help: "lostNodes", 77 | }), 78 | totalNodes: prometheus.NewGauge(prometheus.GaugeOpts{ 79 | Namespace: namespace, 80 | Name: "totalNodes", 81 | Help: "totalNodes", 82 | }), 83 | totalVirtualCores: prometheus.NewGauge(prometheus.GaugeOpts{ 84 | Namespace: namespace, 85 | Name: "totalVirtualCores", 86 | Help: "totalVirtualCores", 87 | }), 88 | availableMB: prometheus.NewGauge(prometheus.GaugeOpts{ 89 | Namespace: namespace, 90 | Name: "availableMB", 91 | Help: "availableMB", 92 | }), 93 | reservedMB: prometheus.NewGauge(prometheus.GaugeOpts{ 94 | Namespace: namespace, 95 | Name: "reservedMB", 96 | Help: "reservedMB", 97 | }), 98 | appsKilled: prometheus.NewGauge(prometheus.GaugeOpts{ 99 | Namespace: namespace, 100 | Name: "appsKilled", 101 | Help: "appsKilled", 102 | }), 103 | appsFailed: prometheus.NewGauge(prometheus.GaugeOpts{ 104 | Namespace: namespace, 105 | Name: "appsFailed", 106 | Help: "appsFailed", 107 | }), 108 | appsRunning: prometheus.NewGauge(prometheus.GaugeOpts{ 109 | Namespace: namespace, 110 | Name: "appsRunning", 111 | Help: "appsRunning", 112 | }), 113 | appsPending: prometheus.NewGauge(prometheus.GaugeOpts{ 114 | Namespace: namespace, 115 | Name: "appsPending", 116 | Help: "appsPending", 117 | }), 118 | appsCompleted: prometheus.NewGauge(prometheus.GaugeOpts{ 119 | Namespace: namespace, 120 | Name: "appsCompleted", 121 | Help: "appsCompleted", 122 | }), 123 | appsSubmitted: prometheus.NewGauge(prometheus.GaugeOpts{ 124 | Namespace: namespace, 125 | Name: "appsSubmitted", 126 | Help: "appsSubmitted", 127 | }), 128 | allocatedMB: prometheus.NewGauge(prometheus.GaugeOpts{ 129 | Namespace: namespace, 130 | Name: "allocatedMB", 131 | Help: "allocatedMB", 132 | }), 133 | reservedVirtualCores: prometheus.NewGauge(prometheus.GaugeOpts{ 134 | Namespace: namespace, 135 | Name: "reservedVirtualCores", 136 | Help: "reservedVirtualCores", 137 | }), 138 | availableVirtualCores: prometheus.NewGauge(prometheus.GaugeOpts{ 139 | Namespace: namespace, 140 | Name: "availableVirtualCores", 141 | Help: "availableVirtualCores", 142 | }), 143 | allocatedVirtualCores: prometheus.NewGauge(prometheus.GaugeOpts{ 144 | Namespace: namespace, 145 | Name: "allocatedVirtualCores", 146 | Help: "allocatedVirtualCores", 147 | }), 148 | containersAllocated: prometheus.NewGauge(prometheus.GaugeOpts{ 149 | Namespace: namespace, 150 | Name: "containersAllocated", 151 | Help: "containersAllocated", 152 | }), 153 | containersReserved: prometheus.NewGauge(prometheus.GaugeOpts{ 154 | Namespace: namespace, 155 | Name: "containersReserved", 156 | Help: "containersReserved", 157 | }), 158 | containersPending: prometheus.NewGauge(prometheus.GaugeOpts{ 159 | Namespace: namespace, 160 | Name: "containersPending", 161 | Help: "containersPending", 162 | }), 163 | totalMB: prometheus.NewGauge(prometheus.GaugeOpts{ 164 | Namespace: namespace, 165 | Name: "totalMB", 166 | Help: "totalMB", 167 | }), 168 | } 169 | } 170 | 171 | // Describe implements the prometheus.Collector interface. 172 | func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { 173 | e.activeNodes.Describe(ch) 174 | e.rebootedNodes.Describe(ch) 175 | e.decommissionedNodes.Describe(ch) 176 | e.unhealthyNodes.Describe(ch) 177 | e.lostNodes.Describe(ch) 178 | e.totalNodes.Describe(ch) 179 | e.totalVirtualCores.Describe(ch) 180 | e.availableMB.Describe(ch) 181 | e.reservedMB.Describe(ch) 182 | e.appsKilled.Describe(ch) 183 | e.appsFailed.Describe(ch) 184 | e.appsRunning.Describe(ch) 185 | e.appsPending.Describe(ch) 186 | e.appsCompleted.Describe(ch) 187 | e.appsSubmitted.Describe(ch) 188 | e.allocatedMB.Describe(ch) 189 | e.reservedVirtualCores.Describe(ch) 190 | e.availableVirtualCores.Describe(ch) 191 | e.allocatedVirtualCores.Describe(ch) 192 | e.containersAllocated.Describe(ch) 193 | e.containersReserved.Describe(ch) 194 | e.containersPending.Describe(ch) 195 | e.totalMB.Describe(ch) 196 | } 197 | 198 | // Collect implements the prometheus.Collector interface. 199 | func (e *Exporter) Collect(ch chan<- prometheus.Metric) { 200 | resp, err := http.Get(e.url + "/ws/v1/cluster/metrics") 201 | if err != nil { 202 | log.Error(err) 203 | } 204 | defer resp.Body.Close() 205 | data, err := ioutil.ReadAll(resp.Body) 206 | if err != nil { 207 | log.Error(err) 208 | } 209 | /* 210 | "clusterMetrics": { 211 | "activeNodes": 3, 212 | "rebootedNodes": 0, 213 | "decommissionedNodes": 0, 214 | "unhealthyNodes": 0, 215 | "lostNodes": 0, 216 | "totalNodes": 3, 217 | "totalVirtualCores": 9, 218 | "availableMB": 6144, 219 | "reservedMB": 0, 220 | "appsKilled": 0, 221 | "appsFailed": 1, 222 | "appsRunning": 0, 223 | "appsPending": 0, 224 | "appsCompleted": 9, 225 | "appsSubmitted": 10, 226 | "allocatedMB": 0, 227 | "reservedVirtualCores": 0, 228 | "availableVirtualCores": 9, 229 | "allocatedVirtualCores": 0, 230 | "containersAllocated": 0, 231 | "containersReserved": 0, 232 | "containersPending": 0, 233 | "totalMB": 6144 234 | } 235 | */ 236 | var f interface{} 237 | err = json.Unmarshal(data, &f) 238 | if err != nil { 239 | log.Error(err) 240 | } 241 | m := f.(map[string]interface{}) 242 | cm := m["clusterMetrics"].(map[string]interface{}) 243 | e.activeNodes.Set(cm["activeNodes"].(float64)) 244 | e.rebootedNodes.Set(cm["rebootedNodes"].(float64)) 245 | e.decommissionedNodes.Set(cm["decommissionedNodes"].(float64)) 246 | e.unhealthyNodes.Set(cm["unhealthyNodes"].(float64)) 247 | e.lostNodes.Set(cm["lostNodes"].(float64)) 248 | e.totalNodes.Set(cm["totalNodes"].(float64)) 249 | e.totalVirtualCores.Set(cm["totalVirtualCores"].(float64)) 250 | e.availableMB.Set(cm["availableMB"].(float64)) 251 | e.reservedMB.Set(cm["reservedMB"].(float64)) 252 | e.appsKilled.Set(cm["appsKilled"].(float64)) 253 | e.appsFailed.Set(cm["appsFailed"].(float64)) 254 | e.appsRunning.Set(cm["appsRunning"].(float64)) 255 | e.appsPending.Set(cm["appsPending"].(float64)) 256 | e.appsCompleted.Set(cm["appsCompleted"].(float64)) 257 | e.appsSubmitted.Set(cm["appsSubmitted"].(float64)) 258 | e.allocatedMB.Set(cm["allocatedMB"].(float64)) 259 | e.reservedVirtualCores.Set(cm["reservedVirtualCores"].(float64)) 260 | e.availableVirtualCores.Set(cm["availableVirtualCores"].(float64)) 261 | e.allocatedVirtualCores.Set(cm["allocatedVirtualCores"].(float64)) 262 | e.containersAllocated.Set(cm["containersAllocated"].(float64)) 263 | e.containersReserved.Set(cm["containersReserved"].(float64)) 264 | e.containersPending.Set(cm["containersPending"].(float64)) 265 | e.totalMB.Set(cm["totalMB"].(float64)) 266 | 267 | e.activeNodes.Collect(ch) 268 | e.rebootedNodes.Collect(ch) 269 | e.decommissionedNodes.Collect(ch) 270 | e.unhealthyNodes.Collect(ch) 271 | e.lostNodes.Collect(ch) 272 | e.totalNodes.Collect(ch) 273 | e.totalVirtualCores.Collect(ch) 274 | e.availableMB.Collect(ch) 275 | e.reservedMB.Collect(ch) 276 | e.appsKilled.Collect(ch) 277 | e.appsFailed.Collect(ch) 278 | e.appsRunning.Collect(ch) 279 | e.appsPending.Collect(ch) 280 | e.appsCompleted.Collect(ch) 281 | e.appsSubmitted.Collect(ch) 282 | e.allocatedMB.Collect(ch) 283 | e.reservedVirtualCores.Collect(ch) 284 | e.availableVirtualCores.Collect(ch) 285 | e.allocatedVirtualCores.Collect(ch) 286 | e.containersAllocated.Collect(ch) 287 | e.containersReserved.Collect(ch) 288 | e.containersPending.Collect(ch) 289 | e.totalMB.Collect(ch) 290 | 291 | } 292 | 293 | func main() { 294 | flag.Parse() 295 | 296 | exporter := NewExporter(*resourceManagerUrl) 297 | prometheus.MustRegister(exporter) 298 | 299 | log.Printf("Starting Server: %s", *listenAddress) 300 | http.Handle(*metricsPath, prometheus.Handler()) 301 | http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 302 | w.Write([]byte(` 303 | ResourceManager Exporter 304 | 305 |

ResourceManager Exporter

306 |

Metrics

307 | 308 | `)) 309 | }) 310 | err := http.ListenAndServe(*listenAddress, nil) 311 | if err != nil { 312 | log.Fatal(err) 313 | } 314 | } 315 | --------------------------------------------------------------------------------