├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── alert.go ├── metadata.go ├── monitor.go └── offset.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.a 3 | *.so 4 | 5 | _obj 6 | _test 7 | 8 | *.sh 9 | monitor 10 | 11 | *.exe 12 | *.test 13 | *.prof 14 | 15 | vendor 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | WORKDIR=`pwd` 2 | 3 | default: build 4 | 5 | install: 6 | go get github.com/smallnest/rpcx/... 7 | 8 | vet: 9 | go vet ./... 10 | 11 | tools: 12 | go get honnef.co/go/tools/cmd/staticcheck 13 | go get honnef.co/go/tools/cmd/gosimple 14 | go get honnef.co/go/tools/cmd/unused 15 | go get github.com/gordonklaus/ineffassign 16 | go get github.com/fzipp/gocyclo 17 | go get github.com/golang/lint/golint 18 | 19 | lint: 20 | golint ./... 21 | 22 | staticcheck: 23 | staticcheck ./... 24 | 25 | gosimple: 26 | gosimple ./... 27 | 28 | unused: 29 | unused ./... 30 | 31 | ineffassign: 32 | ineffassign . 33 | 34 | gocyclo: 35 | @ gocyclo -over 25 $(shell find . -name "*.go" |egrep -v "pb\.go|_test\.go") 36 | 37 | check: staticcheck gosimple unused ineffassign gocyclo 38 | 39 | doc: 40 | godoc -http=:6060 41 | 42 | deps: 43 | go list -f '{{ join .Deps "\n"}}' ./... |grep "/"| grep "\." | sort |uniq 44 | 45 | fmt: 46 | go fmt ./... 47 | 48 | build: 49 | go build -o kafka_monitor ./... 50 | 51 | test: 52 | go test ./... 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## KafkaMonitor 2 | 3 | the design and implementation reference to [kt](https://github.com/fgeller/kt) and [kafkatools](https://github.com/jurriaan/kafkatools) 4 | 5 | 6 | ### Install 7 | 8 | ``` 9 | go get github.com/smallnest/kafka_monitor 10 | ``` 11 | 12 | ### Run 13 | 14 | ``` 15 | # change the below options with your environments 16 | informEmail=your_email@xxxxxx.com 17 | smtpHost=xxxxxx 18 | smtpPort=25 19 | smtpUser=xxxxxx@xxxxxx.com 20 | smtpPassword=xxxxxx 21 | 22 | brokers=10.77.11.1:9092,10.77.11.2:9092,10.77.11.3:9092,10.77.11.4:9092 23 | version=0.9.0.1 24 | zkAddr=10.77.0.1:2181,10.77.0.2:2181,10.77.0.3:2181,10.77.0.4:2181,10.77.0.5:2181 25 | 26 | ./kafka_monitor -brokers ${brokers} -version ${version} -email=${informEmail} -smtp.host ${smtpHost} -smtp.port ${smtpPort) -smtp.user ${smtpUser} -smtp.password ${smtpPassword} -zkAddr ${zkAddr} -basePath "/kafka" 27 | ``` 28 | 29 | 30 | ### Output 31 | 32 | Once some partitions are not writable, down or some consumers don't work, an email will be sent to your email with details. 33 | 34 | Normal kafka cluster info is output in console: 35 | 36 | ``` 37 | Brokers: 10.77.11.1:9092,10.77.11.2:9092,10.77.11.4:9092:10.77.11.3:9092 38 | Version: 0.9.0.1 39 | Topic: products, Group: default, Partitions: 32 40 | 41 | +-----------+-------------------+--------+---------------+---------------+ 42 | | PARTITION | LEADER ADDRESS | LEADER | REPLICAS | ISR | 43 | +-----------+-------------------+--------+---------------+---------------+ 44 | | 0 | 10.77.11.1:9092 | 1 | [1 2 3] | [1 2 3] | 45 | | 1 | 10.77.11.2:9092 | 2 | [1 2 4] | [1 2 4] | 46 | | 2 | 10.77.11.4:9092 | 4 | [2 4 3] | [2 4 3] | 47 | | 3 | 10.77.11.3:9092 | 3 | [1 4 3] | [1 4 3] | 48 | | 4 | 10.77.11.1:9092 | 1 | [1 2 4] | [1 2 4] | 49 | | 5 | 10.77.11.2:9092 | 2 | [2 4 3] | [2 4 3] | 50 | | 6 | 10.77.11.4:9092 | 4 | [1 4 3] | [1 4 3] | 51 | | 7 | 10.77.11.3:9092 | 3 | [1 2 3] | [1 2 3] | 52 | | 8 | 10.77.11.1:9092 | 1 | [1 4 3] | [1 4 3] | 53 | | 9 | 10.77.11.2:9092 | 2 | [1 2 3] | [1 2 3] | 54 | | 10 | 10.77.11.4:9092 | 4 | [1 2 4] | [1 2 4] | 55 | | 11 | 10.77.11.3:9092 | 3 | [2 4 3] | [2 4 3] | 56 | | 12 | 10.77.11.1:9092 | 1 | [1 2 3] | [1 2 3] | 57 | | 13 | 10.77.11.2:9092 | 2 | [1 2 4] | [1 2 4] | 58 | | 14 | 10.77.11.4:9092 | 4 | [2 4 3] | [2 4 3] | 59 | | 15 | 10.77.11.3:9092 | 3 | [1 4 3] | [1 4 3] | 60 | | 16 | 10.77.11.1:9092 | 1 | [1 2 4] | [1 2 4] | 61 | | 17 | 10.77.11.2:9092 | 2 | [2 4 3] | [2 4 3] | 62 | | 18 | 10.77.11.4:9092 | 4 | [1 4 3] | [1 4 3] | 63 | | 19 | 10.77.11.3:9092 | 3 | [1 2 3] | [1 2 3] | 64 | | 20 | 10.77.11.1:9092 | 1 | [1 4 3] | [1 4 3] | 65 | | 21 | 10.77.11.2:9092 | 2 | [1 2 3] | [1 2 3] | 66 | | 22 | 10.77.11.4:9092 | 4 | [1 2 4] | [1 2 4] | 67 | | 23 | 10.77.11.3:9092 | 3 | [2 4 3] | [2 4 3] | 68 | | 24 | 10.77.11.1:9092 | 1 | [1 2 3] | [1 2 3] | 69 | | 25 | 10.77.11.2:9092 | 2 | [1 2 4] | [1 2 4] | 70 | | 26 | 10.77.11.4:9092 | 4 | [2 4 3] | [2 4 3] | 71 | | 27 | 10.77.11.3:9092 | 3 | [1 4 3] | [1 4 3] | 72 | | 28 | 10.77.11.1:9092 | 1 | [1 2 4] | [1 2 4] | 73 | | 29 | 10.77.11.2:9092 | 2 | [2 4 3] | [2 4 3] | 74 | | 30 | 10.77.11.4:9092 | 4 | [1 4 3] | [1 4 3] | 75 | | 31 | 10.77.11.3:9092 | 3 | [1 2 3] | [1 2 3] | 76 | +-----------+-------------------+--------+---------------+---------------+ 77 | 78 | 79 | +-----------+------------+--------------+-----+ 80 | | PARTITION | END OF LOG | GROUP OFFSET | LAG | 81 | +-----------+------------+--------------+-----+ 82 | | 0 | 1106509557 | 1106509542 | 15 | 83 | | 1 | 1106517793 | 1106517779 | 14 | 84 | | 2 | 1106573067 | 1106573047 | 20 | 85 | | 3 | 1106531253 | 1106531253 | 0 | 86 | | 4 | 1106529394 | 1106529385 | 9 | 87 | | 5 | 11065504 | 1106550146 | 4 | 88 | | 6 | 1106467253 | 1106467248 | 5 | 89 | | 7 | 1106479964 | 1106479964 | 0 | 90 | | 8 | 1106464607 | 1106464594 | 13 | 91 | | 9 | 1106503601 | 1106503589 | 12 | 92 | | 10 | 1106543568 | 1106543552 | 16 | 93 | | 11 | 1106558780 | 1106558768 | 12 | 94 | | 12 | 1106494251 | 1106494238 | 13 | 95 | | 13 | 1106561496 | 1106561490 | 6 | 96 | | 14 | 1106521678 | 1106521673 | 5 | 97 | | 15 | 1106569720 | 1106569686 | 34 | 98 | | 16 | 1106573067 | 1106573049 | 18 | 99 | | 17 | 1106521317 | 1106521305 | 12 | 100 | | 18 | 1106534494 | 1106534484 | 10 | 101 | | 19 | 1106481325 | 1106481268 | 57 | 102 | | 20 | 1106530486 | 1106530465 | 21 | 103 | | 21 | 1106500650 | 1106500624 | 26 | 104 | | 22 | 1106552135 | 1106552113 | 22 | 105 | | 23 | 1106551799 | 1106551771 | 28 | 106 | | 24 | 1106564761 | 1106564750 | 11 | 107 | | 25 | 1106537078 | 1106537064 | 14 | 108 | | 26 | 1106573795 | 1106573781 | 14 | 109 | | 27 | 1106493569 | 1106493554 | 15 | 110 | | 28 | 1106500029 | 1106500015 | 14 | 111 | | 29 | 1106528095 | 1106528078 | 17 | 112 | | 30 | 1106493651 | 1106493629 | 22 | 113 | | 31 | 1106562315 | 1106562312 | 3 | 114 | +-----------+------------+--------------+-----+ 115 | | TOTAL | 482 | 116 | +-----------+------------+--------------+-----+ 117 | ``` -------------------------------------------------------------------------------- /alert.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "strings" 7 | 8 | gomail "gopkg.in/gomail.v2" 9 | ) 10 | 11 | func alert(informEmail string, subject string, content []byte, 12 | smtpHost string, smtpPort int, smtpUser string, smtpPassword string) { 13 | m := gomail.NewMessage() 14 | m.SetHeader("From", "kafka_monitor@sina.com") 15 | m.SetHeader("To", informEmail) 16 | m.SetHeader("Subject", subject) 17 | 18 | c := string(content) 19 | c = strings.Replace(c, "[30m", "", -1) 20 | c = strings.Replace(c, "[31m", "", -1) 21 | c = strings.Replace(c, "[32m", "", -1) 22 | c = strings.Replace(c, "[33m", "", -1) 23 | c = strings.Replace(c, "[34m", "", -1) 24 | c = strings.Replace(c, "[35m", "", -1) 25 | c = strings.Replace(c, "[36m", "", -1) 26 | c = strings.Replace(c, "[37m", "", -1) 27 | c = strings.Replace(c, "[0m", "", -1) 28 | 29 | m.SetBody("text/plain", string(content)) 30 | 31 | d := gomail.NewDialer(smtpHost, smtpPort, smtpUser, smtpPassword) 32 | if err := d.DialAndSend(m); err != nil { 33 | fmt.Printf("failed to send alarm: %v\n", err) 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /metadata.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "strings" 7 | 8 | "github.com/Shopify/sarama" 9 | "github.com/samuel/go-zookeeper/zk" 10 | ) 11 | 12 | type PartitionInfo struct { 13 | Topic string 14 | Partition int32 15 | Leader int32 16 | LeaderAddress string 17 | Replicas []int32 18 | Isr string 19 | } 20 | 21 | func GetPartitionInfo(client sarama.Client, topic string, partitions []int32, c *zk.Conn, basePath string) []*PartitionInfo { 22 | var infos []*PartitionInfo 23 | for _, partition := range partitions { 24 | replicas, err := client.Replicas(topic, partition) 25 | if err != nil { 26 | fmt.Printf("failed to get replicas for topic=%s, partition=%d\n", topic, partition) 27 | leader, _ := client.Leader(topic, partition) 28 | infos = append(infos, &PartitionInfo{Topic: topic, Partition: partition, Leader: leader.ID(), Replicas: nil}) 29 | continue 30 | } 31 | 32 | leader, _ := client.Leader(topic, partition) 33 | 34 | info := &PartitionInfo{ 35 | Topic: topic, 36 | Partition: partition, 37 | Leader: leader.ID(), 38 | LeaderAddress: leader.Addr(), 39 | Replicas: replicas, 40 | Isr: getIsr(c, basePath, topic, partition)} 41 | infos = append(infos, info) 42 | } 43 | 44 | return infos 45 | } 46 | 47 | func getIsr(conn *zk.Conn, basePath string, topic string, partition int32) string { 48 | if conn == nil { 49 | return "" 50 | } 51 | path := fmt.Sprintf("%s/brokers/topics/%s/partitions/%d/state", basePath, topic, partition) 52 | bytes, _, err := conn.Get(path) 53 | if err != nil { 54 | fmt.Printf("failed to get ISR for topic=%s, partition=%d becaue of %v\n", topic, partition, err) 55 | return "" 56 | } 57 | 58 | //{"controller_epoch":6,"leader":1,"version":1,"leader_epoch":23,"isr":[1,4,32]} 59 | isr := string(bytes) 60 | i := strings.Index(isr, `isr":[`) 61 | isr = isr[i+6:] 62 | i = strings.Index(isr, "]") 63 | isr = isr[:i] 64 | return strings.Replace(isr, ",", " ", -1) 65 | } 66 | -------------------------------------------------------------------------------- /monitor.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "sort" 6 | "strconv" 7 | "strings" 8 | "time" 9 | 10 | "flag" 11 | 12 | "fmt" 13 | 14 | "os" 15 | 16 | "github.com/Shopify/sarama" 17 | "github.com/fatih/color" 18 | "github.com/olekukonko/tablewriter" 19 | "github.com/samuel/go-zookeeper/zk" 20 | ) 21 | 22 | var ( 23 | zkAddr = flag.String("zkAddr", "", "zookeeper address") 24 | basePath = flag.String("basePath", "/kafka", "kafka base path in zookeeper") 25 | brokers = flag.String("brokers", "localhost:9092", "brokers' address") 26 | topic = flag.String("topic", "trigger", "topic name") 27 | group = flag.String("group", "default", "consumer group name") 28 | version = flag.String("version", "0.10.0.1", "kafka version. min version is 0.8.2.0") 29 | lagThreshold = flag.Int("lagThreshold", 1000, "alarm lag threshold for partition") 30 | totalLagThreshold = flag.Int("totalLagThreshold", 5000, "alarm total lag threshold for topic") 31 | interval = flag.Duration("duration", time.Minute, "check interval time") 32 | informEmail = flag.String("email", "xxx@xxxxxx.com", "inform user email") 33 | smtpHost = flag.String("smtp.host", "smtp.sina.com", "smtp host for sending alarms") 34 | smtpPort = flag.Int("smtp.port", 25, "smtp port for sending alarms") 35 | smtpUser = flag.String("smtp.user", "kafka_monitor", "smtp user for sending alarms") 36 | smtpPassword = flag.String("smtp.password", "xxxxxx", "smtp user password for sending alarms") 37 | ) 38 | 39 | var ( 40 | maybeProblem = false 41 | restored = true 42 | lastTriggeredTime time.Time 43 | mergeAlertDuration = 5 * time.Minute 44 | kafkaVersions = kafkaVersion() 45 | ) 46 | 47 | func kafkaVersion() map[string]sarama.KafkaVersion { 48 | m := make(map[string]sarama.KafkaVersion) 49 | m["0.8.2.0"] = sarama.V0_8_2_0 50 | m["0.8.2.1"] = sarama.V0_8_2_1 51 | m["0.8.2.2"] = sarama.V0_8_2_2 52 | m["0.9.0.0"] = sarama.V0_9_0_0 53 | m["0.9.0.1"] = sarama.V0_9_0_1 54 | m["0.10.0.0"] = sarama.V0_10_0_0 55 | m["0.10.0.1"] = sarama.V0_10_0_1 56 | m["0.10.1.0"] = sarama.V0_10_1_0 57 | return m 58 | } 59 | 60 | func main() { 61 | flag.Parse() 62 | 63 | for { 64 | check() 65 | } 66 | } 67 | 68 | func check() { 69 | kafkaBrokers := strings.Split(*brokers, ",") 70 | sort.Sort(sort.StringSlice(kafkaBrokers)) 71 | 72 | v := kafkaVersions[*version] 73 | client := NewSaramaClient(kafkaBrokers, v) 74 | 75 | var buf bytes.Buffer 76 | var err error 77 | var c *zk.Conn 78 | 79 | defer func() { 80 | if r := recover(); r != nil { 81 | fmt.Printf("check error: %v", r) 82 | 83 | bytes := buf.Bytes() 84 | os.Stdout.Write(bytes) 85 | 86 | client.Close() 87 | 88 | subject := fmt.Sprintf("Alarm: topic=%s, brokers: %s", *topic, *brokers) 89 | alert(*informEmail, subject, []byte(fmt.Sprintf("%v", r)), *smtpHost, *smtpPort, *smtpUser, *smtpPassword) 90 | } 91 | }() 92 | 93 | if *zkAddr != "" { 94 | c, _, err = zk.Connect(strings.Split(*zkAddr, ","), 30*time.Second) 95 | if err != nil { 96 | panic(err) 97 | } 98 | 99 | defer c.Close() 100 | } 101 | 102 | ticker := time.NewTicker(*interval) 103 | for range ticker.C { 104 | buf.Reset() 105 | 106 | //check brokers change event 107 | newKafkaBrokers := runtimeKafkaBrokers(client) 108 | s1 := strings.Join(kafkaBrokers, ",") 109 | s2 := strings.Join(newKafkaBrokers, ",") 110 | if s1 != s2 { 111 | subject := fmt.Sprintf("Broker changed: topic=%s, brokers: %s", *topic, *brokers) 112 | alert(*informEmail, subject, []byte(fmt.Sprintf("prior brokers: %s \n current brokers: %s\n", s1, s2)), *smtpHost, *smtpPort, *smtpUser, *smtpPassword) 113 | 114 | if len(newKafkaBrokers) > 0 { 115 | kafkaBrokers = newKafkaBrokers 116 | *brokers = s2 117 | } 118 | } 119 | 120 | partitions, err := client.Partitions(*topic) 121 | if err != nil { 122 | fmt.Printf("failed to get partitions for topic=%s, err=%v\n", *topic, err) 123 | panic(err) 124 | } 125 | 126 | writablePartitions, err := client.WritablePartitions(*topic) 127 | if err != nil { 128 | fmt.Printf("failed to get writable partitions for topic=%s, err=%v\n", *topic, err) 129 | panic(err) 130 | } 131 | 132 | if len(partitions) != len(writablePartitions) { 133 | buf.WriteString("some partitions are not writable\n") 134 | buf.WriteString(fmt.Sprintf("all partitions: %v\n", partitions)) 135 | buf.WriteString(fmt.Sprintf("writable partitions: %v\n", writablePartitions)) 136 | //TODO print unwritable partitions 137 | maybeProblem = true 138 | } 139 | 140 | buf.WriteString(fmt.Sprintf("Time: %s\n", time.Now().Format("2006-01-02 15:04:05"))) 141 | buf.WriteString(fmt.Sprintf("Brokers: %s\nVersion: %s\n", color.GreenString(*brokers), color.GreenString(*version))) 142 | buf.WriteString(fmt.Sprintf("Topic: %s, Group: %s, Partitions: %s\n\n", 143 | color.GreenString(*topic), color.GreenString(*group), color.GreenString(strconv.Itoa(len(partitions))))) 144 | 145 | infos := GetPartitionInfo(client, *topic, partitions, c, *basePath) 146 | if len(infos) > 0 { 147 | table := tablewriter.NewWriter(&buf) 148 | table.SetHeader([]string{"partition", "leader address", "leader", "replicas", "isr"}) 149 | for _, info := range infos { 150 | replicas := fmt.Sprintf("%v", info.Replicas) 151 | replicas = replicas[1 : len(replicas)-1] 152 | replicas = compareString(replicas) 153 | isr := compareString(info.Isr) 154 | 155 | table.Append([]string{strconv.Itoa(int(info.Partition)), info.LeaderAddress, strconv.Itoa(int(info.Leader)), "[" + replicas + "]", "[" + isr + "]"}) 156 | 157 | if replicas != isr { 158 | maybeProblem = true 159 | } 160 | } 161 | 162 | table.SetAlignment(tablewriter.ALIGN_LEFT) 163 | table.Render() 164 | 165 | buf.WriteString("\n\n") 166 | } 167 | //lag 168 | if offsets, err := FetchOffsets(client, *topic, *group); err == nil { 169 | 170 | table := tablewriter.NewWriter(&buf) 171 | table.SetHeader([]string{"partition", "end of log", "group offset", "lag"}) 172 | 173 | var totalLag int64 174 | for _, info := range offsets { 175 | lag := info.PartitionOffset - info.GroupOffset 176 | if int(lag) > *lagThreshold { 177 | maybeProblem = true 178 | } 179 | 180 | totalLag += lag 181 | table.Append([]string{strconv.Itoa(int(info.Partition)), strconv.Itoa(int(info.PartitionOffset)), 182 | strconv.Itoa(int(info.GroupOffset)), convertLag(lag, *lagThreshold)}) 183 | } 184 | 185 | if int(totalLag) > *totalLagThreshold { 186 | maybeProblem = true 187 | } 188 | 189 | table.SetFooter([]string{"", "", "Total", convertLag(totalLag, *totalLagThreshold)}) 190 | table.SetAlignment(tablewriter.ALIGN_LEFT) 191 | table.SetFooterAlignment(tablewriter.ALIGN_LEFT) 192 | table.Render() 193 | } 194 | 195 | //output tables to stdout 196 | bytes := buf.Bytes() 197 | os.Stdout.Write(bytes) 198 | 199 | //first issue check or ignore check exceeds mergeAlertDuration 200 | if maybeProblem && time.Since(lastTriggeredTime) > mergeAlertDuration { 201 | restored = false 202 | lastTriggeredTime = time.Now() 203 | subject := fmt.Sprintf("Alarm: topic=%s, brokers: %s", *topic, *brokers) 204 | alert(*informEmail, subject, bytes, *smtpHost, *smtpPort, *smtpUser, *smtpPassword) 205 | } 206 | 207 | //fixed 208 | if !maybeProblem && !restored { 209 | subject := fmt.Sprintf("Fixed: topic=%s, brokers: %s", *topic, *brokers) 210 | alert(*informEmail, subject, bytes, *smtpHost, *smtpPort, *smtpUser, *smtpPassword) 211 | restored = true 212 | lastTriggeredTime = time.Unix(0, 0) 213 | } 214 | 215 | maybeProblem = false 216 | } 217 | } 218 | 219 | func compareString(s string) string { 220 | s1 := strings.Split(s, " ") 221 | sort.Sort(sort.StringSlice(s1)) 222 | 223 | return strings.Join(s1, " ") 224 | } 225 | func convertLag(lag int64, threshold int) string { 226 | lagStr := strconv.Itoa(int(lag)) 227 | if int(lag) > threshold { 228 | lagStr = color.RedString(lagStr) 229 | } 230 | 231 | return lagStr 232 | } 233 | 234 | func runtimeKafkaBrokers(client sarama.Client) []string { 235 | brokers := client.Brokers() 236 | var fetchedBrokers []string 237 | for _, b := range brokers { 238 | fetchedBrokers = append(fetchedBrokers, b.Addr()) 239 | } 240 | 241 | sort.Sort(sort.StringSlice(fetchedBrokers)) 242 | return fetchedBrokers 243 | } 244 | 245 | func NewSaramaClient(brokers []string, version sarama.KafkaVersion) sarama.Client { 246 | config := sarama.NewConfig() 247 | config.Version = version 248 | config.Consumer.Return.Errors = true 249 | config.Metadata.RefreshFrequency = 1 * time.Minute 250 | config.Metadata.Retry.Max = 10 251 | config.Net.MaxOpenRequests = 10 252 | config.Net.DialTimeout = 10 * time.Second 253 | config.Net.ReadTimeout = 10 * time.Second 254 | config.Net.WriteTimeout = 10 * time.Second 255 | 256 | client, err := sarama.NewClient(brokers, config) 257 | 258 | if err != nil { 259 | panic("Failed to start client: " + err.Error()) 260 | } 261 | 262 | return client 263 | } 264 | -------------------------------------------------------------------------------- /offset.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/Shopify/sarama" 7 | ) 8 | 9 | type OffsetInfo struct { 10 | Topic string 11 | Partition int32 12 | PartitionOffset int64 13 | GroupOffset int64 14 | } 15 | 16 | func FetchOffsets(client sarama.Client, topic string, group string) ([]*OffsetInfo, error) { 17 | var infos []*OffsetInfo 18 | 19 | partitions, err := client.Partitions(topic) 20 | if err != nil { 21 | fmt.Printf("failed to get partitions for topic=%s, err=%v\n", topic, err) 22 | return nil, err 23 | } 24 | 25 | offsetManager, err := sarama.NewOffsetManagerFromClient(group, client) 26 | if err != nil { 27 | fmt.Printf("failed to create offset manager for topic=%s, group=%s err=%v\n", topic, group, err) 28 | return nil, err 29 | } 30 | 31 | for _, partition := range partitions { 32 | partitionOffsetManager, err := offsetManager.ManagePartition(topic, partition) 33 | if err != nil { 34 | fmt.Printf("failed to get partition manager for topic=%s, partition=%d, err=%v\n", topic, partition, err) 35 | continue 36 | } 37 | cgOffset, _ := partitionOffsetManager.NextOffset() 38 | 39 | pOffset, err := client.GetOffset(topic, partition, sarama.OffsetNewest) 40 | if err != nil { 41 | fmt.Printf("failed to get partition offset for topic=%s, partition=%d, err=%v\n", topic, partition, err) 42 | continue 43 | } 44 | info := &OffsetInfo{ 45 | Topic: topic, 46 | Partition: partition, 47 | PartitionOffset: pOffset, 48 | GroupOffset: cgOffset, 49 | } 50 | 51 | infos = append(infos, info) 52 | } 53 | 54 | return infos, nil 55 | } 56 | --------------------------------------------------------------------------------