├── .gitignore ├── README.md ├── metrics.go ├── main.go ├── vendor └── vendor.json └── kafka.go /.gitignore: -------------------------------------------------------------------------------- 1 | vendor/*/ 2 | kafka-offset-exporter 3 | debug 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kafka-offset-exporter 2 | 3 | ---- 4 | 5 | ## no longer maintained 6 | 7 | My employer used to rely heavily on Kafka and so I could dogfood and iterate on 8 | this project regularly. Unfortunately, this is no longer the case and I don't 9 | have the resources to maintain and/or develop this anymore. 10 | 11 | I don't feel comfortable recommending people to use outdated and unmaintained 12 | software, so please consider using an established fork, forking this yourself, 13 | or creating a new-and-improved exporter as an alternative. 14 | 15 | Thanks all! 16 | 17 | ---- 18 | 19 | This is a Prometheus exporter for topic and consumer group offsets in Kafka. 20 | Your Kafka cluster must be on version 0.10.0.0 or above for this to work. 21 | 22 | ## Usage 23 | 24 | The only required parameter is a set of brokers to bootstrap into the cluster. 25 | 26 | By default, the oldest and newest offsets of all topics are retrieved and 27 | reported. You can also enable offset reporting for any consumer group but note 28 | that due to the way Sarama works, this requires querying for offsets for _all_ 29 | partitions for each consumer group which can take a long time. It is recommended 30 | to filter both topics and consumer groups to just the ones you care about. 31 | 32 | ``` 33 | $ ./kafka-offset-exporter -help 34 | Usage of ./kafka-offset-exporter: 35 | -brokers string 36 | Kafka brokers to connect to, comma-separated 37 | -fetchMax duration 38 | Max time before requesting updates from broker (default 40s) 39 | -fetchMin duration 40 | Min time before requesting updates from broker (default 15s) 41 | -groups string 42 | Also fetch offsets for consumer groups matching this regex (default none) 43 | -level string 44 | Logger level (default "info") 45 | -path string 46 | Path to export metrics on (default "/") 47 | -port int 48 | Port to export metrics on (default 9000) 49 | -refresh duration 50 | Time between refreshing cluster metadata (default 1m0s) 51 | -topics string 52 | Only fetch offsets for topics matching this regex (default all) 53 | ``` 54 | -------------------------------------------------------------------------------- /metrics.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "math" 7 | "net/http" 8 | "sync" 9 | "time" 10 | 11 | log "github.com/Sirupsen/logrus" 12 | "github.com/prometheus/client_golang/prometheus" 13 | "github.com/prometheus/client_golang/prometheus/promhttp" 14 | ) 15 | 16 | var ( 17 | metricOffsetOldest = prometheus.NewGaugeVec( 18 | prometheus.GaugeOpts{ 19 | Name: "kafka_offset_oldest", 20 | Help: "Oldest offset for a partition", 21 | }, 22 | []string{ 23 | "topic", 24 | "partition", 25 | }, 26 | ) 27 | metricOffsetNewest = prometheus.NewGaugeVec( 28 | prometheus.GaugeOpts{ 29 | Name: "kafka_offset_newest", 30 | Help: "Newest offset for a partition", 31 | }, 32 | []string{ 33 | "topic", 34 | "partition", 35 | }, 36 | ) 37 | metricOffsetConsumer = prometheus.NewGaugeVec( 38 | prometheus.GaugeOpts{ 39 | Name: "kafka_offset_consumer", 40 | Help: "Current offset for a consumer group", 41 | }, 42 | []string{ 43 | "topic", 44 | "partition", 45 | "group", 46 | }, 47 | ) 48 | ) 49 | 50 | func init() { 51 | prometheus.MustRegister(metricOffsetOldest) 52 | prometheus.MustRegister(metricOffsetNewest) 53 | prometheus.MustRegister(metricOffsetConsumer) 54 | } 55 | 56 | type serverConfig struct { 57 | port int 58 | path string 59 | } 60 | 61 | func mustNewServerConfig(port int, path string) serverConfig { 62 | if port < 0 || port > math.MaxUint16 { 63 | log.Fatal("Invalid port number") 64 | } 65 | return serverConfig{ 66 | port: port, 67 | path: path, 68 | } 69 | } 70 | 71 | func startMetricsServer(wg *sync.WaitGroup, shutdown chan struct{}, cfg serverConfig) { 72 | go func() { 73 | wg.Add(1) 74 | defer wg.Done() 75 | 76 | mux := http.NewServeMux() 77 | mux.Handle(cfg.path, promhttp.Handler()) 78 | srv := &http.Server{ 79 | Addr: fmt.Sprintf(":%d", cfg.port), 80 | Handler: mux, 81 | } 82 | go func() { 83 | log.WithField("port", cfg.port). 84 | WithField("path", cfg.path). 85 | Info("Starting metrics HTTP server") 86 | srv.ListenAndServe() 87 | }() 88 | 89 | <-shutdown 90 | log.Info("Shutting down metrics HTTP server") 91 | ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 92 | defer cancel() 93 | srv.Shutdown(ctx) 94 | }() 95 | } 96 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "os" 6 | "os/signal" 7 | "strings" 8 | "sync" 9 | "syscall" 10 | "time" 11 | 12 | "github.com/Shopify/sarama" 13 | log "github.com/Sirupsen/logrus" 14 | ) 15 | 16 | func main() { 17 | brokerString := flag.String("brokers", "", "Kafka brokers to connect to, comma-separated") 18 | topics := flag.String("topics", "", "Only fetch offsets for topics matching this regex (default all)") 19 | groups := flag.String("groups", "", "Also fetch offsets for consumer groups matching this regex (default none)") 20 | port := flag.Int("port", 9000, "Port to export metrics on") 21 | path := flag.String("path", "/", "Path to export metrics on") 22 | refresh := flag.Duration("refresh", 1*time.Minute, "Time between refreshing cluster metadata") 23 | fetchMin := flag.Duration("fetchMin", 15*time.Second, "Min time before requesting updates from broker") 24 | fetchMax := flag.Duration("fetchMax", 40*time.Second, "Max time before requesting updates from broker") 25 | level := flag.String("level", "info", "Logger level") 26 | flag.Parse() 27 | 28 | mustSetupLogger(*level) 29 | serverConfig := mustNewServerConfig(*port, *path) 30 | scrapeConfig := mustNewScrapeConfig(*refresh, *fetchMin, *fetchMax, *topics, *groups) 31 | 32 | kafka := mustNewKafka(*brokerString) 33 | defer kafka.Close() 34 | 35 | enforceGracefulShutdown(func(wg *sync.WaitGroup, shutdown chan struct{}) { 36 | startKafkaScraper(wg, shutdown, kafka, scrapeConfig) 37 | startMetricsServer(wg, shutdown, serverConfig) 38 | }) 39 | } 40 | 41 | func enforceGracefulShutdown(f func(wg *sync.WaitGroup, shutdown chan struct{})) { 42 | wg := &sync.WaitGroup{} 43 | shutdown := make(chan struct{}) 44 | signals := make(chan os.Signal) 45 | signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP) 46 | go func() { 47 | <-signals 48 | close(shutdown) 49 | }() 50 | 51 | log.Info("Graceful shutdown enabled") 52 | f(wg, shutdown) 53 | 54 | <-shutdown 55 | wg.Wait() 56 | } 57 | 58 | func mustNewKafka(brokerString string) sarama.Client { 59 | brokers := strings.Split(brokerString, ",") 60 | for i := range brokers { 61 | brokers[i] = strings.TrimSpace(brokers[i]) 62 | if !strings.ContainsRune(brokers[i], ':') { 63 | brokers[i] += ":9092" 64 | } 65 | } 66 | log.WithField("brokers.bootstrap", brokers).Info("connecting to cluster with bootstrap hosts") 67 | 68 | cfg := sarama.NewConfig() 69 | cfg.Version = sarama.V1_0_0_0 70 | client, err := sarama.NewClient(brokers, cfg) 71 | if err != nil { 72 | log.Fatal(err) 73 | } 74 | 75 | var addrs []string 76 | for _, b := range client.Brokers() { 77 | addrs = append(addrs, b.Addr()) 78 | } 79 | log.WithField("brokers", addrs).Info("connected to cluster") 80 | 81 | return client 82 | } 83 | 84 | func mustSetupLogger(level string) { 85 | logLevel, err := log.ParseLevel(level) 86 | if err != nil { 87 | log.Fatal(err) 88 | } 89 | 90 | log.SetLevel(logLevel) 91 | log.SetFormatter(&log.JSONFormatter{}) 92 | } 93 | -------------------------------------------------------------------------------- /vendor/vendor.json: -------------------------------------------------------------------------------- 1 | { 2 | "comment": "", 3 | "ignore": "test", 4 | "package": [ 5 | { 6 | "checksumSHA1": "Eq40BGwy01qYvfjJnZLImoGGJ7k=", 7 | "path": "github.com/Shopify/sarama", 8 | "revision": "d4ece5d05a86b2b64248f4943f872bea961ec835", 9 | "revisionTime": "2017-04-04T10:29:40Z" 10 | }, 11 | { 12 | "checksumSHA1": "ZKxETlJdB2XubMrZnXB0FQimVA8=", 13 | "path": "github.com/Sirupsen/logrus", 14 | "revision": "10f801ebc38b33738c9d17d50860f484a0988ff5", 15 | "revisionTime": "2017-03-17T14:32:14Z" 16 | }, 17 | { 18 | "checksumSHA1": "spyv5/YFBjYyZLZa1U2LBfDR8PM=", 19 | "path": "github.com/beorn7/perks/quantile", 20 | "revision": "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9", 21 | "revisionTime": "2016-08-04T10:47:26Z" 22 | }, 23 | { 24 | "checksumSHA1": "dvabztWVQX8f6oMLRyv4dLH+TGY=", 25 | "path": "github.com/davecgh/go-spew/spew", 26 | "revision": "346938d642f2ec3594ed81d874461961cd0faa76", 27 | "revisionTime": "2016-10-29T20:57:26Z" 28 | }, 29 | { 30 | "checksumSHA1": "y2Kh4iPlgCPXSGTCcFpzePYdzzg=", 31 | "path": "github.com/eapache/go-resiliency/breaker", 32 | "revision": "b86b1ec0dd4209a588dc1285cdd471e73525c0b3", 33 | "revisionTime": "2016-01-04T19:15:39Z" 34 | }, 35 | { 36 | "checksumSHA1": "WHl96RVZlOOdF4Lb1OOadMpw8ls=", 37 | "path": "github.com/eapache/go-xerial-snappy", 38 | "revision": "bb955e01b9346ac19dc29eb16586c90ded99a98c", 39 | "revisionTime": "2016-06-09T14:24:08Z" 40 | }, 41 | { 42 | "checksumSHA1": "oCCs6kDanizatplM5e/hX76busE=", 43 | "path": "github.com/eapache/queue", 44 | "revision": "44cc805cf13205b55f69e14bcb69867d1ae92f98", 45 | "revisionTime": "2016-08-05T00:47:13Z" 46 | }, 47 | { 48 | "checksumSHA1": "kBeNcaKk56FguvPSUCEaH6AxpRc=", 49 | "path": "github.com/golang/protobuf/proto", 50 | "revision": "2bba0603135d7d7f5cb73b2125beeda19c09f4ef", 51 | "revisionTime": "2017-03-31T03:19:02Z" 52 | }, 53 | { 54 | "checksumSHA1": "p/8vSviYF91gFflhrt5vkyksroo=", 55 | "path": "github.com/golang/snappy", 56 | "revision": "553a641470496b2327abcac10b36396bd98e45c9", 57 | "revisionTime": "2017-02-15T23:32:05Z" 58 | }, 59 | { 60 | "checksumSHA1": "BM6ZlNJmtKy3GBoWwg2X55gnZ4A=", 61 | "path": "github.com/klauspost/crc32", 62 | "revision": "1bab8b35b6bb565f92cbc97939610af9369f942a", 63 | "revisionTime": "2017-02-10T14:05:23Z" 64 | }, 65 | { 66 | "checksumSHA1": "bKMZjd2wPw13VwoE7mBeSv5djFA=", 67 | "path": "github.com/matttproud/golang_protobuf_extensions/pbutil", 68 | "revision": "c12348ce28de40eed0136aa2b644d0ee0650e56c", 69 | "revisionTime": "2016-04-24T11:30:07Z" 70 | }, 71 | { 72 | "checksumSHA1": "WmrPO1ovmQ7t7hs9yZGbr2SAoM4=", 73 | "path": "github.com/pierrec/lz4", 74 | "revision": "f5b77fd73d83122495309c0f459b810f83cc291f", 75 | "revisionTime": "2017-03-31T16:49:28Z" 76 | }, 77 | { 78 | "checksumSHA1": "IT4sX58d+e8osXHV5U6YCSdB/uE=", 79 | "path": "github.com/pierrec/xxHash/xxHash32", 80 | "revision": "5a004441f897722c627870a981d02b29924215fa", 81 | "revisionTime": "2016-01-12T16:53:51Z" 82 | }, 83 | { 84 | "checksumSHA1": "d2irkxoHgazkTuLIvJGiYwagl8o=", 85 | "path": "github.com/prometheus/client_golang/prometheus", 86 | "revision": "08fd2e12372a66e68e30523c7642e0cbc3e4fbde", 87 | "revisionTime": "2017-04-01T10:34:46Z" 88 | }, 89 | { 90 | "checksumSHA1": "lG3//eDlwqA4IOuAPrNtLh9G0TA=", 91 | "path": "github.com/prometheus/client_golang/prometheus/promhttp", 92 | "revision": "08fd2e12372a66e68e30523c7642e0cbc3e4fbde", 93 | "revisionTime": "2017-04-01T10:34:46Z" 94 | }, 95 | { 96 | "checksumSHA1": "DvwvOlPNAgRntBzt3b3OSRMS2N4=", 97 | "path": "github.com/prometheus/client_model/go", 98 | "revision": "6f3806018612930941127f2a7c6c453ba2c527d2", 99 | "revisionTime": "2017-02-16T18:52:47Z" 100 | }, 101 | { 102 | "checksumSHA1": "Wtpzndm/+bdwwNU5PCTfb4oUhc8=", 103 | "path": "github.com/prometheus/common/expfmt", 104 | "revision": "49fee292b27bfff7f354ee0f64e1bc4850462edf", 105 | "revisionTime": "2017-02-20T10:38:46Z" 106 | }, 107 | { 108 | "checksumSHA1": "GWlM3d2vPYyNATtTFgftS10/A9w=", 109 | "path": "github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg", 110 | "revision": "49fee292b27bfff7f354ee0f64e1bc4850462edf", 111 | "revisionTime": "2017-02-20T10:38:46Z" 112 | }, 113 | { 114 | "checksumSHA1": "0LL9u9tfv1KPBjNEiMDP6q7lpog=", 115 | "path": "github.com/prometheus/common/model", 116 | "revision": "49fee292b27bfff7f354ee0f64e1bc4850462edf", 117 | "revisionTime": "2017-02-20T10:38:46Z" 118 | }, 119 | { 120 | "checksumSHA1": "cD4xn1qxbkiuXqUExpdnDroCTrY=", 121 | "path": "github.com/prometheus/procfs", 122 | "revision": "a1dba9ce8baed984a2495b658c82687f8157b98f", 123 | "revisionTime": "2017-02-16T22:32:56Z" 124 | }, 125 | { 126 | "checksumSHA1": "kOWRcAHWFkId0aCIOSOyjzC0Zfc=", 127 | "path": "github.com/prometheus/procfs/xfs", 128 | "revision": "a1dba9ce8baed984a2495b658c82687f8157b98f", 129 | "revisionTime": "2017-02-16T22:32:56Z" 130 | }, 131 | { 132 | "checksumSHA1": "KAzbLjI9MzW2tjfcAsK75lVRp6I=", 133 | "path": "github.com/rcrowley/go-metrics", 134 | "revision": "1f30fe9094a513ce4c700b9a54458bbb0c96996c", 135 | "revisionTime": "2016-11-28T21:05:44Z" 136 | }, 137 | { 138 | "checksumSHA1": "ArDa4bMPKzhiS1I7iioemBwZ6tE=", 139 | "path": "golang.org/x/sys/unix", 140 | "revision": "7de4796419dc3b554e6dab3a119a62469569d299", 141 | "revisionTime": "2017-04-06T16:25:34Z" 142 | } 143 | ], 144 | "rootPath": "github.com/echojc/kafka-exporter" 145 | } 146 | -------------------------------------------------------------------------------- /kafka.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "math/rand" 6 | "regexp" 7 | "strconv" 8 | "sync" 9 | "time" 10 | 11 | "github.com/Shopify/sarama" 12 | log "github.com/Sirupsen/logrus" 13 | "github.com/prometheus/client_golang/prometheus" 14 | ) 15 | 16 | func init() { 17 | rand.Seed(time.Now().UnixNano()) 18 | } 19 | 20 | type scrapeConfig struct { 21 | FetchMinInterval time.Duration 22 | FetchMaxInterval time.Duration 23 | MetadataRefreshInterval time.Duration 24 | TopicsFilter *regexp.Regexp 25 | GroupsFilter *regexp.Regexp 26 | } 27 | 28 | func mustNewScrapeConfig(refresh time.Duration, fetchMin time.Duration, fetchMax time.Duration, topics string, groups string) scrapeConfig { 29 | topicsFilter, err := regexp.Compile(topics) 30 | if err != nil { 31 | log.Fatal(err) 32 | } 33 | 34 | // empty group should match nothing instead of everything 35 | if groups == "" { 36 | groups = ".^" 37 | } 38 | groupsFilter, err := regexp.Compile(groups) 39 | if err != nil { 40 | log.Fatal(err) 41 | } 42 | 43 | return scrapeConfig{ 44 | FetchMinInterval: fetchMin, 45 | FetchMaxInterval: fetchMax, 46 | MetadataRefreshInterval: refresh, 47 | TopicsFilter: topicsFilter, 48 | GroupsFilter: groupsFilter, 49 | } 50 | } 51 | 52 | func startKafkaScraper(wg *sync.WaitGroup, shutdown chan struct{}, kafka sarama.Client, cfg scrapeConfig) { 53 | go refreshMetadataPeriodically(wg, shutdown, kafka, cfg) 54 | for _, broker := range kafka.Brokers() { 55 | go manageBroker(wg, shutdown, broker, kafka, cfg) 56 | } 57 | } 58 | 59 | func refreshMetadataPeriodically(wg *sync.WaitGroup, shutdown chan struct{}, kafka sarama.Client, cfg scrapeConfig) { 60 | wg.Add(1) 61 | defer wg.Done() 62 | 63 | log.WithField("interval", cfg.MetadataRefreshInterval.String()). 64 | Info("Starting metadata refresh thread") 65 | 66 | wait := time.After(0) 67 | for { 68 | select { 69 | case <-wait: 70 | log.Debug("Refreshing cluster metadata") 71 | if err := kafka.RefreshMetadata(); err != nil { 72 | log.WithField("error", err).Warn("Failed to update cluster metadata") 73 | } 74 | case <-shutdown: 75 | log.Info("Shutting down metadata refresh thread") 76 | return 77 | } 78 | wait = time.After(cfg.MetadataRefreshInterval) 79 | } 80 | } 81 | 82 | func manageBroker(wg *sync.WaitGroup, shutdown chan struct{}, broker *sarama.Broker, kafka sarama.Client, cfg scrapeConfig) { 83 | wg.Add(1) 84 | defer wg.Done() 85 | 86 | log.WithField("broker", broker.Addr()). 87 | WithField("interval.min", cfg.FetchMinInterval.String()). 88 | WithField("interval.max", cfg.FetchMaxInterval.String()). 89 | Info("Starting handler for broker") 90 | 91 | wait := time.After(0) 92 | for { 93 | select { 94 | case <-wait: 95 | log.WithField("broker", broker.Addr()).Debug("Updating metrics") 96 | 97 | // ensure broker is connected 98 | if err := connect(broker); err != nil { 99 | log.WithField("broker", broker.Addr()). 100 | WithField("error", err). 101 | Error("Failed to connect to broker") 102 | break 103 | } 104 | 105 | // fetch groups coordinated by this broker 106 | var groups []string 107 | groupsResponse, err := broker.ListGroups(&sarama.ListGroupsRequest{}) 108 | if err != nil { 109 | log.WithField("broker", broker.Addr()). 110 | WithField("error", err). 111 | Error("Failed to retrieve consumer groups") 112 | } else if groupsResponse.Err != sarama.ErrNoError { 113 | log.WithField("broker", broker.Addr()). 114 | WithField("error", groupsResponse.Err). 115 | Error("Failed to retrieve consumer groups") 116 | } else { 117 | for group := range groupsResponse.Groups { 118 | if !cfg.GroupsFilter.MatchString(group) { 119 | log.WithField("group", group).Info("not found group") 120 | continue 121 | } 122 | 123 | groupCoordinator, err := kafka.Coordinator(group) 124 | if err != nil { 125 | log.WithField("broker", broker.Addr()). 126 | WithField("group", group). 127 | WithField("error", err). 128 | Warn("Failed to identify broker for consumer group") 129 | continue 130 | } 131 | 132 | if broker == groupCoordinator { 133 | groups = append(groups, group) 134 | } 135 | } 136 | } 137 | if len(groups) == 0 { 138 | log.WithField("broker", broker.Addr()).Debug("No consumer groups to fetch offsets for") 139 | } 140 | 141 | // build requests 142 | partitionCount := 0 143 | oldestRequest := sarama.OffsetRequest{} 144 | newestRequest := sarama.OffsetRequest{} 145 | var groupRequests []sarama.OffsetFetchRequest 146 | for _, group := range groups { 147 | groupRequests = append(groupRequests, sarama.OffsetFetchRequest{ConsumerGroup: group, Version: 1}) 148 | } 149 | 150 | // fetch partitions led by this broker, and add all partitions to group requests 151 | topics, err := kafka.Topics() 152 | if err != nil { 153 | log.WithField("broker", broker.Addr()). 154 | WithField("error", err). 155 | Error("Failed to get topics") 156 | break 157 | } 158 | 159 | for _, topic := range topics { 160 | if !cfg.TopicsFilter.MatchString(topic) { 161 | continue 162 | } 163 | 164 | partitions, err := kafka.Partitions(topic) 165 | if err != nil { 166 | log.WithField("broker", broker.Addr()). 167 | WithField("topic", topic). 168 | WithField("error", err). 169 | Warn("Failed to get partitions for topic") 170 | continue 171 | } 172 | 173 | for _, partition := range partitions { 174 | // all partitions need to be added to group requests 175 | for i := range groupRequests { 176 | groupRequests[i].AddPartition(topic, partition) 177 | } 178 | 179 | partitionLeader, err := kafka.Leader(topic, partition) 180 | if err != nil { 181 | log.WithField("broker", broker.Addr()). 182 | WithField("topic", topic). 183 | WithField("partition", partition). 184 | WithField("error", err). 185 | Warn("Failed to identify broker for partition") 186 | continue 187 | } 188 | 189 | if broker == partitionLeader { 190 | oldestRequest.AddBlock(topic, partition, sarama.OffsetOldest, 1) 191 | newestRequest.AddBlock(topic, partition, sarama.OffsetNewest, 1) 192 | partitionCount++ 193 | } 194 | } 195 | } 196 | 197 | if partitionCount == 0 { 198 | log.WithField("broker", broker.Addr()).Debug("No partitions for broker to fetch") 199 | } 200 | 201 | log.WithField("broker", broker.Addr()). 202 | WithField("partition.count", partitionCount). 203 | WithField("group.count", len(groupRequests)). 204 | Debug("Sending requests") 205 | 206 | requestWG := &sync.WaitGroup{} 207 | requestWG.Add(2 + len(groupRequests)) 208 | go func() { 209 | defer requestWG.Done() 210 | handleTopicOffsetRequest(broker, &oldestRequest, "oldest", metricOffsetOldest) 211 | }() 212 | go func() { 213 | defer requestWG.Done() 214 | handleTopicOffsetRequest(broker, &newestRequest, "newest", metricOffsetNewest) 215 | }() 216 | for i := range groupRequests { 217 | go func(request *sarama.OffsetFetchRequest) { 218 | defer requestWG.Done() 219 | handleGroupOffsetRequest(broker, request, metricOffsetConsumer) 220 | }(&groupRequests[i]) 221 | } 222 | requestWG.Wait() 223 | 224 | case <-shutdown: 225 | log.WithField("broker", broker.Addr()).Info("Shutting down handler for broker") 226 | return 227 | } 228 | 229 | min := int64(cfg.FetchMinInterval) 230 | max := int64(cfg.FetchMaxInterval) 231 | duration := time.Duration(rand.Int63n(max-min) + min) 232 | 233 | wait = time.After(duration) 234 | log.WithField("broker", broker.Addr()). 235 | WithField("interval.rand", duration.String()). 236 | Debug("Updated metrics and waiting for next run") 237 | } 238 | } 239 | 240 | func connect(broker *sarama.Broker) error { 241 | if ok, _ := broker.Connected(); ok { 242 | return nil 243 | } 244 | 245 | cfg := sarama.NewConfig() 246 | cfg.Version = sarama.V0_10_0_0 247 | if err := broker.Open(cfg); err != nil { 248 | return err 249 | } 250 | 251 | if connected, err := broker.Connected(); err != nil { 252 | return err 253 | } else if !connected { 254 | return errors.New("Unknown failure") 255 | } 256 | 257 | return nil 258 | } 259 | 260 | func handleGroupOffsetRequest(broker *sarama.Broker, request *sarama.OffsetFetchRequest, metric *prometheus.GaugeVec) { 261 | response, err := broker.FetchOffset(request) 262 | if err != nil { 263 | log.WithField("broker", broker.Addr()). 264 | WithField("group", request.ConsumerGroup). 265 | WithField("error", err). 266 | Error("Failed to request group offsets") 267 | return 268 | } 269 | 270 | for topic, partitions := range response.Blocks { 271 | for partition, block := range partitions { 272 | if block == nil { 273 | log.WithField("broker", broker.Addr()). 274 | WithField("group", request.ConsumerGroup). 275 | WithField("topic", topic). 276 | WithField("partition", partition). 277 | Warn("Failed to get data for group") 278 | continue 279 | } else if block.Err != sarama.ErrNoError { 280 | log.WithField("broker", broker.Addr()). 281 | WithField("group", request.ConsumerGroup). 282 | WithField("topic", topic). 283 | WithField("partition", partition). 284 | WithField("error", block.Err). 285 | Warn("Failed getting data for group") 286 | continue 287 | } else if block.Offset < 0 { 288 | continue 289 | } 290 | 291 | metric.With(prometheus.Labels{ 292 | "topic": topic, 293 | "partition": strconv.Itoa(int(partition)), 294 | "group": request.ConsumerGroup, 295 | }).Set(float64(block.Offset)) 296 | } 297 | } 298 | } 299 | 300 | func handleTopicOffsetRequest(broker *sarama.Broker, request *sarama.OffsetRequest, offsetType string, metric *prometheus.GaugeVec) { 301 | response, err := broker.GetAvailableOffsets(request) 302 | if err != nil { 303 | log.WithField("broker", broker.Addr()). 304 | WithField("offset.type", offsetType). 305 | WithField("error", err). 306 | Error("Failed to request topic offsets") 307 | return 308 | } 309 | 310 | for topic, partitions := range response.Blocks { 311 | for partition, block := range partitions { 312 | if block == nil { 313 | log.WithField("broker", broker.Addr()). 314 | WithField("topic", topic). 315 | WithField("partition", partition). 316 | Warn("Failed to get data for partition") 317 | continue 318 | } else if block.Err != sarama.ErrNoError { 319 | log.WithField("broker", broker.Addr()). 320 | WithField("topic", topic). 321 | WithField("partition", partition). 322 | WithField("error", block.Err). 323 | Warn("Failed getting offsets for partition") 324 | continue 325 | } else if len(block.Offsets) != 1 { 326 | log.WithField("broker", broker.Addr()). 327 | WithField("topic", topic). 328 | WithField("partition", partition). 329 | Warn("Got unexpected offset data for partition") 330 | continue 331 | } 332 | 333 | metric.With(prometheus.Labels{ 334 | "topic": topic, 335 | "partition": strconv.Itoa(int(partition)), 336 | }).Set(float64(block.Offsets[0])) 337 | } 338 | } 339 | } 340 | --------------------------------------------------------------------------------