├── .gitignore ├── LICENSE ├── avro-serialization ├── README.md ├── pom.xml └── src │ ├── main │ ├── java │ │ └── wrapper │ │ │ ├── Receiver.java │ │ │ └── Sender.java │ └── resources │ │ └── avro │ │ ├── MessageToSend.avsc │ │ ├── MessageToSendV2.avsc │ │ ├── Metric.avsc │ │ └── MetricV2.avsc │ └── test │ ├── java │ ├── schema │ │ ├── AvroSchemaTest.java │ │ └── PlainAvroSchemaEvolutionTest.java │ └── wrapper │ │ ├── AvroGenericSenderTest.java │ │ └── AvroSpecificSenderTest.java │ └── resources │ └── log4j.properties ├── metrics ├── README.md ├── connect │ ├── KafkaConnectMetrics.json │ └── connect.json ├── consumer │ ├── ApacheKafkaConsumerMetrics.json │ └── consumer.json └── kafka │ ├── ApacheKafkaBrokerMetrics.json │ └── kafka.json └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | /.idea 3 | /.idea_modules 4 | /.classpath 5 | /.project 6 | /.settings 7 | *.iml 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 kijanowski 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /avro-serialization/README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | This module is about using the Schema Registry and the corresponding Avro serializer and deserializer to exchange messages through Kafka. 3 | 4 | It is based on Confluent's examples which can be found here: 5 | https://github.com/confluentinc/examples 6 | 7 | Also worth to watch an introduction to Confluent's Schema Registry to get a basic idea what it is about: 8 | https://vimeo.com/167028700 9 | 10 | The corresponding blog post is available at: 11 | https://softwaremill.com/dog-ate-my-schema 12 | 13 | # Prerequisites 14 | The Avro "specific" example relies on a class (Metric.class), which is auto-generated based on *src/main/resources/avro/Metric.avsc*. 15 | Tis is done by the avro maven plugin and can be achieved running: 16 | 17 | ```$ mvn clean package -DskipTests``` 18 | 19 | Zookeeper, Kafka and the Schema Registry have to be running: 20 | 21 | ``` 22 | $ ./bin/zookeeper-server-start ./etc/kafka/zookeeper.properties & 23 | 24 | $ ./bin/kafka-server-start ./etc/kafka/server.properties & 25 | 26 | $ ./bin/schema-registry-start ./etc/schema-registry/schema-registry.properties & 27 | ``` 28 | # Avro generic approach 29 | The AvroGenericSenderTest evaluates a generic approach, where the schema has to be provided with every request. Sending a message with an incompatible schema ends with a SerializationException and the invalid message never reaches the Kafka topic. 30 | 31 | # Avro specific approach 32 | In this approach, a schema is provided in form of an Avro schema file, which is translated into a class file, which then can be used to create objects, either via constructors or builders. 33 | 34 | # Conclusions and miscellaneous notes 35 | Using the Schema Registry is in that sense transparent, that only the Avro serializers/deserializers have to be put into the Consumer's / Producer's config properties as well as the URL to the Schema Registry. 36 | 37 | There is development overhead related to providing the schema, no matter if using the generic nor specific approach. A price to pay for "schema-aware" messages. 38 | 39 | There should be a performance decrease since each send request requires the schema to be verified for compatibility. It has been promised to keep it low due to caching on both, the serializer level and Schema Registry level. 40 | 41 | By default schema events (register, update) are persisted in a Kafka topic called _schemas. 42 | 43 | Confluent ships a command line tool that uses the Schema Registry to inspect messages 44 | 45 | ```$ ./bin/kafka-avro-console-consumer --from-beginning --zookeeper 127.0.0.1:2181 --topic --property schema.registry.url=http://127.0.0.1:8081``` 46 | 47 | A schema is tied to a Kafka topic. Since a Kafka message comes in form of a Key-Value pair, a schema has a suffix, '-key' or '-value' respectively for the Key part and Value part of the message. 48 | 49 | The Schema Registry exposes a REST interface, which can be used to evaluate the schema 50 | 51 | *Get all schema names (subjects)* 52 | 53 | ```curl -X GET -i http://localhost:8081/subjects``` 54 | 55 | *Retrieve all version of a particular schema* 56 | 57 | ```$ curl -X GET -i http://localhost:8081/subjects//versions``` 58 | 59 | *Retrieve a particular version of a given schema* 60 | ```$ curl -X GET -i http://localhost:8081/subjects//versions/1``` 61 | ```$ curl -X GET -i http://localhost:8081/subjects//versions/latest``` 62 | 63 | 64 | ## Compatibility Level 65 | To check the global compatibility level, which by default is **BACKWARD**, run: 66 | 67 | ```$ curl -X GET -i http://localhost:8081/config/``` 68 | 69 | The compatibility level can be set on a per schema basis: 70 | 71 | ```$ curl -X PUT -i -H "Content-Type: application/vnd.schemaregistry.v1+json" --data '{"compatibility": "backward"}' http://localhost:8081/config/``` 72 | 73 | And listed: 74 | 75 | ```$ curl -X GET -i http://localhost:8081/config/``` 76 | -------------------------------------------------------------------------------- /avro-serialization/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | confluentio 7 | keyar 8 | 1.0-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | avro-serialization 13 | 14 | 15 | 16 | org.apache.avro 17 | avro 18 | ${avro.version} 19 | 20 | 21 | org.apache.avro 22 | avro-maven-plugin 23 | ${avro.version} 24 | 25 | 26 | io.confluent 27 | kafka-avro-serializer 28 | ${confluent.version} 29 | 30 | 31 | org.apache.kafka 32 | kafka-clients 33 | ${kafka.version} 34 | 35 | 36 | 37 | 38 | io.confluent 39 | common-config 40 | ${confluent.version} 41 | 42 | 43 | io.confluent 44 | kafka-schema-registry-client 45 | ${confluent.version} 46 | 47 | 48 | io.confluent 49 | common-utils 50 | ${confluent.version} 51 | 52 | 53 | com.fasterxml.jackson.core 54 | jackson-core 55 | 2.5.4 56 | 57 | 58 | com.fasterxml.jackson.core 59 | jackson-databind 60 | 2.5.4 61 | 62 | 63 | 64 | 65 | org.slf4j 66 | slf4j-api 67 | 1.7.12 68 | 69 | 70 | org.slf4j 71 | slf4j-log4j12 72 | 1.7.12 73 | 74 | 75 | log4j 76 | log4j 77 | 78 | 79 | 80 | 81 | junit 82 | junit 83 | 4.12 84 | test 85 | 86 | 87 | org.assertj 88 | assertj-core 89 | 1.2.0 90 | test 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | org.apache.avro 99 | avro-maven-plugin 100 | ${avro.version} 101 | 102 | 103 | generate-sources 104 | 105 | schema 106 | protocol 107 | idl-protocol 108 | 109 | 110 | ${project.basedir}/src/main/resources/avro 111 | 112 | 113 | 114 | 115 | 116 | org.apache.maven.plugins 117 | maven-compiler-plugin 118 | 119 | 1.8 120 | 1.8 121 | 122 | 123 | 124 | org.apache.maven.plugins 125 | maven-shade-plugin 126 | 127 | 128 | package 129 | 130 | shade 131 | 132 | 133 | 134 | 135 | uber-${artifactId}-${version} 136 | 137 | 138 | 139 | 140 | 141 | 142 | -------------------------------------------------------------------------------- /avro-serialization/src/main/java/wrapper/Receiver.java: -------------------------------------------------------------------------------- 1 | package wrapper; 2 | 3 | import org.apache.kafka.clients.consumer.Consumer; 4 | import org.apache.kafka.clients.consumer.ConsumerConfig; 5 | import org.apache.kafka.clients.consumer.ConsumerRecord; 6 | import org.apache.kafka.clients.consumer.ConsumerRecords; 7 | import org.apache.kafka.clients.consumer.KafkaConsumer; 8 | import org.apache.log4j.Logger; 9 | 10 | import java.util.ArrayList; 11 | import java.util.Arrays; 12 | import java.util.List; 13 | import java.util.Properties; 14 | import java.util.UUID; 15 | 16 | public class Receiver { 17 | 18 | private static final Logger LOG = Logger.getLogger(Receiver.class); 19 | 20 | private final Consumer consumer; 21 | 22 | private final String topic; 23 | 24 | public Receiver(String topic, boolean specificAvroReader) { 25 | 26 | this.topic = topic; 27 | 28 | Properties properties = new Properties(); 29 | 30 | properties.put("bootstrap.servers", "localhost:9092"); 31 | properties.put(ConsumerConfig.GROUP_ID_CONFIG, UUID.randomUUID().toString()); 32 | properties.put(ConsumerConfig.CLIENT_ID_CONFIG, "client_id"); 33 | properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); 34 | properties.put("enable.auto.commit", "false"); 35 | properties.put("auto.commit.interval.ms", "1000"); 36 | properties.put("session.timeout.ms", "30000"); 37 | properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, 38 | io.confluent.kafka.serializers.KafkaAvroDeserializer.class); 39 | properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, 40 | io.confluent.kafka.serializers.KafkaAvroDeserializer.class); 41 | properties.put("schema.registry.url", "http://localhost:8081"); 42 | properties.put("specific.avro.reader", String.valueOf(specificAvroReader)); 43 | 44 | this.consumer = new KafkaConsumer<>(properties); 45 | consumer.subscribe(Arrays.asList(topic)); 46 | LOG.info("Created Kafka Consumer for topic " + topic); 47 | 48 | } 49 | 50 | public List receive() { 51 | 52 | List buffer = new ArrayList<>(); 53 | 54 | ConsumerRecords records; 55 | do { 56 | records = consumer.poll(100); 57 | if (records.count() > 0) { 58 | for (ConsumerRecord record : records) { 59 | buffer.add(record.value()); 60 | } 61 | consumer.commitSync(); 62 | consumer.close(); 63 | LOG.info("Consumer closed successfully"); 64 | } 65 | } while (records.count() == 0); 66 | return buffer; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /avro-serialization/src/main/java/wrapper/Sender.java: -------------------------------------------------------------------------------- 1 | package wrapper; 2 | 3 | import org.apache.kafka.clients.producer.KafkaProducer; 4 | import org.apache.kafka.clients.producer.Producer; 5 | import org.apache.kafka.clients.producer.ProducerConfig; 6 | import org.apache.kafka.clients.producer.ProducerRecord; 7 | import org.apache.kafka.clients.producer.RecordMetadata; 8 | import org.apache.kafka.common.errors.SerializationException; 9 | import org.apache.log4j.Logger; 10 | 11 | import java.util.Properties; 12 | import java.util.concurrent.Future; 13 | 14 | public class Sender { 15 | 16 | private static final Logger LOG = Logger.getLogger(Sender.class); 17 | 18 | 19 | private final Producer producer; 20 | private final String topic; 21 | 22 | public Sender(String topic) { 23 | 24 | this.topic = topic; 25 | 26 | Properties properties = new Properties(); 27 | properties.put("bootstrap.servers", "localhost:9092"); 28 | properties.put("acks", "all"); 29 | properties.put("retries", 0); 30 | properties.put("batch.size", 16384); 31 | properties.put("linger.ms", 1); 32 | properties.put("buffer.memory", 33554432); 33 | 34 | properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, 35 | io.confluent.kafka.serializers.KafkaAvroSerializer.class); 36 | properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, 37 | io.confluent.kafka.serializers.KafkaAvroSerializer.class); 38 | properties.put("schema.registry.url", "http://localhost:8081"); 39 | 40 | this.producer = new KafkaProducer<>(properties); 41 | LOG.info("Created Kafka Producer for topic " + topic); 42 | 43 | } 44 | 45 | public Future send(V message) { 46 | 47 | ProducerRecord record = new ProducerRecord<>(topic, message); 48 | try { 49 | return producer.send(record); 50 | } catch(SerializationException se) { 51 | LOG.error("Couldn't send message:", se); 52 | throw se; 53 | } 54 | 55 | } 56 | 57 | public void close() { 58 | LOG.info("Closing Kafka Producer"); 59 | producer.close(); 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /avro-serialization/src/main/resources/avro/MessageToSend.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type":"record", 3 | "name":"MessageToSend", 4 | "namespace":"com.softwaremill.schema", 5 | "fields":[ 6 | { 7 | "name":"type", 8 | "type":"string" 9 | }, 10 | { 11 | "name":"correlationId", 12 | "type":"string" 13 | }, 14 | { 15 | "name":"payload", 16 | "type":[ 17 | { 18 | "type":"record", 19 | "name":"Sms", 20 | "fields":[ 21 | { 22 | "name":"phoneNumber", 23 | "type":"string" 24 | }, 25 | { 26 | "name":"text", 27 | "type":"string" 28 | } 29 | ] 30 | }, 31 | { 32 | "type":"record", 33 | "name":"Email", 34 | "fields":[ 35 | { 36 | "name":"addressTo", 37 | "type":"string" 38 | }, 39 | { 40 | "name":"title", 41 | "type":"string" 42 | }, 43 | { 44 | "name":"text", 45 | "type":"string" 46 | } 47 | ] 48 | }, 49 | { 50 | "type":"record", 51 | "name":"PushNotification", 52 | "fields":[ 53 | { 54 | "name":"arn", 55 | "type":"string" 56 | }, 57 | { 58 | "name":"text", 59 | "type":"string" 60 | } 61 | ] 62 | } 63 | ] 64 | } 65 | ] 66 | } -------------------------------------------------------------------------------- /avro-serialization/src/main/resources/avro/MessageToSendV2.avsc: -------------------------------------------------------------------------------- 1 | { 2 | "type":"record", 3 | "name":"MessageToSendV2", 4 | "namespace":"com.softwaremill.schema", 5 | "fields":[ 6 | { 7 | "name":"type", 8 | "type":"string" 9 | }, 10 | { 11 | "name":"correlationId", 12 | "type":"string" 13 | }, 14 | { 15 | "name":"payload", 16 | "type":[ 17 | { 18 | "type":"record", 19 | "name":"Sms", 20 | "fields":[ 21 | { 22 | "name":"phoneNumber", 23 | "type":"string" 24 | }, 25 | { 26 | "name":"text", 27 | "type":"string" 28 | } 29 | ] 30 | }, 31 | { 32 | "type":"record", 33 | "name":"EmailV2", 34 | "aliases": ["Email"], 35 | "fields":[ 36 | { 37 | "name":"addressFrom", 38 | "type":"string", 39 | "default":"andrzej@test.pl" 40 | }, 41 | { 42 | "name":"addressTo", 43 | "type":"string" 44 | }, 45 | { 46 | "name":"title", 47 | "type":"string" 48 | }, 49 | { 50 | "name":"text", 51 | "type":"string" 52 | } 53 | ] 54 | }, 55 | { 56 | "type":"record", 57 | "name":"PushNotification", 58 | "fields":[ 59 | { 60 | "name":"arn", 61 | "type":"string" 62 | }, 63 | { 64 | "name":"text", 65 | "type":"string" 66 | } 67 | ] 68 | } 69 | ] 70 | } 71 | ] 72 | } -------------------------------------------------------------------------------- /avro-serialization/src/main/resources/avro/Metric.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "keyar.domain", 2 | "type": "record", 3 | "name": "Metric", 4 | "fields": [ 5 | {"name": "ip", "type": "string"}, 6 | {"name": "name", "type": "string"}, 7 | {"name": "value", "type": "float"} 8 | ] 9 | } -------------------------------------------------------------------------------- /avro-serialization/src/main/resources/avro/MetricV2.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "keyar.domain", 2 | "type": "record", 3 | "name": "MetricV2", 4 | "fields": [ 5 | {"name": "ip", "type": "string"}, 6 | {"name": "name", "type": "string"}, 7 | {"name": "value", "type": "float"}, 8 | {"name": "time", "type": "string", "default":"12345"} 9 | ] 10 | } -------------------------------------------------------------------------------- /avro-serialization/src/test/java/schema/AvroSchemaTest.java: -------------------------------------------------------------------------------- 1 | package schema; 2 | 3 | import org.apache.avro.Schema; 4 | import org.apache.avro.SchemaBuilder; 5 | import org.apache.log4j.Logger; 6 | import org.junit.Test; 7 | import wrapper.AvroGenericSenderTest; 8 | 9 | public class AvroSchemaTest { 10 | 11 | private static final Logger LOG = Logger.getLogger(AvroGenericSenderTest.class); 12 | 13 | @Test 14 | public void shouldCreateStandardSchema() { 15 | //given 16 | 17 | //when 18 | Schema schema = SchemaBuilder 19 | .record("HandshakeRequest").namespace("com.softwaremill.schema") 20 | .fields() 21 | .name("clientHash").type().fixed("MD5").size(16).noDefault() 22 | .name("clientProtocol").type().nullable().stringType().noDefault() 23 | .name("serverHash").type("MD5").noDefault() 24 | .name("meta").type().nullable().map().values().bytesType().noDefault() 25 | .endRecord(); 26 | 27 | //then 28 | LOG.info(schema.toString()); 29 | } 30 | 31 | @Test 32 | public void shouldCreateSchemaWithUnion() { 33 | //when 34 | Schema schema = SchemaBuilder 35 | .record("MessageToSend").namespace("com.softwaremill.schema") 36 | .fields() 37 | .name("text").type().stringType().noDefault() 38 | .name("correlationId").type().stringType().noDefault() 39 | .name("payload").type().unionOf() 40 | .record("Sms").namespace("com.softwaremill.schema") 41 | .fields() 42 | .name("phoneNumber").type().stringType().noDefault() 43 | .name("text").type().stringType().noDefault() 44 | .endRecord().and() 45 | .record("Email").namespace("com.softwaremill.schema") 46 | .fields() 47 | .name("addressTo").type().stringType().noDefault() 48 | .name("title").type().stringType().noDefault() 49 | .name("text").type().stringType().noDefault() 50 | .endRecord() 51 | .and() 52 | .record("PushNotification").namespace("com.softwaremill.schema") 53 | .fields() 54 | .name("arn").type().stringType().noDefault() 55 | .name("text").type().stringType().noDefault() 56 | .endRecord() 57 | .endUnion() 58 | .noDefault() 59 | .endRecord(); 60 | 61 | //then 62 | LOG.info(schema.toString()); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /avro-serialization/src/test/java/schema/PlainAvroSchemaEvolutionTest.java: -------------------------------------------------------------------------------- 1 | package schema; 2 | 3 | import com.softwaremill.schema.Email; 4 | import com.softwaremill.schema.EmailV2; 5 | import com.softwaremill.schema.MessageToSend; 6 | import com.softwaremill.schema.MessageToSendV2; 7 | import keyar.domain.Metric; 8 | import keyar.domain.MetricV2; 9 | import org.apache.avro.file.DataFileReader; 10 | import org.apache.avro.file.DataFileWriter; 11 | import org.apache.avro.io.DatumReader; 12 | import org.apache.avro.io.DatumWriter; 13 | import org.apache.avro.specific.SpecificDatumReader; 14 | import org.apache.avro.specific.SpecificDatumWriter; 15 | import org.junit.Test; 16 | 17 | import java.io.File; 18 | import java.io.IOException; 19 | 20 | import static org.assertj.core.api.Assertions.assertThat; 21 | 22 | public class PlainAvroSchemaEvolutionTest { 23 | 24 | @Test 25 | public void shouldDeserializeSimplePojoWithNewSchema() throws IOException { 26 | //given 27 | final File file = new File("target/metrics.avro"); 28 | Metric oldMetric = new Metric("Ip", "MyName", 1.23f); 29 | final String defaultTime = "12345"; 30 | 31 | DatumWriter metricWriter = new SpecificDatumWriter<>(Metric.class); 32 | DataFileWriter dataFileWriter = new DataFileWriter<>(metricWriter); 33 | dataFileWriter.create(oldMetric.getSchema(), file); 34 | dataFileWriter.append(oldMetric); 35 | dataFileWriter.close(); 36 | 37 | //when 38 | // Deserialize Users from disk 39 | DatumReader metricDatumReader = new SpecificDatumReader<>(MetricV2.class); 40 | DataFileReader dataFileReader = new DataFileReader<>(file, metricDatumReader); 41 | 42 | //then 43 | final MetricV2 metricV2 = dataFileReader.next(); 44 | assertThat(metricV2).isEqualTo(new MetricV2("Ip", "MyName", 1.23f, defaultTime)); 45 | } 46 | 47 | @Test 48 | public void shouldDeserializePojoWithUnionsAndNewSchema() throws IOException { 49 | //given 50 | final File file = new File("target/messageToSend.avro"); 51 | String defaultAddressFrom = "andrzej@test.pl"; 52 | Email email = new Email("addressTo", "title", "text"); 53 | MessageToSend messageToSend = new MessageToSend("type", "correlationId", email); 54 | 55 | DatumWriter writer = new SpecificDatumWriter<>(MessageToSend.class); 56 | DataFileWriter dataFileWriter = new DataFileWriter<>(writer); 57 | dataFileWriter.create(messageToSend.getSchema(), file); 58 | dataFileWriter.append(messageToSend); 59 | dataFileWriter.close(); 60 | 61 | //when 62 | DatumReader reader = new SpecificDatumReader<>(MessageToSendV2.class); 63 | DataFileReader dataFileReader = new DataFileReader<>(file, reader); 64 | 65 | //then 66 | final MessageToSendV2 messageToSendV2 = dataFileReader.next(); 67 | //in case of new pojo name in union - it is necessary to provide an alias 68 | final EmailV2 emailV2 = (EmailV2) messageToSendV2.getPayload(); 69 | assertThat(emailV2).isEqualTo(new EmailV2(defaultAddressFrom, "addressTo", "title", "text")); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /avro-serialization/src/test/java/wrapper/AvroGenericSenderTest.java: -------------------------------------------------------------------------------- 1 | package wrapper; 2 | 3 | import org.apache.avro.Schema; 4 | import org.apache.avro.generic.GenericData; 5 | import org.apache.avro.generic.GenericRecord; 6 | import org.apache.kafka.clients.producer.RecordMetadata; 7 | import org.apache.kafka.common.errors.SerializationException; 8 | import org.apache.log4j.Logger; 9 | import org.junit.Assert; 10 | import org.junit.Test; 11 | 12 | import java.util.List; 13 | import java.util.UUID; 14 | import java.util.concurrent.Future; 15 | 16 | public class AvroGenericSenderTest { 17 | 18 | private static final Logger LOG = Logger.getLogger(AvroGenericSenderTest.class); 19 | 20 | private static final String SIMPLE_SCHEMA = 21 | "{\"namespace\": \"keyar.domain\", \"type\": \"record\", " + 22 | "\"name\": \"generic_avro\"," + 23 | "\"fields\": [" + 24 | "{\"name\": \"name\", \"type\": \"string\"}," + 25 | "{\"name\": \"value\", \"type\": \"float\"}" + 26 | "]}"; 27 | 28 | private static final String INCOMPATIBLE_SCHEMA = "{\"namespace\": \"keyar.domain\", \"type\": \"record\", " + 29 | "\"name\": \"generic_avro\"," + 30 | "\"fields\": [" + 31 | "{\"name\": \"ip\", \"type\": \"string\"}," + // new incompatible field 32 | "{\"name\": \"name\", \"type\": \"string\"}," + 33 | "{\"name\": \"value\", \"type\": \"float\"}" + 34 | "]" + 35 | "}"; 36 | 37 | private static final String COMPATIBLE_SCHEMA = "{\"namespace\": \"keyar.domain\", \"type\": \"record\", " + 38 | "\"name\": \"generic_avro\"," + 39 | "\"fields\": [" + 40 | "{\"name\": \"ip\", \"type\": \"string\", \"default\": \"DEFAULT\"}," + // new compatible field 41 | "{\"name\": \"name\", \"type\": \"string\"}," + 42 | "{\"name\": \"value\", \"type\": \"float\"}" + 43 | "]" + 44 | "}"; 45 | 46 | @Test 47 | public void testAvroSerialization() throws Exception { 48 | 49 | // given 50 | String topic = UUID.randomUUID().toString(); 51 | Sender sender = new Sender<>(topic); 52 | 53 | Schema.Parser parser = new Schema.Parser(); 54 | Schema schema = parser.parse(SIMPLE_SCHEMA); 55 | 56 | // when 57 | GenericRecord metric = new GenericData.Record(schema); 58 | metric.put("name", "ABC"); 59 | metric.put("value", 123f); 60 | 61 | Future result = sender.send(metric); 62 | 63 | // then 64 | RecordMetadata record = (RecordMetadata) result.get(); 65 | Assert.assertEquals(topic, record.topic()); 66 | sender.close(); 67 | 68 | // receive 69 | Receiver receiver = new Receiver<>(topic, false); 70 | List results = receiver.receive(); 71 | results.forEach(LOG::info); 72 | 73 | Assert.assertEquals(1, results.size()); 74 | 75 | Assert.assertEquals("ABC", results.get(0).get("name").toString()); 76 | Assert.assertEquals(123f, results.get(0).get("value")); 77 | 78 | } 79 | 80 | /** 81 | * Should throw SerializationException, since the schema is not backward compatible 82 | * Adding a new field to the schema does not allow the reader to read messages persisted with older schemas, 83 | * if no default value is given 84 | */ 85 | @Test 86 | public void testAvroBackwardIncompatibleSerialization() throws Exception { 87 | 88 | 89 | // given 90 | String topic = UUID.randomUUID().toString(); 91 | Sender sender = new Sender<>(topic); 92 | 93 | Schema.Parser parser1 = new Schema.Parser(); 94 | Schema.Parser parser2 = new Schema.Parser(); 95 | Schema simpleSchema = parser1.parse(SIMPLE_SCHEMA); 96 | Schema incompatibleSchema = parser2.parse(INCOMPATIBLE_SCHEMA); 97 | 98 | // when 99 | GenericRecord metric1 = new GenericData.Record(simpleSchema); 100 | metric1.put("name", "ABC"); 101 | metric1.put("value", 123f); 102 | 103 | GenericRecord metric2 = new GenericData.Record(incompatibleSchema); 104 | metric2.put("ip", "IP"); 105 | metric2.put("name", "ABC"); 106 | metric2.put("value", 123f); 107 | 108 | sender.send(metric1); 109 | try { 110 | sender.send(metric2); 111 | Assert.fail("Should not send due to serialization exception"); 112 | } catch (SerializationException se) { 113 | // expected 114 | } 115 | } 116 | 117 | /** 118 | * Should not throw SerializationException, since the schema is backward compatible 119 | * Adding a new field to the schema does allow the reader to read messages persisted with older schemas, 120 | * if a default value is given 121 | */ 122 | @Test 123 | public void testAvroBackwardCompatibleSerialization() throws Exception { 124 | 125 | 126 | // given 127 | String topic = UUID.randomUUID().toString(); 128 | Sender sender = new Sender<>(topic); 129 | 130 | Schema.Parser parser1 = new Schema.Parser(); 131 | Schema.Parser parser2 = new Schema.Parser(); 132 | Schema simpleSchema = parser1.parse(SIMPLE_SCHEMA); 133 | Schema compatibleSchema = parser2.parse(COMPATIBLE_SCHEMA); 134 | 135 | // when 136 | GenericRecord metric1 = new GenericData.Record(simpleSchema); 137 | metric1.put("name", "ABC"); 138 | metric1.put("value", 123f); 139 | 140 | GenericRecord metric2 = new GenericData.Record(compatibleSchema); 141 | metric2.put("ip", "IP"); 142 | metric2.put("name", "ABC"); 143 | metric2.put("value", 123f); 144 | 145 | Future result1 = sender.send(metric1); 146 | Future result2 = sender.send(metric2); 147 | 148 | // then 149 | RecordMetadata record = (RecordMetadata) result1.get(); 150 | Assert.assertEquals(topic, record.topic()); 151 | record = (RecordMetadata) result2.get(); 152 | Assert.assertEquals(topic, record.topic()); 153 | 154 | // receive 155 | Receiver receiver = new Receiver<>(topic, false); 156 | List results = receiver.receive(); 157 | results.forEach(LOG::info); 158 | 159 | Assert.assertEquals(2, results.size()); 160 | 161 | Assert.assertEquals("ABC", results.get(0).get("name").toString()); 162 | Assert.assertEquals(123f, results.get(0).get("value")); 163 | 164 | Assert.assertEquals("IP", results.get(1).get("ip").toString()); 165 | Assert.assertEquals("ABC", results.get(1).get("name").toString()); 166 | Assert.assertEquals(123f, results.get(1).get("value")); 167 | } 168 | 169 | 170 | @Test 171 | public void testAvroSimpleSerialization() throws Exception { 172 | 173 | // given 174 | String topic = UUID.randomUUID().toString(); 175 | Sender sender = new Sender<>(topic); 176 | 177 | // when 178 | Future result = sender.send("Test Data"); 179 | 180 | // then 181 | RecordMetadata record = (RecordMetadata) result.get(); 182 | Assert.assertEquals(topic, record.topic()); 183 | 184 | // receive 185 | Receiver receiver = new Receiver<>(topic, false); 186 | List results = receiver.receive(); 187 | results.forEach(LOG::info); 188 | 189 | Assert.assertEquals(1, results.size()); 190 | 191 | Assert.assertEquals("Test Data", results.get(0).toString()); 192 | 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /avro-serialization/src/test/java/wrapper/AvroSpecificSenderTest.java: -------------------------------------------------------------------------------- 1 | package wrapper; 2 | 3 | import keyar.domain.Metric; 4 | import org.apache.avro.generic.GenericRecord; 5 | import org.apache.avro.specific.SpecificData; 6 | import org.apache.kafka.clients.producer.RecordMetadata; 7 | import org.apache.log4j.Logger; 8 | import org.junit.Assert; 9 | import org.junit.Test; 10 | 11 | import java.util.List; 12 | import java.util.UUID; 13 | import java.util.concurrent.Future; 14 | 15 | public class AvroSpecificSenderTest { 16 | 17 | private static final Logger LOG = Logger.getLogger(AvroSpecificSenderTest.class); 18 | 19 | @Test 20 | public void testAvroSerialization() throws Exception { 21 | 22 | // given 23 | 24 | // this is an autogenrated class, need to run maven clean package to get it 25 | // $ mvn clean package -DskipTests 26 | Metric m1 = new Metric(); 27 | m1.setIp("IP"); 28 | m1.setName("ABC"); 29 | m1.setValue(123f); 30 | 31 | Metric m2 = new Metric("IP", "ABC", 234f); 32 | 33 | Metric m3 = Metric.newBuilder().setIp("IP").setName("ABC").setValue(345f).build(); 34 | 35 | String topic = UUID.randomUUID().toString(); 36 | Sender sender = new Sender<>(topic); 37 | 38 | // when 39 | Future result1 = sender.send(m1); 40 | Future result2 = sender.send(m2); 41 | Future result3 = sender.send(m3); 42 | 43 | // then 44 | RecordMetadata record = (RecordMetadata) result1.get(); 45 | Assert.assertEquals(topic, record.topic()); 46 | record = (RecordMetadata) result2.get(); 47 | Assert.assertEquals(topic, record.topic()); 48 | record = (RecordMetadata) result3.get(); 49 | Assert.assertEquals(topic, record.topic()); 50 | 51 | Receiver receiver = new Receiver<>(topic, true); 52 | List results = receiver.receive(); 53 | results.forEach((result) -> { 54 | Metric metric = (Metric) SpecificData.get().deepCopy(Metric.SCHEMA$, result); 55 | LOG.info(metric); 56 | }); 57 | 58 | Assert.assertEquals(3, results.size()); 59 | 60 | Assert.assertEquals("IP", results.get(0).get("ip").toString()); 61 | Assert.assertEquals("ABC", results.get(0).get("name").toString()); 62 | Assert.assertEquals(123f, results.get(0).get("value")); 63 | 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /avro-serialization/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # The logging properties used 3 | # 4 | log4j.rootLogger=INFO, out 5 | 6 | log4j.appender.out=org.apache.log4j.ConsoleAppender 7 | log4j.appender.out.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.out.layout.ConversionPattern=[%d %c] %p %m%n 9 | log4j.appender.out.Threshold=INFO -------------------------------------------------------------------------------- /metrics/README.md: -------------------------------------------------------------------------------- 1 | Grafana Dashboards and jmxtrans config files for Apache Kafka. 2 | The corresponding blog post is available at https://softwaremill.com/monitoring-apache-kafka-with-influxdb-grafana 3 | 4 | -------------------------------------------------------------------------------- /metrics/connect/KafkaConnectMetrics.json: -------------------------------------------------------------------------------- 1 | { 2 | "__inputs": [ 3 | { 4 | "name": "DS_INFLUX", 5 | "label": "influx", 6 | "description": "", 7 | "type": "datasource", 8 | "pluginId": "influxdb", 9 | "pluginName": "InfluxDB" 10 | } 11 | ], 12 | "__requires": [ 13 | { 14 | "type": "panel", 15 | "id": "graph", 16 | "name": "Graph", 17 | "version": "" 18 | }, 19 | { 20 | "type": "grafana", 21 | "id": "grafana", 22 | "name": "Grafana", 23 | "version": "3.1.0" 24 | }, 25 | { 26 | "type": "datasource", 27 | "id": "influxdb", 28 | "name": "InfluxDB", 29 | "version": "1.0.0" 30 | } 31 | ], 32 | "id": null, 33 | "title": "Kafka Connect Metrics", 34 | "tags": [], 35 | "style": "dark", 36 | "timezone": "browser", 37 | "editable": true, 38 | "hideControls": false, 39 | "sharedCrosshair": false, 40 | "rows": [ 41 | { 42 | "collapse": false, 43 | "editable": true, 44 | "height": "250px", 45 | "panels": [ 46 | { 47 | "aliasColors": {}, 48 | "bars": false, 49 | "datasource": "${DS_INFLUX}", 50 | "editable": true, 51 | "error": false, 52 | "fill": 1, 53 | "grid": { 54 | "threshold1": null, 55 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 56 | "threshold2": null, 57 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 58 | }, 59 | "id": 1, 60 | "isNew": true, 61 | "legend": { 62 | "avg": false, 63 | "current": false, 64 | "max": false, 65 | "min": false, 66 | "show": true, 67 | "total": false, 68 | "values": false 69 | }, 70 | "lines": true, 71 | "linewidth": 2, 72 | "links": [], 73 | "nullPointMode": "connected", 74 | "percentage": false, 75 | "pointradius": 5, 76 | "points": false, 77 | "renderer": "flot", 78 | "seriesOverrides": [], 79 | "span": 12, 80 | "stack": false, 81 | "steppedLine": false, 82 | "targets": [ 83 | { 84 | "dsType": "influxdb", 85 | "groupBy": [ 86 | { 87 | "params": [ 88 | "$interval" 89 | ], 90 | "type": "time" 91 | }, 92 | { 93 | "params": [ 94 | "null" 95 | ], 96 | "type": "fill" 97 | } 98 | ], 99 | "measurement": "kafkaConnectMemory", 100 | "policy": "default", 101 | "refId": "A", 102 | "resultFormat": "time_series", 103 | "select": [ 104 | [ 105 | { 106 | "params": [ 107 | "used" 108 | ], 109 | "type": "field" 110 | }, 111 | { 112 | "params": [], 113 | "type": "last" 114 | }, 115 | { 116 | "params": [ 117 | "heap" 118 | ], 119 | "type": "alias" 120 | } 121 | ] 122 | ], 123 | "tags": [ 124 | { 125 | "key": "attributeName", 126 | "operator": "=", 127 | "value": "HeapMemoryUsage" 128 | }, 129 | { 130 | "condition": "AND", 131 | "key": "hostname", 132 | "operator": "=", 133 | "value": "kafka-connect" 134 | } 135 | ] 136 | }, 137 | { 138 | "dsType": "influxdb", 139 | "groupBy": [ 140 | { 141 | "params": [ 142 | "$interval" 143 | ], 144 | "type": "time" 145 | }, 146 | { 147 | "params": [ 148 | "null" 149 | ], 150 | "type": "fill" 151 | } 152 | ], 153 | "measurement": "kafkaConnectMemory", 154 | "policy": "default", 155 | "refId": "B", 156 | "resultFormat": "time_series", 157 | "select": [ 158 | [ 159 | { 160 | "params": [ 161 | "used" 162 | ], 163 | "type": "field" 164 | }, 165 | { 166 | "params": [], 167 | "type": "last" 168 | }, 169 | { 170 | "params": [ 171 | "non-heap" 172 | ], 173 | "type": "alias" 174 | } 175 | ] 176 | ], 177 | "tags": [ 178 | { 179 | "key": "attributeName", 180 | "operator": "=", 181 | "value": "NonHeapMemoryUsage" 182 | }, 183 | { 184 | "condition": "AND", 185 | "key": "hostname", 186 | "operator": "=", 187 | "value": "kafka-connect" 188 | } 189 | ] 190 | } 191 | ], 192 | "timeFrom": null, 193 | "timeShift": null, 194 | "title": "JVM", 195 | "tooltip": { 196 | "msResolution": false, 197 | "shared": true, 198 | "sort": 0, 199 | "value_type": "cumulative" 200 | }, 201 | "type": "graph", 202 | "xaxis": { 203 | "show": true 204 | }, 205 | "yaxes": [ 206 | { 207 | "format": "bytes", 208 | "label": null, 209 | "logBase": 1, 210 | "max": null, 211 | "min": null, 212 | "show": true 213 | }, 214 | { 215 | "format": "short", 216 | "label": null, 217 | "logBase": 1, 218 | "max": null, 219 | "min": null, 220 | "show": true 221 | } 222 | ] 223 | } 224 | ], 225 | "title": "Row" 226 | }, 227 | { 228 | "collapse": false, 229 | "editable": true, 230 | "height": "250px", 231 | "panels": [ 232 | { 233 | "aliasColors": {}, 234 | "bars": false, 235 | "datasource": "${DS_INFLUX}", 236 | "editable": true, 237 | "error": false, 238 | "fill": 1, 239 | "grid": { 240 | "threshold1": null, 241 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 242 | "threshold2": null, 243 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 244 | }, 245 | "id": 2, 246 | "isNew": true, 247 | "legend": { 248 | "avg": false, 249 | "current": false, 250 | "max": false, 251 | "min": false, 252 | "show": true, 253 | "total": false, 254 | "values": false 255 | }, 256 | "lines": true, 257 | "linewidth": 2, 258 | "links": [], 259 | "nullPointMode": "connected", 260 | "percentage": false, 261 | "pointradius": 5, 262 | "points": false, 263 | "renderer": "flot", 264 | "seriesOverrides": [], 265 | "span": 4, 266 | "stack": false, 267 | "steppedLine": false, 268 | "targets": [ 269 | { 270 | "dsType": "influxdb", 271 | "groupBy": [ 272 | { 273 | "params": [ 274 | "$interval" 275 | ], 276 | "type": "time" 277 | }, 278 | { 279 | "params": [ 280 | "null" 281 | ], 282 | "type": "fill" 283 | } 284 | ], 285 | "measurement": "ConnectMetrics", 286 | "policy": "default", 287 | "refId": "A", 288 | "resultFormat": "time_series", 289 | "select": [ 290 | [ 291 | { 292 | "params": [ 293 | "request-latency-avg" 294 | ], 295 | "type": "field" 296 | }, 297 | { 298 | "params": [], 299 | "type": "mean" 300 | } 301 | ] 302 | ], 303 | "tags": [] 304 | } 305 | ], 306 | "timeFrom": null, 307 | "timeShift": null, 308 | "title": "The average request latency in ms.", 309 | "tooltip": { 310 | "msResolution": false, 311 | "shared": true, 312 | "sort": 0, 313 | "value_type": "cumulative" 314 | }, 315 | "type": "graph", 316 | "xaxis": { 317 | "show": true 318 | }, 319 | "yaxes": [ 320 | { 321 | "format": "bytes", 322 | "label": null, 323 | "logBase": 1, 324 | "max": null, 325 | "min": null, 326 | "show": true 327 | }, 328 | { 329 | "format": "short", 330 | "label": null, 331 | "logBase": 1, 332 | "max": null, 333 | "min": null, 334 | "show": true 335 | } 336 | ] 337 | }, 338 | { 339 | "aliasColors": {}, 340 | "bars": false, 341 | "datasource": "${DS_INFLUX}", 342 | "editable": true, 343 | "error": false, 344 | "fill": 1, 345 | "grid": { 346 | "threshold1": null, 347 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 348 | "threshold2": null, 349 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 350 | }, 351 | "id": 3, 352 | "isNew": true, 353 | "legend": { 354 | "avg": false, 355 | "current": false, 356 | "max": false, 357 | "min": false, 358 | "show": true, 359 | "total": false, 360 | "values": false 361 | }, 362 | "lines": true, 363 | "linewidth": 2, 364 | "links": [], 365 | "nullPointMode": "connected", 366 | "percentage": false, 367 | "pointradius": 5, 368 | "points": false, 369 | "renderer": "flot", 370 | "seriesOverrides": [], 371 | "span": 4, 372 | "stack": false, 373 | "steppedLine": false, 374 | "targets": [ 375 | { 376 | "dsType": "influxdb", 377 | "groupBy": [ 378 | { 379 | "params": [ 380 | "$interval" 381 | ], 382 | "type": "time" 383 | }, 384 | { 385 | "params": [ 386 | "null" 387 | ], 388 | "type": "fill" 389 | } 390 | ], 391 | "measurement": "ConnectMetrics", 392 | "policy": "default", 393 | "refId": "A", 394 | "resultFormat": "time_series", 395 | "select": [ 396 | [ 397 | { 398 | "params": [ 399 | "request-latency-max" 400 | ], 401 | "type": "field" 402 | }, 403 | { 404 | "params": [], 405 | "type": "last" 406 | } 407 | ] 408 | ], 409 | "tags": [] 410 | } 411 | ], 412 | "timeFrom": null, 413 | "timeShift": null, 414 | "title": "The maximum request latency in ms.", 415 | "tooltip": { 416 | "msResolution": false, 417 | "shared": true, 418 | "sort": 0, 419 | "value_type": "cumulative" 420 | }, 421 | "type": "graph", 422 | "xaxis": { 423 | "show": true 424 | }, 425 | "yaxes": [ 426 | { 427 | "format": "short", 428 | "label": null, 429 | "logBase": 1, 430 | "max": null, 431 | "min": null, 432 | "show": true 433 | }, 434 | { 435 | "format": "short", 436 | "label": null, 437 | "logBase": 1, 438 | "max": null, 439 | "min": null, 440 | "show": true 441 | } 442 | ] 443 | }, 444 | { 445 | "aliasColors": {}, 446 | "bars": false, 447 | "datasource": "${DS_INFLUX}", 448 | "editable": true, 449 | "error": false, 450 | "fill": 1, 451 | "grid": { 452 | "threshold1": null, 453 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 454 | "threshold2": null, 455 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 456 | }, 457 | "id": 4, 458 | "isNew": true, 459 | "legend": { 460 | "avg": false, 461 | "current": false, 462 | "max": false, 463 | "min": false, 464 | "show": true, 465 | "total": false, 466 | "values": false 467 | }, 468 | "lines": true, 469 | "linewidth": 2, 470 | "links": [], 471 | "nullPointMode": "connected", 472 | "percentage": false, 473 | "pointradius": 5, 474 | "points": false, 475 | "renderer": "flot", 476 | "seriesOverrides": [], 477 | "span": 4, 478 | "stack": false, 479 | "steppedLine": false, 480 | "targets": [ 481 | { 482 | "dsType": "influxdb", 483 | "groupBy": [ 484 | { 485 | "params": [ 486 | "$interval" 487 | ], 488 | "type": "time" 489 | }, 490 | { 491 | "params": [ 492 | "null" 493 | ], 494 | "type": "fill" 495 | } 496 | ], 497 | "measurement": "ConnectMetrics", 498 | "policy": "default", 499 | "refId": "A", 500 | "resultFormat": "time_series", 501 | "select": [ 502 | [ 503 | { 504 | "params": [ 505 | "request-rate" 506 | ], 507 | "type": "field" 508 | }, 509 | { 510 | "params": [], 511 | "type": "last" 512 | } 513 | ] 514 | ], 515 | "tags": [] 516 | } 517 | ], 518 | "timeFrom": null, 519 | "timeShift": null, 520 | "title": "The average number of requests sent per second.", 521 | "tooltip": { 522 | "msResolution": false, 523 | "shared": true, 524 | "sort": 0, 525 | "value_type": "cumulative" 526 | }, 527 | "type": "graph", 528 | "xaxis": { 529 | "show": true 530 | }, 531 | "yaxes": [ 532 | { 533 | "format": "short", 534 | "label": null, 535 | "logBase": 1, 536 | "max": null, 537 | "min": null, 538 | "show": true 539 | }, 540 | { 541 | "format": "short", 542 | "label": null, 543 | "logBase": 1, 544 | "max": null, 545 | "min": null, 546 | "show": true 547 | } 548 | ] 549 | }, 550 | { 551 | "aliasColors": {}, 552 | "bars": false, 553 | "datasource": "${DS_INFLUX}", 554 | "editable": true, 555 | "error": false, 556 | "fill": 1, 557 | "grid": { 558 | "threshold1": null, 559 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 560 | "threshold2": null, 561 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 562 | }, 563 | "id": 5, 564 | "isNew": true, 565 | "legend": { 566 | "avg": false, 567 | "current": false, 568 | "max": false, 569 | "min": false, 570 | "show": true, 571 | "total": false, 572 | "values": false 573 | }, 574 | "lines": true, 575 | "linewidth": 2, 576 | "links": [], 577 | "nullPointMode": "connected", 578 | "percentage": false, 579 | "pointradius": 5, 580 | "points": false, 581 | "renderer": "flot", 582 | "seriesOverrides": [], 583 | "span": 6, 584 | "stack": false, 585 | "steppedLine": false, 586 | "targets": [ 587 | { 588 | "dsType": "influxdb", 589 | "groupBy": [ 590 | { 591 | "params": [ 592 | "$interval" 593 | ], 594 | "type": "time" 595 | }, 596 | { 597 | "params": [ 598 | "null" 599 | ], 600 | "type": "fill" 601 | } 602 | ], 603 | "measurement": "ConnectMetrics", 604 | "policy": "default", 605 | "refId": "A", 606 | "resultFormat": "time_series", 607 | "select": [ 608 | [ 609 | { 610 | "params": [ 611 | "response-rate" 612 | ], 613 | "type": "field" 614 | }, 615 | { 616 | "params": [], 617 | "type": "last" 618 | } 619 | ] 620 | ], 621 | "tags": [] 622 | } 623 | ], 624 | "timeFrom": null, 625 | "timeShift": null, 626 | "title": "Lag", 627 | "tooltip": { 628 | "msResolution": false, 629 | "shared": true, 630 | "sort": 0, 631 | "value_type": "cumulative" 632 | }, 633 | "type": "graph", 634 | "xaxis": { 635 | "show": true 636 | }, 637 | "yaxes": [ 638 | { 639 | "format": "short", 640 | "label": null, 641 | "logBase": 1, 642 | "max": null, 643 | "min": null, 644 | "show": true 645 | }, 646 | { 647 | "format": "short", 648 | "label": null, 649 | "logBase": 1, 650 | "max": null, 651 | "min": null, 652 | "show": true 653 | } 654 | ] 655 | }, 656 | { 657 | "aliasColors": {}, 658 | "bars": false, 659 | "datasource": "${DS_INFLUX}", 660 | "editable": true, 661 | "error": false, 662 | "fill": 1, 663 | "grid": { 664 | "threshold1": null, 665 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 666 | "threshold2": null, 667 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 668 | }, 669 | "id": 6, 670 | "isNew": true, 671 | "legend": { 672 | "avg": false, 673 | "current": false, 674 | "max": false, 675 | "min": false, 676 | "show": true, 677 | "total": false, 678 | "values": false 679 | }, 680 | "lines": true, 681 | "linewidth": 2, 682 | "links": [], 683 | "nullPointMode": "connected", 684 | "percentage": false, 685 | "pointradius": 5, 686 | "points": false, 687 | "renderer": "flot", 688 | "seriesOverrides": [], 689 | "span": 6, 690 | "stack": false, 691 | "steppedLine": false, 692 | "targets": [ 693 | { 694 | "dsType": "influxdb", 695 | "groupBy": [ 696 | { 697 | "params": [ 698 | "$interval" 699 | ], 700 | "type": "time" 701 | }, 702 | { 703 | "params": [ 704 | "null" 705 | ], 706 | "type": "fill" 707 | } 708 | ], 709 | "measurement": "ConnectMetrics", 710 | "policy": "default", 711 | "refId": "A", 712 | "resultFormat": "time_series", 713 | "select": [ 714 | [ 715 | { 716 | "params": [ 717 | "incoming-byte-rate" 718 | ], 719 | "type": "field" 720 | }, 721 | { 722 | "params": [], 723 | "type": "last" 724 | } 725 | ] 726 | ], 727 | "tags": [] 728 | } 729 | ], 730 | "timeFrom": null, 731 | "timeShift": null, 732 | "title": "The average number of incoming bytes received per second from all servers.", 733 | "tooltip": { 734 | "msResolution": false, 735 | "shared": true, 736 | "sort": 0, 737 | "value_type": "cumulative" 738 | }, 739 | "type": "graph", 740 | "xaxis": { 741 | "show": true 742 | }, 743 | "yaxes": [ 744 | { 745 | "format": "short", 746 | "label": null, 747 | "logBase": 1, 748 | "max": null, 749 | "min": null, 750 | "show": true 751 | }, 752 | { 753 | "format": "short", 754 | "label": null, 755 | "logBase": 1, 756 | "max": null, 757 | "min": null, 758 | "show": true 759 | } 760 | ] 761 | } 762 | ], 763 | "title": "New row" 764 | } 765 | ], 766 | "time": { 767 | "from": "now-1h", 768 | "to": "now" 769 | }, 770 | "timepicker": { 771 | "refresh_intervals": [ 772 | "5s", 773 | "10s", 774 | "30s", 775 | "1m", 776 | "5m", 777 | "15m", 778 | "30m", 779 | "1h", 780 | "2h", 781 | "1d" 782 | ], 783 | "time_options": [ 784 | "5m", 785 | "15m", 786 | "1h", 787 | "6h", 788 | "12h", 789 | "24h", 790 | "2d", 791 | "7d", 792 | "30d" 793 | ] 794 | }, 795 | "templating": { 796 | "list": [] 797 | }, 798 | "annotations": { 799 | "list": [] 800 | }, 801 | "schemaVersion": 12, 802 | "version": 2, 803 | "links": [], 804 | "gnetId": null 805 | } -------------------------------------------------------------------------------- /metrics/connect/connect.json: -------------------------------------------------------------------------------- 1 | { 2 | "servers": [ 3 | { 4 | "port": "${port1}", 5 | "host": "${url1}", 6 | "alias": "kafka-connect", 7 | "queries": [ 8 | { 9 | "outputWriters": [ 10 | { 11 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 12 | "url": "${influxUrl}", 13 | "database": "${influxDb}", 14 | "username": "${influxUser}", 15 | "password": "${influxPwd}" 16 | } 17 | ], 18 | "obj": "java.lang:type=Memory", 19 | "attr": [ 20 | "HeapMemoryUsage", 21 | "NonHeapMemoryUsage" 22 | ], 23 | "resultAlias": "kafkaConnectMemory" 24 | }, 25 | { 26 | "outputWriters": [ 27 | { 28 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 29 | "url": "${influxUrl}", 30 | "database": "${influxDb}", 31 | "username": "${influxUser}", 32 | "password": "${influxPwd}" 33 | } 34 | ], 35 | "obj": "kafka.producer:type=producer-metrics,client-id=*", 36 | "attr": [ 37 | "request-latency-avg", 38 | "request-latency-max", 39 | "request-rate", 40 | "response-rate", 41 | "incoming-byte-rate" 42 | ], 43 | "resultAlias": "ConnectMetrics" 44 | } 45 | ], 46 | "numQueryThreads": 2 47 | } 48 | ] 49 | } -------------------------------------------------------------------------------- /metrics/consumer/ApacheKafkaConsumerMetrics.json: -------------------------------------------------------------------------------- 1 | { 2 | "__inputs": [ 3 | { 4 | "name": "DS_INFLUX", 5 | "label": "influx", 6 | "description": "", 7 | "type": "datasource", 8 | "pluginId": "influxdb", 9 | "pluginName": "InfluxDB" 10 | } 11 | ], 12 | "__requires": [ 13 | { 14 | "type": "panel", 15 | "id": "graph", 16 | "name": "Graph", 17 | "version": "" 18 | }, 19 | { 20 | "type": "grafana", 21 | "id": "grafana", 22 | "name": "Grafana", 23 | "version": "3.1.0" 24 | }, 25 | { 26 | "type": "datasource", 27 | "id": "influxdb", 28 | "name": "InfluxDB", 29 | "version": "1.0.0" 30 | } 31 | ], 32 | "id": null, 33 | "title": "Apache Kafka Consumer Metrics", 34 | "tags": [], 35 | "style": "dark", 36 | "timezone": "browser", 37 | "editable": true, 38 | "hideControls": false, 39 | "sharedCrosshair": false, 40 | "rows": [ 41 | { 42 | "collapse": false, 43 | "editable": true, 44 | "height": "250px", 45 | "panels": [ 46 | { 47 | "aliasColors": {}, 48 | "bars": false, 49 | "datasource": "${DS_INFLUX}", 50 | "editable": true, 51 | "error": false, 52 | "fill": 1, 53 | "grid": { 54 | "threshold1": null, 55 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 56 | "threshold2": null, 57 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 58 | }, 59 | "id": 1, 60 | "isNew": true, 61 | "legend": { 62 | "avg": false, 63 | "current": false, 64 | "max": false, 65 | "min": false, 66 | "show": true, 67 | "total": false, 68 | "values": false 69 | }, 70 | "lines": true, 71 | "linewidth": 2, 72 | "links": [], 73 | "nullPointMode": "connected", 74 | "percentage": false, 75 | "pointradius": 5, 76 | "points": false, 77 | "renderer": "flot", 78 | "seriesOverrides": [], 79 | "span": 12, 80 | "stack": false, 81 | "steppedLine": false, 82 | "targets": [ 83 | { 84 | "dsType": "influxdb", 85 | "groupBy": [ 86 | { 87 | "params": [ 88 | "$interval" 89 | ], 90 | "type": "time" 91 | }, 92 | { 93 | "params": [ 94 | "null" 95 | ], 96 | "type": "fill" 97 | } 98 | ], 99 | "measurement": "jvmMemory", 100 | "policy": "default", 101 | "refId": "A", 102 | "resultFormat": "time_series", 103 | "select": [ 104 | [ 105 | { 106 | "params": [ 107 | "used" 108 | ], 109 | "type": "field" 110 | }, 111 | { 112 | "params": [], 113 | "type": "last" 114 | }, 115 | { 116 | "params": [ 117 | "heap" 118 | ], 119 | "type": "alias" 120 | } 121 | ] 122 | ], 123 | "tags": [ 124 | { 125 | "key": "attributeName", 126 | "operator": "=", 127 | "value": "HeapMemoryUsage" 128 | }, 129 | { 130 | "condition": "AND", 131 | "key": "hostname", 132 | "operator": "=", 133 | "value": "java-consumer" 134 | } 135 | ] 136 | }, 137 | { 138 | "dsType": "influxdb", 139 | "groupBy": [ 140 | { 141 | "params": [ 142 | "$interval" 143 | ], 144 | "type": "time" 145 | }, 146 | { 147 | "params": [ 148 | "null" 149 | ], 150 | "type": "fill" 151 | } 152 | ], 153 | "measurement": "jvmMemory", 154 | "policy": "default", 155 | "refId": "B", 156 | "resultFormat": "time_series", 157 | "select": [ 158 | [ 159 | { 160 | "params": [ 161 | "used" 162 | ], 163 | "type": "field" 164 | }, 165 | { 166 | "params": [], 167 | "type": "last" 168 | }, 169 | { 170 | "params": [ 171 | "non-heap" 172 | ], 173 | "type": "alias" 174 | } 175 | ] 176 | ], 177 | "tags": [ 178 | { 179 | "key": "attributeName", 180 | "operator": "=", 181 | "value": "NonHeapMemoryUsage" 182 | }, 183 | { 184 | "condition": "AND", 185 | "key": "hostname", 186 | "operator": "=", 187 | "value": "java-consumer" 188 | } 189 | ] 190 | } 191 | ], 192 | "timeFrom": null, 193 | "timeShift": null, 194 | "title": "JVM", 195 | "tooltip": { 196 | "msResolution": false, 197 | "shared": true, 198 | "sort": 0, 199 | "value_type": "cumulative" 200 | }, 201 | "type": "graph", 202 | "xaxis": { 203 | "show": true 204 | }, 205 | "yaxes": [ 206 | { 207 | "format": "bytes", 208 | "label": null, 209 | "logBase": 1, 210 | "max": null, 211 | "min": null, 212 | "show": true 213 | }, 214 | { 215 | "format": "short", 216 | "label": null, 217 | "logBase": 1, 218 | "max": null, 219 | "min": null, 220 | "show": true 221 | } 222 | ] 223 | } 224 | ], 225 | "title": "Row" 226 | }, 227 | { 228 | "collapse": false, 229 | "editable": true, 230 | "height": "250px", 231 | "panels": [ 232 | { 233 | "aliasColors": {}, 234 | "bars": false, 235 | "datasource": "${DS_INFLUX}", 236 | "editable": true, 237 | "error": false, 238 | "fill": 1, 239 | "grid": { 240 | "threshold1": null, 241 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 242 | "threshold2": null, 243 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 244 | }, 245 | "id": 2, 246 | "isNew": true, 247 | "legend": { 248 | "avg": false, 249 | "current": false, 250 | "max": false, 251 | "min": false, 252 | "show": true, 253 | "total": false, 254 | "values": false 255 | }, 256 | "lines": true, 257 | "linewidth": 2, 258 | "links": [], 259 | "nullPointMode": "connected", 260 | "percentage": false, 261 | "pointradius": 5, 262 | "points": false, 263 | "renderer": "flot", 264 | "seriesOverrides": [], 265 | "span": 4, 266 | "stack": false, 267 | "steppedLine": false, 268 | "targets": [ 269 | { 270 | "dsType": "influxdb", 271 | "groupBy": [ 272 | { 273 | "params": [ 274 | "$interval" 275 | ], 276 | "type": "time" 277 | }, 278 | { 279 | "params": [ 280 | "null" 281 | ], 282 | "type": "fill" 283 | } 284 | ], 285 | "measurement": "ConsumerMetrics", 286 | "policy": "default", 287 | "refId": "A", 288 | "resultFormat": "time_series", 289 | "select": [ 290 | [ 291 | { 292 | "params": [ 293 | "bytes-consumed-rate" 294 | ], 295 | "type": "field" 296 | }, 297 | { 298 | "params": [], 299 | "type": "mean" 300 | } 301 | ] 302 | ], 303 | "tags": [] 304 | } 305 | ], 306 | "timeFrom": null, 307 | "timeShift": null, 308 | "title": "Bytes Consumed per sec (avg)", 309 | "tooltip": { 310 | "msResolution": false, 311 | "shared": true, 312 | "sort": 0, 313 | "value_type": "cumulative" 314 | }, 315 | "type": "graph", 316 | "xaxis": { 317 | "show": true 318 | }, 319 | "yaxes": [ 320 | { 321 | "format": "bytes", 322 | "label": null, 323 | "logBase": 1, 324 | "max": null, 325 | "min": null, 326 | "show": true 327 | }, 328 | { 329 | "format": "short", 330 | "label": null, 331 | "logBase": 1, 332 | "max": null, 333 | "min": null, 334 | "show": true 335 | } 336 | ] 337 | }, 338 | { 339 | "aliasColors": {}, 340 | "bars": false, 341 | "datasource": "${DS_INFLUX}", 342 | "editable": true, 343 | "error": false, 344 | "fill": 1, 345 | "grid": { 346 | "threshold1": null, 347 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 348 | "threshold2": null, 349 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 350 | }, 351 | "id": 3, 352 | "isNew": true, 353 | "legend": { 354 | "avg": false, 355 | "current": false, 356 | "max": false, 357 | "min": false, 358 | "show": true, 359 | "total": false, 360 | "values": false 361 | }, 362 | "lines": true, 363 | "linewidth": 2, 364 | "links": [], 365 | "nullPointMode": "connected", 366 | "percentage": false, 367 | "pointradius": 5, 368 | "points": false, 369 | "renderer": "flot", 370 | "seriesOverrides": [], 371 | "span": 4, 372 | "stack": false, 373 | "steppedLine": false, 374 | "targets": [ 375 | { 376 | "dsType": "influxdb", 377 | "groupBy": [ 378 | { 379 | "params": [ 380 | "$interval" 381 | ], 382 | "type": "time" 383 | }, 384 | { 385 | "params": [ 386 | "null" 387 | ], 388 | "type": "fill" 389 | } 390 | ], 391 | "measurement": "ConsumerMetrics", 392 | "policy": "default", 393 | "refId": "A", 394 | "resultFormat": "time_series", 395 | "select": [ 396 | [ 397 | { 398 | "params": [ 399 | "fetch-latency-avg" 400 | ], 401 | "type": "field" 402 | }, 403 | { 404 | "params": [], 405 | "type": "last" 406 | } 407 | ] 408 | ], 409 | "tags": [] 410 | } 411 | ], 412 | "timeFrom": null, 413 | "timeShift": null, 414 | "title": "Fetch Latency (avg)", 415 | "tooltip": { 416 | "msResolution": false, 417 | "shared": true, 418 | "sort": 0, 419 | "value_type": "cumulative" 420 | }, 421 | "type": "graph", 422 | "xaxis": { 423 | "show": true 424 | }, 425 | "yaxes": [ 426 | { 427 | "format": "short", 428 | "label": null, 429 | "logBase": 1, 430 | "max": null, 431 | "min": null, 432 | "show": true 433 | }, 434 | { 435 | "format": "short", 436 | "label": null, 437 | "logBase": 1, 438 | "max": null, 439 | "min": null, 440 | "show": true 441 | } 442 | ] 443 | }, 444 | { 445 | "aliasColors": {}, 446 | "bars": false, 447 | "datasource": "${DS_INFLUX}", 448 | "editable": true, 449 | "error": false, 450 | "fill": 1, 451 | "grid": { 452 | "threshold1": null, 453 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 454 | "threshold2": null, 455 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 456 | }, 457 | "id": 4, 458 | "isNew": true, 459 | "legend": { 460 | "avg": false, 461 | "current": false, 462 | "max": false, 463 | "min": false, 464 | "show": true, 465 | "total": false, 466 | "values": false 467 | }, 468 | "lines": true, 469 | "linewidth": 2, 470 | "links": [], 471 | "nullPointMode": "connected", 472 | "percentage": false, 473 | "pointradius": 5, 474 | "points": false, 475 | "renderer": "flot", 476 | "seriesOverrides": [], 477 | "span": 4, 478 | "stack": false, 479 | "steppedLine": false, 480 | "targets": [ 481 | { 482 | "dsType": "influxdb", 483 | "groupBy": [ 484 | { 485 | "params": [ 486 | "$interval" 487 | ], 488 | "type": "time" 489 | }, 490 | { 491 | "params": [ 492 | "null" 493 | ], 494 | "type": "fill" 495 | } 496 | ], 497 | "measurement": "ConsumerMetrics", 498 | "policy": "default", 499 | "refId": "A", 500 | "resultFormat": "time_series", 501 | "select": [ 502 | [ 503 | { 504 | "params": [ 505 | "fetch-rate" 506 | ], 507 | "type": "field" 508 | }, 509 | { 510 | "params": [], 511 | "type": "last" 512 | } 513 | ] 514 | ], 515 | "tags": [] 516 | } 517 | ], 518 | "timeFrom": null, 519 | "timeShift": null, 520 | "title": "Fetch Requests per second", 521 | "tooltip": { 522 | "msResolution": false, 523 | "shared": true, 524 | "sort": 0, 525 | "value_type": "cumulative" 526 | }, 527 | "type": "graph", 528 | "xaxis": { 529 | "show": true 530 | }, 531 | "yaxes": [ 532 | { 533 | "format": "short", 534 | "label": null, 535 | "logBase": 1, 536 | "max": null, 537 | "min": null, 538 | "show": true 539 | }, 540 | { 541 | "format": "short", 542 | "label": null, 543 | "logBase": 1, 544 | "max": null, 545 | "min": null, 546 | "show": true 547 | } 548 | ] 549 | }, 550 | { 551 | "aliasColors": {}, 552 | "bars": false, 553 | "datasource": "${DS_INFLUX}", 554 | "editable": true, 555 | "error": false, 556 | "fill": 1, 557 | "grid": { 558 | "threshold1": null, 559 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 560 | "threshold2": null, 561 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 562 | }, 563 | "id": 5, 564 | "isNew": true, 565 | "legend": { 566 | "avg": false, 567 | "current": false, 568 | "max": false, 569 | "min": false, 570 | "show": true, 571 | "total": false, 572 | "values": false 573 | }, 574 | "lines": true, 575 | "linewidth": 2, 576 | "links": [], 577 | "nullPointMode": "connected", 578 | "percentage": false, 579 | "pointradius": 5, 580 | "points": false, 581 | "renderer": "flot", 582 | "seriesOverrides": [], 583 | "span": 6, 584 | "stack": false, 585 | "steppedLine": false, 586 | "targets": [ 587 | { 588 | "dsType": "influxdb", 589 | "groupBy": [ 590 | { 591 | "params": [ 592 | "$interval" 593 | ], 594 | "type": "time" 595 | }, 596 | { 597 | "params": [ 598 | "null" 599 | ], 600 | "type": "fill" 601 | } 602 | ], 603 | "measurement": "ConsumerMetrics", 604 | "policy": "default", 605 | "refId": "A", 606 | "resultFormat": "time_series", 607 | "select": [ 608 | [ 609 | { 610 | "params": [ 611 | "records-lag-max" 612 | ], 613 | "type": "field" 614 | }, 615 | { 616 | "params": [], 617 | "type": "last" 618 | } 619 | ] 620 | ], 621 | "tags": [] 622 | } 623 | ], 624 | "timeFrom": null, 625 | "timeShift": null, 626 | "title": "Lag", 627 | "tooltip": { 628 | "msResolution": false, 629 | "shared": true, 630 | "sort": 0, 631 | "value_type": "cumulative" 632 | }, 633 | "type": "graph", 634 | "xaxis": { 635 | "show": true 636 | }, 637 | "yaxes": [ 638 | { 639 | "format": "short", 640 | "label": null, 641 | "logBase": 1, 642 | "max": null, 643 | "min": null, 644 | "show": true 645 | }, 646 | { 647 | "format": "short", 648 | "label": null, 649 | "logBase": 1, 650 | "max": null, 651 | "min": null, 652 | "show": true 653 | } 654 | ] 655 | }, 656 | { 657 | "aliasColors": {}, 658 | "bars": false, 659 | "datasource": "${DS_INFLUX}", 660 | "editable": true, 661 | "error": false, 662 | "fill": 1, 663 | "grid": { 664 | "threshold1": null, 665 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 666 | "threshold2": null, 667 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 668 | }, 669 | "id": 6, 670 | "isNew": true, 671 | "legend": { 672 | "avg": false, 673 | "current": false, 674 | "max": false, 675 | "min": false, 676 | "show": true, 677 | "total": false, 678 | "values": false 679 | }, 680 | "lines": true, 681 | "linewidth": 2, 682 | "links": [], 683 | "nullPointMode": "connected", 684 | "percentage": false, 685 | "pointradius": 5, 686 | "points": false, 687 | "renderer": "flot", 688 | "seriesOverrides": [], 689 | "span": 6, 690 | "stack": false, 691 | "steppedLine": false, 692 | "targets": [ 693 | { 694 | "dsType": "influxdb", 695 | "groupBy": [ 696 | { 697 | "params": [ 698 | "$interval" 699 | ], 700 | "type": "time" 701 | }, 702 | { 703 | "params": [ 704 | "null" 705 | ], 706 | "type": "fill" 707 | } 708 | ], 709 | "measurement": "ConsumerMetrics", 710 | "policy": "default", 711 | "refId": "A", 712 | "resultFormat": "time_series", 713 | "select": [ 714 | [ 715 | { 716 | "params": [ 717 | "records-per-request-avg" 718 | ], 719 | "type": "field" 720 | }, 721 | { 722 | "params": [], 723 | "type": "last" 724 | } 725 | ] 726 | ], 727 | "tags": [] 728 | } 729 | ], 730 | "timeFrom": null, 731 | "timeShift": null, 732 | "title": "records per request", 733 | "tooltip": { 734 | "msResolution": false, 735 | "shared": true, 736 | "sort": 0, 737 | "value_type": "cumulative" 738 | }, 739 | "type": "graph", 740 | "xaxis": { 741 | "show": true 742 | }, 743 | "yaxes": [ 744 | { 745 | "format": "short", 746 | "label": null, 747 | "logBase": 1, 748 | "max": null, 749 | "min": null, 750 | "show": true 751 | }, 752 | { 753 | "format": "short", 754 | "label": null, 755 | "logBase": 1, 756 | "max": null, 757 | "min": null, 758 | "show": true 759 | } 760 | ] 761 | } 762 | ], 763 | "title": "New row" 764 | } 765 | ], 766 | "time": { 767 | "from": "now-1h", 768 | "to": "now" 769 | }, 770 | "timepicker": { 771 | "refresh_intervals": [ 772 | "5s", 773 | "10s", 774 | "30s", 775 | "1m", 776 | "5m", 777 | "15m", 778 | "30m", 779 | "1h", 780 | "2h", 781 | "1d" 782 | ], 783 | "time_options": [ 784 | "5m", 785 | "15m", 786 | "1h", 787 | "6h", 788 | "12h", 789 | "24h", 790 | "2d", 791 | "7d", 792 | "30d" 793 | ] 794 | }, 795 | "templating": { 796 | "list": [] 797 | }, 798 | "annotations": { 799 | "list": [] 800 | }, 801 | "schemaVersion": 12, 802 | "version": 2, 803 | "links": [], 804 | "gnetId": null 805 | } -------------------------------------------------------------------------------- /metrics/consumer/consumer.json: -------------------------------------------------------------------------------- 1 | { 2 | "servers": [ 3 | { 4 | "port": "${port1}", 5 | "host": "${url1}", 6 | "alias": "java-consumer", 7 | "queries": [ 8 | { 9 | "outputWriters": [ 10 | { 11 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 12 | "url": "${influxUrl}", 13 | "database": "${influxDb}", 14 | "username": "${influxUser}", 15 | "password": "${influxPwd}" 16 | } 17 | ], 18 | "obj": "java.lang:type=Memory", 19 | "attr": [ 20 | "HeapMemoryUsage", 21 | "NonHeapMemoryUsage" 22 | ], 23 | "resultAlias": "jvmMemory" 24 | }, 25 | { 26 | "outputWriters": [ 27 | { 28 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 29 | "url": "${influxUrl}", 30 | "database": "${influxDb}", 31 | "username": "${influxUser}", 32 | "password": "${influxPwd}" 33 | } 34 | ], 35 | "obj": "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*", 36 | "attr": [ 37 | "bytes-consumed-rate", 38 | "fetch-latency-avg", 39 | "fetch-rate", 40 | "records-lag-max", 41 | "records-per-request-avg" 42 | ], 43 | "resultAlias": "ConsumerMetrics" 44 | } 45 | ], 46 | "numQueryThreads": 2 47 | } 48 | ] 49 | } -------------------------------------------------------------------------------- /metrics/kafka/ApacheKafkaBrokerMetrics.json: -------------------------------------------------------------------------------- 1 | { 2 | "__inputs": [ 3 | { 4 | "name": "DS_INFLUX", 5 | "label": "influx", 6 | "description": "", 7 | "type": "datasource", 8 | "pluginId": "influxdb", 9 | "pluginName": "InfluxDB" 10 | } 11 | ], 12 | "__requires": [ 13 | { 14 | "type": "panel", 15 | "id": "graph", 16 | "name": "Graph", 17 | "version": "" 18 | }, 19 | { 20 | "type": "grafana", 21 | "id": "grafana", 22 | "name": "Grafana", 23 | "version": "3.1.0" 24 | }, 25 | { 26 | "type": "datasource", 27 | "id": "influxdb", 28 | "name": "InfluxDB", 29 | "version": "1.0.0" 30 | } 31 | ], 32 | "id": null, 33 | "title": "Apache Kafka Broker Metrics", 34 | "tags": [], 35 | "style": "dark", 36 | "timezone": "browser", 37 | "editable": true, 38 | "hideControls": false, 39 | "sharedCrosshair": false, 40 | "rows": [ 41 | { 42 | "collapse": false, 43 | "editable": true, 44 | "height": "250px", 45 | "panels": [ 46 | { 47 | "aliasColors": {}, 48 | "bars": false, 49 | "datasource": "${DS_INFLUX}", 50 | "editable": true, 51 | "error": false, 52 | "fill": 1, 53 | "grid": { 54 | "threshold1": null, 55 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 56 | "threshold2": null, 57 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 58 | }, 59 | "id": 1, 60 | "isNew": true, 61 | "legend": { 62 | "avg": false, 63 | "current": false, 64 | "max": false, 65 | "min": false, 66 | "show": true, 67 | "total": false, 68 | "values": false 69 | }, 70 | "lines": true, 71 | "linewidth": 2, 72 | "links": [], 73 | "nullPointMode": "connected", 74 | "percentage": false, 75 | "pointradius": 5, 76 | "points": false, 77 | "renderer": "flot", 78 | "seriesOverrides": [], 79 | "span": 6, 80 | "stack": false, 81 | "steppedLine": false, 82 | "targets": [ 83 | { 84 | "dsType": "influxdb", 85 | "groupBy": [ 86 | { 87 | "params": [ 88 | "$interval" 89 | ], 90 | "type": "time" 91 | }, 92 | { 93 | "params": [ 94 | "null" 95 | ], 96 | "type": "fill" 97 | } 98 | ], 99 | "measurement": "jvmMemory", 100 | "policy": "default", 101 | "refId": "A", 102 | "resultFormat": "time_series", 103 | "select": [ 104 | [ 105 | { 106 | "params": [ 107 | "init" 108 | ], 109 | "type": "field" 110 | }, 111 | { 112 | "params": [], 113 | "type": "last" 114 | }, 115 | { 116 | "params": [ 117 | "heap" 118 | ], 119 | "type": "alias" 120 | } 121 | ] 122 | ], 123 | "tags": [ 124 | { 125 | "key": "attributeName", 126 | "operator": "=", 127 | "value": "HeapMemoryUsage" 128 | } 129 | ] 130 | }, 131 | { 132 | "dsType": "influxdb", 133 | "groupBy": [ 134 | { 135 | "params": [ 136 | "$interval" 137 | ], 138 | "type": "time" 139 | }, 140 | { 141 | "params": [ 142 | "null" 143 | ], 144 | "type": "fill" 145 | } 146 | ], 147 | "measurement": "jvmMemory", 148 | "policy": "default", 149 | "refId": "B", 150 | "resultFormat": "time_series", 151 | "select": [ 152 | [ 153 | { 154 | "params": [ 155 | "init" 156 | ], 157 | "type": "field" 158 | }, 159 | { 160 | "params": [], 161 | "type": "last" 162 | }, 163 | { 164 | "params": [ 165 | "non-heap" 166 | ], 167 | "type": "alias" 168 | } 169 | ] 170 | ], 171 | "tags": [ 172 | { 173 | "key": "attributeName", 174 | "operator": "=", 175 | "value": "NonHeapMemoryUsage" 176 | } 177 | ] 178 | }, 179 | { 180 | "dsType": "influxdb", 181 | "groupBy": [ 182 | { 183 | "params": [ 184 | "$interval" 185 | ], 186 | "type": "time" 187 | }, 188 | { 189 | "params": [ 190 | "null" 191 | ], 192 | "type": "fill" 193 | } 194 | ], 195 | "measurement": "os", 196 | "policy": "default", 197 | "refId": "C", 198 | "resultFormat": "time_series", 199 | "select": [ 200 | [ 201 | { 202 | "params": [ 203 | "FreePhysicalMemorySize" 204 | ], 205 | "type": "field" 206 | }, 207 | { 208 | "params": [], 209 | "type": "last" 210 | }, 211 | { 212 | "params": [ 213 | "physical" 214 | ], 215 | "type": "alias" 216 | } 217 | ] 218 | ], 219 | "tags": [] 220 | } 221 | ], 222 | "timeFrom": null, 223 | "timeShift": null, 224 | "title": "Memory (JVM & Physical)", 225 | "tooltip": { 226 | "msResolution": false, 227 | "shared": true, 228 | "sort": 0, 229 | "value_type": "cumulative" 230 | }, 231 | "type": "graph", 232 | "xaxis": { 233 | "show": true 234 | }, 235 | "yaxes": [ 236 | { 237 | "format": "bytes", 238 | "label": null, 239 | "logBase": 1, 240 | "max": null, 241 | "min": null, 242 | "show": true 243 | }, 244 | { 245 | "format": "short", 246 | "label": null, 247 | "logBase": 1, 248 | "max": null, 249 | "min": null, 250 | "show": true 251 | } 252 | ] 253 | }, 254 | { 255 | "aliasColors": {}, 256 | "bars": false, 257 | "datasource": "${DS_INFLUX}", 258 | "editable": true, 259 | "error": false, 260 | "fill": 1, 261 | "grid": { 262 | "threshold1": null, 263 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 264 | "threshold2": null, 265 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 266 | }, 267 | "id": 2, 268 | "isNew": true, 269 | "legend": { 270 | "avg": false, 271 | "current": false, 272 | "max": false, 273 | "min": false, 274 | "show": true, 275 | "total": false, 276 | "values": false 277 | }, 278 | "lines": true, 279 | "linewidth": 2, 280 | "links": [], 281 | "nullPointMode": "connected", 282 | "percentage": false, 283 | "pointradius": 5, 284 | "points": false, 285 | "renderer": "flot", 286 | "seriesOverrides": [], 287 | "span": 6, 288 | "stack": false, 289 | "steppedLine": false, 290 | "targets": [ 291 | { 292 | "dsType": "influxdb", 293 | "groupBy": [ 294 | { 295 | "params": [ 296 | "$interval" 297 | ], 298 | "type": "time" 299 | }, 300 | { 301 | "params": [ 302 | "null" 303 | ], 304 | "type": "fill" 305 | } 306 | ], 307 | "measurement": "os", 308 | "policy": "default", 309 | "refId": "A", 310 | "resultFormat": "time_series", 311 | "select": [ 312 | [ 313 | { 314 | "params": [ 315 | "ProcessCpuLoad" 316 | ], 317 | "type": "field" 318 | }, 319 | { 320 | "params": [], 321 | "type": "last" 322 | }, 323 | { 324 | "params": [ 325 | "process" 326 | ], 327 | "type": "alias" 328 | } 329 | ], 330 | [ 331 | { 332 | "params": [ 333 | "SystemCpuLoad" 334 | ], 335 | "type": "field" 336 | }, 337 | { 338 | "params": [], 339 | "type": "last" 340 | }, 341 | { 342 | "params": [ 343 | "system" 344 | ], 345 | "type": "alias" 346 | } 347 | ], 348 | [ 349 | { 350 | "params": [ 351 | "SystemLoadAverage" 352 | ], 353 | "type": "field" 354 | }, 355 | { 356 | "params": [], 357 | "type": "last" 358 | }, 359 | { 360 | "params": [ 361 | "loadAvg" 362 | ], 363 | "type": "alias" 364 | } 365 | ] 366 | ], 367 | "tags": [] 368 | } 369 | ], 370 | "timeFrom": null, 371 | "timeShift": null, 372 | "title": "CPU", 373 | "tooltip": { 374 | "msResolution": false, 375 | "shared": true, 376 | "sort": 0, 377 | "value_type": "cumulative" 378 | }, 379 | "type": "graph", 380 | "xaxis": { 381 | "show": true 382 | }, 383 | "yaxes": [ 384 | { 385 | "format": "short", 386 | "label": null, 387 | "logBase": 1, 388 | "max": null, 389 | "min": null, 390 | "show": true 391 | }, 392 | { 393 | "format": "short", 394 | "label": null, 395 | "logBase": 1, 396 | "max": null, 397 | "min": null, 398 | "show": true 399 | } 400 | ] 401 | } 402 | ], 403 | "showTitle": true, 404 | "title": "OS Level" 405 | }, 406 | { 407 | "collapse": false, 408 | "editable": true, 409 | "height": "250px", 410 | "panels": [ 411 | { 412 | "aliasColors": {}, 413 | "bars": false, 414 | "datasource": "${DS_INFLUX}", 415 | "editable": true, 416 | "error": false, 417 | "fill": 1, 418 | "grid": { 419 | "threshold1": null, 420 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 421 | "threshold2": null, 422 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 423 | }, 424 | "id": 3, 425 | "isNew": true, 426 | "legend": { 427 | "avg": false, 428 | "current": false, 429 | "max": false, 430 | "min": false, 431 | "show": true, 432 | "total": false, 433 | "values": false 434 | }, 435 | "lines": true, 436 | "linewidth": 2, 437 | "links": [], 438 | "nullPointMode": "connected", 439 | "percentage": false, 440 | "pointradius": 5, 441 | "points": false, 442 | "renderer": "flot", 443 | "seriesOverrides": [], 444 | "span": 4, 445 | "stack": false, 446 | "steppedLine": false, 447 | "targets": [ 448 | { 449 | "dsType": "influxdb", 450 | "groupBy": [ 451 | { 452 | "params": [ 453 | "$interval" 454 | ], 455 | "type": "time" 456 | }, 457 | { 458 | "params": [ 459 | "null" 460 | ], 461 | "type": "fill" 462 | } 463 | ], 464 | "measurement": "LogFlush", 465 | "policy": "default", 466 | "refId": "A", 467 | "resultFormat": "time_series", 468 | "select": [ 469 | [ 470 | { 471 | "params": [ 472 | "98thPercentile" 473 | ], 474 | "type": "field" 475 | }, 476 | { 477 | "params": [], 478 | "type": "last" 479 | } 480 | ] 481 | ], 482 | "tags": [] 483 | } 484 | ], 485 | "timeFrom": null, 486 | "timeShift": null, 487 | "title": "Log Flush latency", 488 | "tooltip": { 489 | "msResolution": false, 490 | "shared": true, 491 | "sort": 0, 492 | "value_type": "cumulative" 493 | }, 494 | "type": "graph", 495 | "xaxis": { 496 | "show": true 497 | }, 498 | "yaxes": [ 499 | { 500 | "format": "short", 501 | "label": null, 502 | "logBase": 1, 503 | "max": null, 504 | "min": null, 505 | "show": true 506 | }, 507 | { 508 | "format": "short", 509 | "label": null, 510 | "logBase": 1, 511 | "max": null, 512 | "min": null, 513 | "show": true 514 | } 515 | ] 516 | }, 517 | { 518 | "aliasColors": {}, 519 | "bars": false, 520 | "datasource": "${DS_INFLUX}", 521 | "editable": true, 522 | "error": false, 523 | "fill": 1, 524 | "grid": { 525 | "threshold1": null, 526 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 527 | "threshold2": null, 528 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 529 | }, 530 | "id": 4, 531 | "isNew": true, 532 | "legend": { 533 | "avg": false, 534 | "current": false, 535 | "max": false, 536 | "min": false, 537 | "show": true, 538 | "total": false, 539 | "values": false 540 | }, 541 | "lines": true, 542 | "linewidth": 2, 543 | "links": [], 544 | "nullPointMode": "connected", 545 | "percentage": false, 546 | "pointradius": 5, 547 | "points": false, 548 | "renderer": "flot", 549 | "seriesOverrides": [], 550 | "span": 4, 551 | "stack": false, 552 | "steppedLine": false, 553 | "targets": [ 554 | { 555 | "dsType": "influxdb", 556 | "groupBy": [ 557 | { 558 | "params": [ 559 | "$interval" 560 | ], 561 | "type": "time" 562 | }, 563 | { 564 | "params": [ 565 | "null" 566 | ], 567 | "type": "fill" 568 | } 569 | ], 570 | "measurement": "UnderReplicatedPartitions", 571 | "policy": "default", 572 | "refId": "A", 573 | "resultFormat": "time_series", 574 | "select": [ 575 | [ 576 | { 577 | "params": [ 578 | "Value" 579 | ], 580 | "type": "field" 581 | }, 582 | { 583 | "params": [], 584 | "type": "last" 585 | } 586 | ] 587 | ], 588 | "tags": [] 589 | } 590 | ], 591 | "timeFrom": null, 592 | "timeShift": null, 593 | "title": "Under Replicated Partitions", 594 | "tooltip": { 595 | "msResolution": false, 596 | "shared": true, 597 | "sort": 0, 598 | "value_type": "cumulative" 599 | }, 600 | "type": "graph", 601 | "xaxis": { 602 | "show": true 603 | }, 604 | "yaxes": [ 605 | { 606 | "format": "short", 607 | "label": null, 608 | "logBase": 1, 609 | "max": null, 610 | "min": null, 611 | "show": true 612 | }, 613 | { 614 | "format": "short", 615 | "label": null, 616 | "logBase": 1, 617 | "max": null, 618 | "min": null, 619 | "show": true 620 | } 621 | ] 622 | }, 623 | { 624 | "aliasColors": {}, 625 | "bars": false, 626 | "datasource": "${DS_INFLUX}", 627 | "editable": true, 628 | "error": false, 629 | "fill": 1, 630 | "grid": { 631 | "threshold1": null, 632 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 633 | "threshold2": null, 634 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 635 | }, 636 | "id": 5, 637 | "isNew": true, 638 | "legend": { 639 | "avg": false, 640 | "current": false, 641 | "max": false, 642 | "min": false, 643 | "show": true, 644 | "total": false, 645 | "values": false 646 | }, 647 | "lines": true, 648 | "linewidth": 2, 649 | "links": [], 650 | "nullPointMode": "connected", 651 | "percentage": false, 652 | "pointradius": 5, 653 | "points": false, 654 | "renderer": "flot", 655 | "seriesOverrides": [], 656 | "span": 4, 657 | "stack": false, 658 | "steppedLine": false, 659 | "targets": [ 660 | { 661 | "dsType": "influxdb", 662 | "groupBy": [ 663 | { 664 | "params": [ 665 | "$interval" 666 | ], 667 | "type": "time" 668 | }, 669 | { 670 | "params": [ 671 | "null" 672 | ], 673 | "type": "fill" 674 | } 675 | ], 676 | "measurement": "MessagesInPerSec", 677 | "policy": "default", 678 | "refId": "A", 679 | "resultFormat": "time_series", 680 | "select": [ 681 | [ 682 | { 683 | "params": [ 684 | "Count" 685 | ], 686 | "type": "field" 687 | }, 688 | { 689 | "params": [], 690 | "type": "last" 691 | } 692 | ] 693 | ], 694 | "tags": [] 695 | }, 696 | { 697 | "dsType": "influxdb", 698 | "groupBy": [ 699 | { 700 | "params": [ 701 | "$interval" 702 | ], 703 | "type": "time" 704 | }, 705 | { 706 | "params": [ 707 | "null" 708 | ], 709 | "type": "fill" 710 | } 711 | ], 712 | "measurement": "MessagesInPerSecPerTopic", 713 | "policy": "default", 714 | "refId": "B", 715 | "resultFormat": "time_series", 716 | "select": [ 717 | [ 718 | { 719 | "params": [ 720 | "Count" 721 | ], 722 | "type": "field" 723 | }, 724 | { 725 | "params": [], 726 | "type": "last" 727 | }, 728 | { 729 | "params": [ 730 | "topic_t1" 731 | ], 732 | "type": "alias" 733 | } 734 | ] 735 | ], 736 | "tags": [ 737 | { 738 | "key": "typeName", 739 | "operator": "=", 740 | "value": "type=BrokerTopicMetrics,name=MessagesInPerSec,topic=t1" 741 | } 742 | ] 743 | }, 744 | { 745 | "dsType": "influxdb", 746 | "groupBy": [ 747 | { 748 | "params": [ 749 | "$interval" 750 | ], 751 | "type": "time" 752 | }, 753 | { 754 | "params": [ 755 | "null" 756 | ], 757 | "type": "fill" 758 | } 759 | ], 760 | "measurement": "MessagesInPerSecPerTopic", 761 | "policy": "default", 762 | "refId": "C", 763 | "resultFormat": "time_series", 764 | "select": [ 765 | [ 766 | { 767 | "params": [ 768 | "Count" 769 | ], 770 | "type": "field" 771 | }, 772 | { 773 | "params": [], 774 | "type": "last" 775 | }, 776 | { 777 | "params": [ 778 | "topic_t2" 779 | ], 780 | "type": "alias" 781 | } 782 | ] 783 | ], 784 | "tags": [ 785 | { 786 | "key": "typeName", 787 | "operator": "=", 788 | "value": "type=BrokerTopicMetrics,name=MessagesInPerSec,topic=t2" 789 | } 790 | ] 791 | } 792 | ], 793 | "timeFrom": null, 794 | "timeShift": null, 795 | "title": "Messages In per sec", 796 | "tooltip": { 797 | "msResolution": false, 798 | "shared": true, 799 | "sort": 0, 800 | "value_type": "cumulative" 801 | }, 802 | "type": "graph", 803 | "xaxis": { 804 | "show": true 805 | }, 806 | "yaxes": [ 807 | { 808 | "format": "short", 809 | "label": null, 810 | "logBase": 1, 811 | "max": null, 812 | "min": null, 813 | "show": true 814 | }, 815 | { 816 | "format": "short", 817 | "label": null, 818 | "logBase": 1, 819 | "max": null, 820 | "min": null, 821 | "show": true 822 | } 823 | ] 824 | }, 825 | { 826 | "aliasColors": {}, 827 | "bars": false, 828 | "datasource": "${DS_INFLUX}", 829 | "editable": true, 830 | "error": false, 831 | "fill": 1, 832 | "grid": { 833 | "threshold1": null, 834 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 835 | "threshold2": null, 836 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 837 | }, 838 | "id": 6, 839 | "isNew": true, 840 | "legend": { 841 | "avg": false, 842 | "current": false, 843 | "max": false, 844 | "min": false, 845 | "show": true, 846 | "total": false, 847 | "values": false 848 | }, 849 | "lines": true, 850 | "linewidth": 2, 851 | "links": [], 852 | "nullPointMode": "connected", 853 | "percentage": false, 854 | "pointradius": 5, 855 | "points": false, 856 | "renderer": "flot", 857 | "seriesOverrides": [], 858 | "span": 6, 859 | "stack": false, 860 | "steppedLine": false, 861 | "targets": [ 862 | { 863 | "dsType": "influxdb", 864 | "groupBy": [ 865 | { 866 | "params": [ 867 | "$interval" 868 | ], 869 | "type": "time" 870 | }, 871 | { 872 | "params": [ 873 | "null" 874 | ], 875 | "type": "fill" 876 | } 877 | ], 878 | "measurement": "BytesInPerSec", 879 | "policy": "default", 880 | "refId": "A", 881 | "resultFormat": "time_series", 882 | "select": [ 883 | [ 884 | { 885 | "params": [ 886 | "Count" 887 | ], 888 | "type": "field" 889 | }, 890 | { 891 | "params": [], 892 | "type": "last" 893 | } 894 | ] 895 | ], 896 | "tags": [] 897 | }, 898 | { 899 | "dsType": "influxdb", 900 | "groupBy": [ 901 | { 902 | "params": [ 903 | "$interval" 904 | ], 905 | "type": "time" 906 | }, 907 | { 908 | "params": [ 909 | "null" 910 | ], 911 | "type": "fill" 912 | } 913 | ], 914 | "measurement": "BytesInPerSecPerTopic", 915 | "policy": "default", 916 | "refId": "B", 917 | "resultFormat": "time_series", 918 | "select": [ 919 | [ 920 | { 921 | "params": [ 922 | "Count" 923 | ], 924 | "type": "field" 925 | }, 926 | { 927 | "params": [], 928 | "type": "last" 929 | }, 930 | { 931 | "params": [ 932 | "topic_t1" 933 | ], 934 | "type": "alias" 935 | } 936 | ] 937 | ], 938 | "tags": [ 939 | { 940 | "key": "typeName", 941 | "operator": "=", 942 | "value": "type=BrokerTopicMetrics,name=BytesInPerSec,topic=t1" 943 | } 944 | ] 945 | }, 946 | { 947 | "dsType": "influxdb", 948 | "groupBy": [ 949 | { 950 | "params": [ 951 | "$interval" 952 | ], 953 | "type": "time" 954 | }, 955 | { 956 | "params": [ 957 | "null" 958 | ], 959 | "type": "fill" 960 | } 961 | ], 962 | "measurement": "BytesInPerSecPerTopic", 963 | "policy": "default", 964 | "refId": "C", 965 | "resultFormat": "time_series", 966 | "select": [ 967 | [ 968 | { 969 | "params": [ 970 | "Count" 971 | ], 972 | "type": "field" 973 | }, 974 | { 975 | "params": [], 976 | "type": "last" 977 | }, 978 | { 979 | "params": [ 980 | "topic_t2" 981 | ], 982 | "type": "alias" 983 | } 984 | ] 985 | ], 986 | "tags": [ 987 | { 988 | "key": "typeName", 989 | "operator": "=", 990 | "value": "type=BrokerTopicMetrics,name=BytesInPerSec,topic=t2" 991 | } 992 | ] 993 | } 994 | ], 995 | "timeFrom": null, 996 | "timeShift": null, 997 | "title": "Bytes In", 998 | "tooltip": { 999 | "msResolution": true, 1000 | "shared": true, 1001 | "sort": 0, 1002 | "value_type": "cumulative" 1003 | }, 1004 | "type": "graph", 1005 | "xaxis": { 1006 | "show": true 1007 | }, 1008 | "yaxes": [ 1009 | { 1010 | "format": "bytes", 1011 | "label": null, 1012 | "logBase": 1, 1013 | "max": null, 1014 | "min": null, 1015 | "show": true 1016 | }, 1017 | { 1018 | "format": "short", 1019 | "label": null, 1020 | "logBase": 1, 1021 | "max": null, 1022 | "min": null, 1023 | "show": true 1024 | } 1025 | ] 1026 | }, 1027 | { 1028 | "aliasColors": {}, 1029 | "bars": false, 1030 | "datasource": "${DS_INFLUX}", 1031 | "editable": true, 1032 | "error": false, 1033 | "fill": 1, 1034 | "grid": { 1035 | "threshold1": null, 1036 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 1037 | "threshold2": null, 1038 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 1039 | }, 1040 | "id": 16, 1041 | "isNew": true, 1042 | "legend": { 1043 | "avg": false, 1044 | "current": false, 1045 | "max": false, 1046 | "min": false, 1047 | "show": true, 1048 | "total": false, 1049 | "values": false 1050 | }, 1051 | "lines": true, 1052 | "linewidth": 2, 1053 | "links": [], 1054 | "nullPointMode": "connected", 1055 | "percentage": false, 1056 | "pointradius": 5, 1057 | "points": false, 1058 | "renderer": "flot", 1059 | "seriesOverrides": [], 1060 | "span": 6, 1061 | "stack": false, 1062 | "steppedLine": false, 1063 | "targets": [ 1064 | { 1065 | "dsType": "influxdb", 1066 | "groupBy": [ 1067 | { 1068 | "params": [ 1069 | "$interval" 1070 | ], 1071 | "type": "time" 1072 | }, 1073 | { 1074 | "params": [ 1075 | "null" 1076 | ], 1077 | "type": "fill" 1078 | } 1079 | ], 1080 | "measurement": "BytesOutPerSec", 1081 | "policy": "default", 1082 | "refId": "A", 1083 | "resultFormat": "time_series", 1084 | "select": [ 1085 | [ 1086 | { 1087 | "params": [ 1088 | "Count" 1089 | ], 1090 | "type": "field" 1091 | }, 1092 | { 1093 | "params": [], 1094 | "type": "last" 1095 | } 1096 | ] 1097 | ], 1098 | "tags": [] 1099 | }, 1100 | { 1101 | "dsType": "influxdb", 1102 | "groupBy": [ 1103 | { 1104 | "params": [ 1105 | "$interval" 1106 | ], 1107 | "type": "time" 1108 | }, 1109 | { 1110 | "params": [ 1111 | "null" 1112 | ], 1113 | "type": "fill" 1114 | } 1115 | ], 1116 | "measurement": "BytesOutPerSecPerTopic", 1117 | "policy": "default", 1118 | "refId": "B", 1119 | "resultFormat": "time_series", 1120 | "select": [ 1121 | [ 1122 | { 1123 | "params": [ 1124 | "Count" 1125 | ], 1126 | "type": "field" 1127 | }, 1128 | { 1129 | "params": [], 1130 | "type": "last" 1131 | }, 1132 | { 1133 | "params": [ 1134 | "topic_t1" 1135 | ], 1136 | "type": "alias" 1137 | } 1138 | ] 1139 | ], 1140 | "tags": [ 1141 | { 1142 | "key": "typeName", 1143 | "operator": "=", 1144 | "value": "type=BrokerTopicMetrics,name=BytesOutPerSec,topic=t1" 1145 | } 1146 | ] 1147 | }, 1148 | { 1149 | "dsType": "influxdb", 1150 | "groupBy": [ 1151 | { 1152 | "params": [ 1153 | "$interval" 1154 | ], 1155 | "type": "time" 1156 | }, 1157 | { 1158 | "params": [ 1159 | "null" 1160 | ], 1161 | "type": "fill" 1162 | } 1163 | ], 1164 | "measurement": "BytesOutPerSecPerTopic", 1165 | "policy": "default", 1166 | "refId": "C", 1167 | "resultFormat": "time_series", 1168 | "select": [ 1169 | [ 1170 | { 1171 | "params": [ 1172 | "Count" 1173 | ], 1174 | "type": "field" 1175 | }, 1176 | { 1177 | "params": [], 1178 | "type": "last" 1179 | }, 1180 | { 1181 | "params": [ 1182 | "topic_t2" 1183 | ], 1184 | "type": "alias" 1185 | } 1186 | ] 1187 | ], 1188 | "tags": [ 1189 | { 1190 | "key": "typeName", 1191 | "operator": "=", 1192 | "value": "type=BrokerTopicMetrics,name=BytesOutPerSec,topic=t2" 1193 | } 1194 | ] 1195 | } 1196 | ], 1197 | "timeFrom": null, 1198 | "timeShift": null, 1199 | "title": "Bytes Out", 1200 | "tooltip": { 1201 | "msResolution": true, 1202 | "shared": true, 1203 | "sort": 0, 1204 | "value_type": "cumulative" 1205 | }, 1206 | "type": "graph", 1207 | "xaxis": { 1208 | "show": true 1209 | }, 1210 | "yaxes": [ 1211 | { 1212 | "format": "bytes", 1213 | "label": null, 1214 | "logBase": 1, 1215 | "max": null, 1216 | "min": null, 1217 | "show": true 1218 | }, 1219 | { 1220 | "format": "short", 1221 | "label": null, 1222 | "logBase": 1, 1223 | "max": null, 1224 | "min": null, 1225 | "show": true 1226 | } 1227 | ] 1228 | }, 1229 | { 1230 | "aliasColors": {}, 1231 | "bars": false, 1232 | "datasource": "${DS_INFLUX}", 1233 | "editable": true, 1234 | "error": false, 1235 | "fill": 1, 1236 | "grid": { 1237 | "threshold1": null, 1238 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 1239 | "threshold2": null, 1240 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 1241 | }, 1242 | "id": 7, 1243 | "isNew": true, 1244 | "legend": { 1245 | "avg": false, 1246 | "current": false, 1247 | "max": false, 1248 | "min": false, 1249 | "show": true, 1250 | "total": false, 1251 | "values": false 1252 | }, 1253 | "lines": true, 1254 | "linewidth": 2, 1255 | "links": [], 1256 | "nullPointMode": "connected", 1257 | "percentage": false, 1258 | "pointradius": 5, 1259 | "points": false, 1260 | "renderer": "flot", 1261 | "seriesOverrides": [], 1262 | "span": 6, 1263 | "stack": false, 1264 | "steppedLine": false, 1265 | "targets": [ 1266 | { 1267 | "dsType": "influxdb", 1268 | "groupBy": [ 1269 | { 1270 | "params": [ 1271 | "$interval" 1272 | ], 1273 | "type": "time" 1274 | }, 1275 | { 1276 | "params": [ 1277 | "null" 1278 | ], 1279 | "type": "fill" 1280 | } 1281 | ], 1282 | "measurement": "BytesInPerSec", 1283 | "policy": "default", 1284 | "refId": "A", 1285 | "resultFormat": "time_series", 1286 | "select": [ 1287 | [ 1288 | { 1289 | "params": [ 1290 | "FiveMinuteRate" 1291 | ], 1292 | "type": "field" 1293 | }, 1294 | { 1295 | "params": [], 1296 | "type": "last" 1297 | } 1298 | ] 1299 | ], 1300 | "tags": [] 1301 | }, 1302 | { 1303 | "dsType": "influxdb", 1304 | "groupBy": [ 1305 | { 1306 | "params": [ 1307 | "$interval" 1308 | ], 1309 | "type": "time" 1310 | }, 1311 | { 1312 | "params": [ 1313 | "null" 1314 | ], 1315 | "type": "fill" 1316 | } 1317 | ], 1318 | "measurement": "BytesInPerSecPerTopic", 1319 | "policy": "default", 1320 | "refId": "B", 1321 | "resultFormat": "time_series", 1322 | "select": [ 1323 | [ 1324 | { 1325 | "params": [ 1326 | "FiveMinuteRate" 1327 | ], 1328 | "type": "field" 1329 | }, 1330 | { 1331 | "params": [], 1332 | "type": "last" 1333 | }, 1334 | { 1335 | "params": [ 1336 | "topic_t1" 1337 | ], 1338 | "type": "alias" 1339 | } 1340 | ] 1341 | ], 1342 | "tags": [ 1343 | { 1344 | "key": "typeName", 1345 | "operator": "=", 1346 | "value": "type=BrokerTopicMetrics,name=BytesInPerSec,topic=t1" 1347 | } 1348 | ] 1349 | }, 1350 | { 1351 | "dsType": "influxdb", 1352 | "groupBy": [ 1353 | { 1354 | "params": [ 1355 | "$interval" 1356 | ], 1357 | "type": "time" 1358 | }, 1359 | { 1360 | "params": [ 1361 | "null" 1362 | ], 1363 | "type": "fill" 1364 | } 1365 | ], 1366 | "measurement": "BytesInPerSecPerTopic", 1367 | "policy": "default", 1368 | "refId": "C", 1369 | "resultFormat": "time_series", 1370 | "select": [ 1371 | [ 1372 | { 1373 | "params": [ 1374 | "FiveMinuteRate" 1375 | ], 1376 | "type": "field" 1377 | }, 1378 | { 1379 | "params": [], 1380 | "type": "last" 1381 | }, 1382 | { 1383 | "params": [ 1384 | "topic_t2" 1385 | ], 1386 | "type": "alias" 1387 | } 1388 | ] 1389 | ], 1390 | "tags": [ 1391 | { 1392 | "key": "typeName", 1393 | "operator": "=", 1394 | "value": "type=BrokerTopicMetrics,name=BytesInPerSec,topic=t2" 1395 | } 1396 | ] 1397 | } 1398 | ], 1399 | "timeFrom": null, 1400 | "timeShift": null, 1401 | "title": "Bytes In (5 minute rate)", 1402 | "tooltip": { 1403 | "msResolution": true, 1404 | "shared": true, 1405 | "sort": 0, 1406 | "value_type": "cumulative" 1407 | }, 1408 | "type": "graph", 1409 | "xaxis": { 1410 | "show": true 1411 | }, 1412 | "yaxes": [ 1413 | { 1414 | "format": "bytes", 1415 | "label": null, 1416 | "logBase": 1, 1417 | "max": null, 1418 | "min": null, 1419 | "show": true 1420 | }, 1421 | { 1422 | "format": "short", 1423 | "label": null, 1424 | "logBase": 1, 1425 | "max": null, 1426 | "min": null, 1427 | "show": true 1428 | } 1429 | ] 1430 | }, 1431 | { 1432 | "aliasColors": {}, 1433 | "bars": false, 1434 | "datasource": "${DS_INFLUX}", 1435 | "editable": true, 1436 | "error": false, 1437 | "fill": 1, 1438 | "grid": { 1439 | "threshold1": null, 1440 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 1441 | "threshold2": null, 1442 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 1443 | }, 1444 | "id": 17, 1445 | "isNew": true, 1446 | "legend": { 1447 | "avg": false, 1448 | "current": false, 1449 | "max": false, 1450 | "min": false, 1451 | "show": true, 1452 | "total": false, 1453 | "values": false 1454 | }, 1455 | "lines": true, 1456 | "linewidth": 2, 1457 | "links": [], 1458 | "nullPointMode": "connected", 1459 | "percentage": false, 1460 | "pointradius": 5, 1461 | "points": false, 1462 | "renderer": "flot", 1463 | "seriesOverrides": [], 1464 | "span": 6, 1465 | "stack": false, 1466 | "steppedLine": false, 1467 | "targets": [ 1468 | { 1469 | "dsType": "influxdb", 1470 | "groupBy": [ 1471 | { 1472 | "params": [ 1473 | "$interval" 1474 | ], 1475 | "type": "time" 1476 | }, 1477 | { 1478 | "params": [ 1479 | "null" 1480 | ], 1481 | "type": "fill" 1482 | } 1483 | ], 1484 | "measurement": "BytesOutPerSec", 1485 | "policy": "default", 1486 | "refId": "A", 1487 | "resultFormat": "time_series", 1488 | "select": [ 1489 | [ 1490 | { 1491 | "params": [ 1492 | "FiveMinuteRate" 1493 | ], 1494 | "type": "field" 1495 | }, 1496 | { 1497 | "params": [], 1498 | "type": "last" 1499 | } 1500 | ] 1501 | ], 1502 | "tags": [] 1503 | }, 1504 | { 1505 | "dsType": "influxdb", 1506 | "groupBy": [ 1507 | { 1508 | "params": [ 1509 | "$interval" 1510 | ], 1511 | "type": "time" 1512 | }, 1513 | { 1514 | "params": [ 1515 | "null" 1516 | ], 1517 | "type": "fill" 1518 | } 1519 | ], 1520 | "measurement": "BytesOutPerSecPerTopic", 1521 | "policy": "default", 1522 | "refId": "B", 1523 | "resultFormat": "time_series", 1524 | "select": [ 1525 | [ 1526 | { 1527 | "params": [ 1528 | "FiveMinuteRate" 1529 | ], 1530 | "type": "field" 1531 | }, 1532 | { 1533 | "params": [], 1534 | "type": "last" 1535 | }, 1536 | { 1537 | "params": [ 1538 | "topic_t1" 1539 | ], 1540 | "type": "alias" 1541 | } 1542 | ] 1543 | ], 1544 | "tags": [ 1545 | { 1546 | "key": "typeName", 1547 | "operator": "=", 1548 | "value": "type=BrokerTopicMetrics,name=BytesOutPerSec,topic=t1" 1549 | } 1550 | ] 1551 | }, 1552 | { 1553 | "dsType": "influxdb", 1554 | "groupBy": [ 1555 | { 1556 | "params": [ 1557 | "$interval" 1558 | ], 1559 | "type": "time" 1560 | }, 1561 | { 1562 | "params": [ 1563 | "null" 1564 | ], 1565 | "type": "fill" 1566 | } 1567 | ], 1568 | "measurement": "BytesOutPerSecPerTopic", 1569 | "policy": "default", 1570 | "refId": "C", 1571 | "resultFormat": "time_series", 1572 | "select": [ 1573 | [ 1574 | { 1575 | "params": [ 1576 | "FiveMinuteRate" 1577 | ], 1578 | "type": "field" 1579 | }, 1580 | { 1581 | "params": [], 1582 | "type": "last" 1583 | }, 1584 | { 1585 | "params": [ 1586 | "topic_t2" 1587 | ], 1588 | "type": "alias" 1589 | } 1590 | ] 1591 | ], 1592 | "tags": [ 1593 | { 1594 | "key": "typeName", 1595 | "operator": "=", 1596 | "value": "type=BrokerTopicMetrics,name=BytesOutPerSec,topic=t2" 1597 | } 1598 | ] 1599 | } 1600 | ], 1601 | "timeFrom": null, 1602 | "timeShift": null, 1603 | "title": "Bytes Out (5 minute rate)", 1604 | "tooltip": { 1605 | "msResolution": true, 1606 | "shared": true, 1607 | "sort": 0, 1608 | "value_type": "cumulative" 1609 | }, 1610 | "type": "graph", 1611 | "xaxis": { 1612 | "show": true 1613 | }, 1614 | "yaxes": [ 1615 | { 1616 | "format": "bytes", 1617 | "label": null, 1618 | "logBase": 1, 1619 | "max": null, 1620 | "min": null, 1621 | "show": true 1622 | }, 1623 | { 1624 | "format": "short", 1625 | "label": null, 1626 | "logBase": 1, 1627 | "max": null, 1628 | "min": null, 1629 | "show": true 1630 | } 1631 | ] 1632 | } 1633 | ], 1634 | "showTitle": true, 1635 | "title": "Critical" 1636 | }, 1637 | { 1638 | "collapse": false, 1639 | "editable": true, 1640 | "height": "250px", 1641 | "panels": [ 1642 | { 1643 | "aliasColors": {}, 1644 | "bars": false, 1645 | "datasource": "${DS_INFLUX}", 1646 | "editable": true, 1647 | "error": false, 1648 | "fill": 1, 1649 | "grid": { 1650 | "threshold1": null, 1651 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 1652 | "threshold2": null, 1653 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 1654 | }, 1655 | "id": 8, 1656 | "isNew": true, 1657 | "legend": { 1658 | "avg": false, 1659 | "current": false, 1660 | "max": false, 1661 | "min": false, 1662 | "show": true, 1663 | "total": false, 1664 | "values": false 1665 | }, 1666 | "lines": true, 1667 | "linewidth": 2, 1668 | "links": [], 1669 | "nullPointMode": "connected", 1670 | "percentage": false, 1671 | "pointradius": 5, 1672 | "points": false, 1673 | "renderer": "flot", 1674 | "seriesOverrides": [], 1675 | "span": 6, 1676 | "stack": false, 1677 | "steppedLine": false, 1678 | "targets": [ 1679 | { 1680 | "dsType": "influxdb", 1681 | "groupBy": [ 1682 | { 1683 | "params": [ 1684 | "$interval" 1685 | ], 1686 | "type": "time" 1687 | }, 1688 | { 1689 | "params": [ 1690 | "null" 1691 | ], 1692 | "type": "fill" 1693 | } 1694 | ], 1695 | "measurement": "PartitionCount", 1696 | "policy": "default", 1697 | "refId": "A", 1698 | "resultFormat": "time_series", 1699 | "select": [ 1700 | [ 1701 | { 1702 | "params": [ 1703 | "Value" 1704 | ], 1705 | "type": "field" 1706 | }, 1707 | { 1708 | "params": [], 1709 | "type": "last" 1710 | } 1711 | ] 1712 | ], 1713 | "tags": [] 1714 | }, 1715 | { 1716 | "dsType": "influxdb", 1717 | "groupBy": [ 1718 | { 1719 | "params": [ 1720 | "$interval" 1721 | ], 1722 | "type": "time" 1723 | }, 1724 | { 1725 | "params": [ 1726 | "null" 1727 | ], 1728 | "type": "fill" 1729 | } 1730 | ], 1731 | "measurement": "LeaderCount", 1732 | "policy": "default", 1733 | "refId": "B", 1734 | "resultFormat": "time_series", 1735 | "select": [ 1736 | [ 1737 | { 1738 | "params": [ 1739 | "Value" 1740 | ], 1741 | "type": "field" 1742 | }, 1743 | { 1744 | "params": [], 1745 | "type": "last" 1746 | } 1747 | ] 1748 | ], 1749 | "tags": [] 1750 | }, 1751 | { 1752 | "dsType": "influxdb", 1753 | "groupBy": [ 1754 | { 1755 | "params": [ 1756 | "$interval" 1757 | ], 1758 | "type": "time" 1759 | }, 1760 | { 1761 | "params": [ 1762 | "null" 1763 | ], 1764 | "type": "fill" 1765 | } 1766 | ], 1767 | "measurement": "OfflinePartitionsCount", 1768 | "policy": "default", 1769 | "refId": "C", 1770 | "resultFormat": "time_series", 1771 | "select": [ 1772 | [ 1773 | { 1774 | "params": [ 1775 | "Value" 1776 | ], 1777 | "type": "field" 1778 | }, 1779 | { 1780 | "params": [], 1781 | "type": "last" 1782 | } 1783 | ] 1784 | ], 1785 | "tags": [] 1786 | } 1787 | ], 1788 | "timeFrom": null, 1789 | "timeShift": null, 1790 | "title": "Partition / Leader Count", 1791 | "tooltip": { 1792 | "msResolution": true, 1793 | "shared": true, 1794 | "sort": 0, 1795 | "value_type": "cumulative" 1796 | }, 1797 | "type": "graph", 1798 | "xaxis": { 1799 | "show": true 1800 | }, 1801 | "yaxes": [ 1802 | { 1803 | "format": "short", 1804 | "label": null, 1805 | "logBase": 1, 1806 | "max": null, 1807 | "min": null, 1808 | "show": true 1809 | }, 1810 | { 1811 | "format": "short", 1812 | "label": null, 1813 | "logBase": 1, 1814 | "max": null, 1815 | "min": null, 1816 | "show": true 1817 | } 1818 | ] 1819 | }, 1820 | { 1821 | "aliasColors": {}, 1822 | "bars": false, 1823 | "datasource": "${DS_INFLUX}", 1824 | "editable": true, 1825 | "error": false, 1826 | "fill": 1, 1827 | "grid": { 1828 | "threshold1": null, 1829 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 1830 | "threshold2": null, 1831 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 1832 | }, 1833 | "id": 9, 1834 | "isNew": true, 1835 | "legend": { 1836 | "avg": false, 1837 | "current": false, 1838 | "max": false, 1839 | "min": false, 1840 | "show": true, 1841 | "total": false, 1842 | "values": false 1843 | }, 1844 | "lines": true, 1845 | "linewidth": 2, 1846 | "links": [], 1847 | "nullPointMode": "connected", 1848 | "percentage": false, 1849 | "pointradius": 5, 1850 | "points": false, 1851 | "renderer": "flot", 1852 | "seriesOverrides": [], 1853 | "span": 6, 1854 | "stack": false, 1855 | "steppedLine": false, 1856 | "targets": [ 1857 | { 1858 | "dsType": "influxdb", 1859 | "groupBy": [ 1860 | { 1861 | "params": [ 1862 | "$interval" 1863 | ], 1864 | "type": "time" 1865 | }, 1866 | { 1867 | "params": [ 1868 | "null" 1869 | ], 1870 | "type": "fill" 1871 | } 1872 | ], 1873 | "measurement": "RequestHandlerAvgIdlePercent", 1874 | "policy": "default", 1875 | "refId": "A", 1876 | "resultFormat": "time_series", 1877 | "select": [ 1878 | [ 1879 | { 1880 | "params": [ 1881 | "MeanRate" 1882 | ], 1883 | "type": "field" 1884 | }, 1885 | { 1886 | "params": [], 1887 | "type": "mean" 1888 | } 1889 | ] 1890 | ], 1891 | "tags": [] 1892 | } 1893 | ], 1894 | "timeFrom": null, 1895 | "timeShift": null, 1896 | "title": "Request Handler avg idle", 1897 | "tooltip": { 1898 | "msResolution": false, 1899 | "shared": true, 1900 | "sort": 0, 1901 | "value_type": "cumulative" 1902 | }, 1903 | "type": "graph", 1904 | "xaxis": { 1905 | "show": true 1906 | }, 1907 | "yaxes": [ 1908 | { 1909 | "format": "short", 1910 | "label": null, 1911 | "logBase": 1, 1912 | "max": null, 1913 | "min": null, 1914 | "show": true 1915 | }, 1916 | { 1917 | "format": "short", 1918 | "label": null, 1919 | "logBase": 1, 1920 | "max": null, 1921 | "min": null, 1922 | "show": true 1923 | } 1924 | ] 1925 | } 1926 | ], 1927 | "showTitle": true, 1928 | "title": "Misc" 1929 | }, 1930 | { 1931 | "collapse": false, 1932 | "editable": true, 1933 | "height": "250px", 1934 | "panels": [ 1935 | { 1936 | "aliasColors": {}, 1937 | "bars": false, 1938 | "datasource": "${DS_INFLUX}", 1939 | "editable": true, 1940 | "error": false, 1941 | "fill": 1, 1942 | "grid": { 1943 | "threshold1": null, 1944 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 1945 | "threshold2": null, 1946 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 1947 | }, 1948 | "id": 10, 1949 | "isNew": true, 1950 | "legend": { 1951 | "avg": false, 1952 | "current": false, 1953 | "max": false, 1954 | "min": false, 1955 | "show": true, 1956 | "total": false, 1957 | "values": false 1958 | }, 1959 | "lines": true, 1960 | "linewidth": 2, 1961 | "links": [], 1962 | "nullPointMode": "connected", 1963 | "percentage": false, 1964 | "pointradius": 5, 1965 | "points": false, 1966 | "renderer": "flot", 1967 | "seriesOverrides": [], 1968 | "span": 6, 1969 | "stack": false, 1970 | "steppedLine": false, 1971 | "targets": [ 1972 | { 1973 | "dsType": "influxdb", 1974 | "groupBy": [ 1975 | { 1976 | "params": [ 1977 | "$interval" 1978 | ], 1979 | "type": "time" 1980 | }, 1981 | { 1982 | "params": [ 1983 | "null" 1984 | ], 1985 | "type": "fill" 1986 | } 1987 | ], 1988 | "measurement": "LeaderElectionRateAndTimeMs", 1989 | "policy": "default", 1990 | "refId": "A", 1991 | "resultFormat": "time_series", 1992 | "select": [ 1993 | [ 1994 | { 1995 | "params": [ 1996 | "Count" 1997 | ], 1998 | "type": "field" 1999 | }, 2000 | { 2001 | "params": [], 2002 | "type": "mean" 2003 | } 2004 | ] 2005 | ], 2006 | "tags": [] 2007 | }, 2008 | { 2009 | "dsType": "influxdb", 2010 | "groupBy": [ 2011 | { 2012 | "params": [ 2013 | "$interval" 2014 | ], 2015 | "type": "time" 2016 | }, 2017 | { 2018 | "params": [ 2019 | "null" 2020 | ], 2021 | "type": "fill" 2022 | } 2023 | ], 2024 | "measurement": "UncleanLeaderElectionsPerSec", 2025 | "policy": "default", 2026 | "refId": "B", 2027 | "resultFormat": "time_series", 2028 | "select": [ 2029 | [ 2030 | { 2031 | "params": [ 2032 | "Count" 2033 | ], 2034 | "type": "field" 2035 | }, 2036 | { 2037 | "params": [], 2038 | "type": "mean" 2039 | } 2040 | ] 2041 | ], 2042 | "tags": [] 2043 | } 2044 | ], 2045 | "timeFrom": null, 2046 | "timeShift": null, 2047 | "title": "Leader Election", 2048 | "tooltip": { 2049 | "msResolution": false, 2050 | "shared": true, 2051 | "sort": 0, 2052 | "value_type": "cumulative" 2053 | }, 2054 | "type": "graph", 2055 | "xaxis": { 2056 | "show": true 2057 | }, 2058 | "yaxes": [ 2059 | { 2060 | "format": "short", 2061 | "label": null, 2062 | "logBase": 1, 2063 | "max": null, 2064 | "min": null, 2065 | "show": true 2066 | }, 2067 | { 2068 | "format": "short", 2069 | "label": null, 2070 | "logBase": 1, 2071 | "max": null, 2072 | "min": null, 2073 | "show": true 2074 | } 2075 | ] 2076 | }, 2077 | { 2078 | "aliasColors": {}, 2079 | "bars": false, 2080 | "datasource": "${DS_INFLUX}", 2081 | "editable": true, 2082 | "error": false, 2083 | "fill": 1, 2084 | "grid": { 2085 | "threshold1": null, 2086 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 2087 | "threshold2": null, 2088 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 2089 | }, 2090 | "id": 11, 2091 | "isNew": true, 2092 | "legend": { 2093 | "avg": false, 2094 | "current": false, 2095 | "max": false, 2096 | "min": false, 2097 | "show": true, 2098 | "total": false, 2099 | "values": false 2100 | }, 2101 | "lines": true, 2102 | "linewidth": 2, 2103 | "links": [], 2104 | "nullPointMode": "connected", 2105 | "percentage": false, 2106 | "pointradius": 5, 2107 | "points": false, 2108 | "renderer": "flot", 2109 | "seriesOverrides": [], 2110 | "span": 6, 2111 | "stack": false, 2112 | "steppedLine": false, 2113 | "targets": [ 2114 | { 2115 | "dsType": "influxdb", 2116 | "groupBy": [ 2117 | { 2118 | "params": [ 2119 | "$interval" 2120 | ], 2121 | "type": "time" 2122 | }, 2123 | { 2124 | "params": [ 2125 | "null" 2126 | ], 2127 | "type": "fill" 2128 | } 2129 | ], 2130 | "measurement": "ActiveControllerCount", 2131 | "policy": "default", 2132 | "refId": "A", 2133 | "resultFormat": "time_series", 2134 | "select": [ 2135 | [ 2136 | { 2137 | "params": [ 2138 | "Value" 2139 | ], 2140 | "type": "field" 2141 | }, 2142 | { 2143 | "params": [], 2144 | "type": "mean" 2145 | } 2146 | ] 2147 | ], 2148 | "tags": [] 2149 | } 2150 | ], 2151 | "timeFrom": null, 2152 | "timeShift": null, 2153 | "title": "Active Controllers", 2154 | "tooltip": { 2155 | "msResolution": false, 2156 | "shared": true, 2157 | "sort": 0, 2158 | "value_type": "cumulative" 2159 | }, 2160 | "type": "graph", 2161 | "xaxis": { 2162 | "show": true 2163 | }, 2164 | "yaxes": [ 2165 | { 2166 | "format": "short", 2167 | "label": null, 2168 | "logBase": 1, 2169 | "max": null, 2170 | "min": null, 2171 | "show": true 2172 | }, 2173 | { 2174 | "format": "short", 2175 | "label": null, 2176 | "logBase": 1, 2177 | "max": null, 2178 | "min": null, 2179 | "show": true 2180 | } 2181 | ] 2182 | } 2183 | ], 2184 | "showTitle": true, 2185 | "title": "Controller" 2186 | }, 2187 | { 2188 | "collapse": false, 2189 | "editable": true, 2190 | "height": "250px", 2191 | "panels": [ 2192 | { 2193 | "aliasColors": {}, 2194 | "bars": false, 2195 | "datasource": "${DS_INFLUX}", 2196 | "editable": true, 2197 | "error": false, 2198 | "fill": 1, 2199 | "grid": { 2200 | "threshold1": null, 2201 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 2202 | "threshold2": null, 2203 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 2204 | }, 2205 | "id": 12, 2206 | "isNew": true, 2207 | "legend": { 2208 | "avg": false, 2209 | "current": false, 2210 | "max": false, 2211 | "min": false, 2212 | "show": true, 2213 | "total": false, 2214 | "values": false 2215 | }, 2216 | "lines": true, 2217 | "linewidth": 2, 2218 | "links": [], 2219 | "nullPointMode": "connected", 2220 | "percentage": false, 2221 | "pointradius": 5, 2222 | "points": false, 2223 | "renderer": "flot", 2224 | "seriesOverrides": [], 2225 | "span": 4, 2226 | "stack": false, 2227 | "steppedLine": false, 2228 | "targets": [ 2229 | { 2230 | "dsType": "influxdb", 2231 | "groupBy": [ 2232 | { 2233 | "params": [ 2234 | "$interval" 2235 | ], 2236 | "type": "time" 2237 | }, 2238 | { 2239 | "params": [ 2240 | "null" 2241 | ], 2242 | "type": "fill" 2243 | } 2244 | ], 2245 | "measurement": "ProduceRequestsPerSec", 2246 | "policy": "default", 2247 | "refId": "A", 2248 | "resultFormat": "time_series", 2249 | "select": [ 2250 | [ 2251 | { 2252 | "params": [ 2253 | "OneMinuteRate" 2254 | ], 2255 | "type": "field" 2256 | }, 2257 | { 2258 | "params": [], 2259 | "type": "mean" 2260 | } 2261 | ] 2262 | ], 2263 | "tags": [] 2264 | }, 2265 | { 2266 | "dsType": "influxdb", 2267 | "groupBy": [ 2268 | { 2269 | "params": [ 2270 | "$interval" 2271 | ], 2272 | "type": "time" 2273 | }, 2274 | { 2275 | "params": [ 2276 | "null" 2277 | ], 2278 | "type": "fill" 2279 | } 2280 | ], 2281 | "measurement": "FetchConsumerRequestsPerSec", 2282 | "policy": "default", 2283 | "refId": "B", 2284 | "resultFormat": "time_series", 2285 | "select": [ 2286 | [ 2287 | { 2288 | "params": [ 2289 | "OneMinuteRate" 2290 | ], 2291 | "type": "field" 2292 | }, 2293 | { 2294 | "params": [], 2295 | "type": "mean" 2296 | } 2297 | ] 2298 | ], 2299 | "tags": [] 2300 | }, 2301 | { 2302 | "dsType": "influxdb", 2303 | "groupBy": [ 2304 | { 2305 | "params": [ 2306 | "$interval" 2307 | ], 2308 | "type": "time" 2309 | }, 2310 | { 2311 | "params": [ 2312 | "null" 2313 | ], 2314 | "type": "fill" 2315 | } 2316 | ], 2317 | "measurement": "FetchFollowerRequestsPerSec", 2318 | "policy": "default", 2319 | "refId": "C", 2320 | "resultFormat": "time_series", 2321 | "select": [ 2322 | [ 2323 | { 2324 | "params": [ 2325 | "OneMinuteRate" 2326 | ], 2327 | "type": "field" 2328 | }, 2329 | { 2330 | "params": [], 2331 | "type": "mean" 2332 | } 2333 | ] 2334 | ], 2335 | "tags": [] 2336 | } 2337 | ], 2338 | "timeFrom": null, 2339 | "timeShift": null, 2340 | "title": "Requests 1 minute rate", 2341 | "tooltip": { 2342 | "msResolution": false, 2343 | "shared": true, 2344 | "sort": 0, 2345 | "value_type": "cumulative" 2346 | }, 2347 | "type": "graph", 2348 | "xaxis": { 2349 | "show": true 2350 | }, 2351 | "yaxes": [ 2352 | { 2353 | "format": "short", 2354 | "label": null, 2355 | "logBase": 1, 2356 | "max": null, 2357 | "min": null, 2358 | "show": true 2359 | }, 2360 | { 2361 | "format": "short", 2362 | "label": null, 2363 | "logBase": 1, 2364 | "max": null, 2365 | "min": null, 2366 | "show": true 2367 | } 2368 | ] 2369 | }, 2370 | { 2371 | "aliasColors": {}, 2372 | "bars": false, 2373 | "datasource": "${DS_INFLUX}", 2374 | "editable": true, 2375 | "error": false, 2376 | "fill": 1, 2377 | "grid": { 2378 | "threshold1": null, 2379 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 2380 | "threshold2": null, 2381 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 2382 | }, 2383 | "id": 13, 2384 | "isNew": true, 2385 | "legend": { 2386 | "avg": false, 2387 | "current": false, 2388 | "max": false, 2389 | "min": false, 2390 | "show": true, 2391 | "total": false, 2392 | "values": false 2393 | }, 2394 | "lines": true, 2395 | "linewidth": 2, 2396 | "links": [], 2397 | "nullPointMode": "connected", 2398 | "percentage": false, 2399 | "pointradius": 5, 2400 | "points": false, 2401 | "renderer": "flot", 2402 | "seriesOverrides": [], 2403 | "span": 4, 2404 | "stack": false, 2405 | "steppedLine": false, 2406 | "targets": [ 2407 | { 2408 | "dsType": "influxdb", 2409 | "groupBy": [ 2410 | { 2411 | "params": [ 2412 | "$interval" 2413 | ], 2414 | "type": "time" 2415 | }, 2416 | { 2417 | "params": [ 2418 | "null" 2419 | ], 2420 | "type": "fill" 2421 | } 2422 | ], 2423 | "measurement": "ProduceTotalTimeMs", 2424 | "policy": "default", 2425 | "refId": "A", 2426 | "resultFormat": "time_series", 2427 | "select": [ 2428 | [ 2429 | { 2430 | "params": [ 2431 | "Mean" 2432 | ], 2433 | "type": "field" 2434 | }, 2435 | { 2436 | "params": [], 2437 | "type": "mean" 2438 | } 2439 | ] 2440 | ], 2441 | "tags": [] 2442 | }, 2443 | { 2444 | "dsType": "influxdb", 2445 | "groupBy": [ 2446 | { 2447 | "params": [ 2448 | "$interval" 2449 | ], 2450 | "type": "time" 2451 | }, 2452 | { 2453 | "params": [ 2454 | "null" 2455 | ], 2456 | "type": "fill" 2457 | } 2458 | ], 2459 | "measurement": "FetchConsumerTotalTimeMs", 2460 | "policy": "default", 2461 | "refId": "B", 2462 | "resultFormat": "time_series", 2463 | "select": [ 2464 | [ 2465 | { 2466 | "params": [ 2467 | "Mean" 2468 | ], 2469 | "type": "field" 2470 | }, 2471 | { 2472 | "params": [], 2473 | "type": "mean" 2474 | } 2475 | ] 2476 | ], 2477 | "tags": [] 2478 | }, 2479 | { 2480 | "dsType": "influxdb", 2481 | "groupBy": [ 2482 | { 2483 | "params": [ 2484 | "$interval" 2485 | ], 2486 | "type": "time" 2487 | }, 2488 | { 2489 | "params": [ 2490 | "null" 2491 | ], 2492 | "type": "fill" 2493 | } 2494 | ], 2495 | "measurement": "FetchFollowerTotalTimeMs", 2496 | "policy": "default", 2497 | "refId": "C", 2498 | "resultFormat": "time_series", 2499 | "select": [ 2500 | [ 2501 | { 2502 | "params": [ 2503 | "Mean" 2504 | ], 2505 | "type": "field" 2506 | }, 2507 | { 2508 | "params": [], 2509 | "type": "mean" 2510 | } 2511 | ] 2512 | ], 2513 | "tags": [] 2514 | } 2515 | ], 2516 | "timeFrom": null, 2517 | "timeShift": null, 2518 | "title": "Requests Total Time", 2519 | "tooltip": { 2520 | "msResolution": false, 2521 | "shared": true, 2522 | "sort": 0, 2523 | "value_type": "cumulative" 2524 | }, 2525 | "type": "graph", 2526 | "xaxis": { 2527 | "show": true 2528 | }, 2529 | "yaxes": [ 2530 | { 2531 | "format": "short", 2532 | "label": null, 2533 | "logBase": 1, 2534 | "max": null, 2535 | "min": null, 2536 | "show": true 2537 | }, 2538 | { 2539 | "format": "short", 2540 | "label": null, 2541 | "logBase": 1, 2542 | "max": null, 2543 | "min": null, 2544 | "show": true 2545 | } 2546 | ] 2547 | }, 2548 | { 2549 | "aliasColors": {}, 2550 | "bars": false, 2551 | "datasource": "${DS_INFLUX}", 2552 | "editable": true, 2553 | "error": false, 2554 | "fill": 1, 2555 | "grid": { 2556 | "threshold1": null, 2557 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 2558 | "threshold2": null, 2559 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 2560 | }, 2561 | "id": 14, 2562 | "isNew": true, 2563 | "legend": { 2564 | "avg": false, 2565 | "current": false, 2566 | "max": false, 2567 | "min": false, 2568 | "show": true, 2569 | "total": false, 2570 | "values": false 2571 | }, 2572 | "lines": true, 2573 | "linewidth": 2, 2574 | "links": [], 2575 | "nullPointMode": "connected", 2576 | "percentage": false, 2577 | "pointradius": 5, 2578 | "points": false, 2579 | "renderer": "flot", 2580 | "seriesOverrides": [], 2581 | "span": 4, 2582 | "stack": false, 2583 | "steppedLine": false, 2584 | "targets": [ 2585 | { 2586 | "dsType": "influxdb", 2587 | "groupBy": [ 2588 | { 2589 | "params": [ 2590 | "$interval" 2591 | ], 2592 | "type": "time" 2593 | }, 2594 | { 2595 | "params": [ 2596 | "null" 2597 | ], 2598 | "type": "fill" 2599 | } 2600 | ], 2601 | "measurement": "ProduceRequestQueueTimeMs", 2602 | "policy": "default", 2603 | "refId": "A", 2604 | "resultFormat": "time_series", 2605 | "select": [ 2606 | [ 2607 | { 2608 | "params": [ 2609 | "Mean" 2610 | ], 2611 | "type": "field" 2612 | }, 2613 | { 2614 | "params": [], 2615 | "type": "mean" 2616 | } 2617 | ] 2618 | ], 2619 | "tags": [] 2620 | }, 2621 | { 2622 | "dsType": "influxdb", 2623 | "groupBy": [ 2624 | { 2625 | "params": [ 2626 | "$interval" 2627 | ], 2628 | "type": "time" 2629 | }, 2630 | { 2631 | "params": [ 2632 | "null" 2633 | ], 2634 | "type": "fill" 2635 | } 2636 | ], 2637 | "measurement": "FetchConsumerRequestQueueTimeMs", 2638 | "policy": "default", 2639 | "refId": "B", 2640 | "resultFormat": "time_series", 2641 | "select": [ 2642 | [ 2643 | { 2644 | "params": [ 2645 | "Mean" 2646 | ], 2647 | "type": "field" 2648 | }, 2649 | { 2650 | "params": [], 2651 | "type": "mean" 2652 | } 2653 | ] 2654 | ], 2655 | "tags": [] 2656 | }, 2657 | { 2658 | "dsType": "influxdb", 2659 | "groupBy": [ 2660 | { 2661 | "params": [ 2662 | "$interval" 2663 | ], 2664 | "type": "time" 2665 | }, 2666 | { 2667 | "params": [ 2668 | "null" 2669 | ], 2670 | "type": "fill" 2671 | } 2672 | ], 2673 | "measurement": "FetchFollowerRequestQueueTimeMs", 2674 | "policy": "default", 2675 | "refId": "C", 2676 | "resultFormat": "time_series", 2677 | "select": [ 2678 | [ 2679 | { 2680 | "params": [ 2681 | "Mean" 2682 | ], 2683 | "type": "field" 2684 | }, 2685 | { 2686 | "params": [], 2687 | "type": "mean" 2688 | } 2689 | ] 2690 | ], 2691 | "tags": [] 2692 | } 2693 | ], 2694 | "timeFrom": null, 2695 | "timeShift": null, 2696 | "title": "Request Queue Times", 2697 | "tooltip": { 2698 | "msResolution": false, 2699 | "shared": true, 2700 | "sort": 0, 2701 | "value_type": "cumulative" 2702 | }, 2703 | "type": "graph", 2704 | "xaxis": { 2705 | "show": true 2706 | }, 2707 | "yaxes": [ 2708 | { 2709 | "format": "short", 2710 | "label": null, 2711 | "logBase": 1, 2712 | "max": null, 2713 | "min": null, 2714 | "show": true 2715 | }, 2716 | { 2717 | "format": "short", 2718 | "label": null, 2719 | "logBase": 1, 2720 | "max": null, 2721 | "min": null, 2722 | "show": true 2723 | } 2724 | ] 2725 | }, 2726 | { 2727 | "aliasColors": {}, 2728 | "bars": false, 2729 | "datasource": "${DS_INFLUX}", 2730 | "editable": true, 2731 | "error": false, 2732 | "fill": 1, 2733 | "grid": { 2734 | "threshold1": null, 2735 | "threshold1Color": "rgba(216, 200, 27, 0.27)", 2736 | "threshold2": null, 2737 | "threshold2Color": "rgba(234, 112, 112, 0.22)" 2738 | }, 2739 | "id": 15, 2740 | "isNew": true, 2741 | "legend": { 2742 | "avg": false, 2743 | "current": false, 2744 | "max": false, 2745 | "min": false, 2746 | "show": true, 2747 | "total": false, 2748 | "values": false 2749 | }, 2750 | "lines": true, 2751 | "linewidth": 2, 2752 | "links": [], 2753 | "nullPointMode": "connected", 2754 | "percentage": false, 2755 | "pointradius": 5, 2756 | "points": false, 2757 | "renderer": "flot", 2758 | "seriesOverrides": [], 2759 | "span": 12, 2760 | "stack": false, 2761 | "steppedLine": false, 2762 | "targets": [ 2763 | { 2764 | "dsType": "influxdb", 2765 | "groupBy": [ 2766 | { 2767 | "params": [ 2768 | "$interval" 2769 | ], 2770 | "type": "time" 2771 | }, 2772 | { 2773 | "params": [ 2774 | "null" 2775 | ], 2776 | "type": "fill" 2777 | } 2778 | ], 2779 | "measurement": "NetworkProcessorAvgIdlePercent ", 2780 | "policy": "default", 2781 | "refId": "A", 2782 | "resultFormat": "time_series", 2783 | "select": [ 2784 | [ 2785 | { 2786 | "params": [ 2787 | "Value" 2788 | ], 2789 | "type": "field" 2790 | }, 2791 | { 2792 | "params": [], 2793 | "type": "last" 2794 | } 2795 | ] 2796 | ], 2797 | "tags": [] 2798 | } 2799 | ], 2800 | "timeFrom": null, 2801 | "timeShift": null, 2802 | "title": "Network Processor avg idle", 2803 | "tooltip": { 2804 | "msResolution": false, 2805 | "shared": true, 2806 | "sort": 0, 2807 | "value_type": "cumulative" 2808 | }, 2809 | "type": "graph", 2810 | "xaxis": { 2811 | "show": true 2812 | }, 2813 | "yaxes": [ 2814 | { 2815 | "format": "short", 2816 | "label": null, 2817 | "logBase": 1, 2818 | "max": null, 2819 | "min": null, 2820 | "show": true 2821 | }, 2822 | { 2823 | "format": "short", 2824 | "label": null, 2825 | "logBase": 1, 2826 | "max": null, 2827 | "min": null, 2828 | "show": true 2829 | } 2830 | ] 2831 | } 2832 | ], 2833 | "showTitle": true, 2834 | "title": "Network" 2835 | } 2836 | ], 2837 | "time": { 2838 | "from": "now-1h", 2839 | "to": "now" 2840 | }, 2841 | "timepicker": { 2842 | "refresh_intervals": [ 2843 | "5s", 2844 | "10s", 2845 | "30s", 2846 | "1m", 2847 | "5m", 2848 | "15m", 2849 | "30m", 2850 | "1h", 2851 | "2h", 2852 | "1d" 2853 | ], 2854 | "time_options": [ 2855 | "5m", 2856 | "15m", 2857 | "1h", 2858 | "6h", 2859 | "12h", 2860 | "24h", 2861 | "2d", 2862 | "7d", 2863 | "30d" 2864 | ] 2865 | }, 2866 | "templating": { 2867 | "list": [] 2868 | }, 2869 | "annotations": { 2870 | "list": [] 2871 | }, 2872 | "refresh": false, 2873 | "schemaVersion": 12, 2874 | "version": 24, 2875 | "links": [], 2876 | "gnetId": null 2877 | } -------------------------------------------------------------------------------- /metrics/kafka/kafka.json: -------------------------------------------------------------------------------- 1 | { 2 | "servers": [ 3 | { 4 | "port": "${port1}", 5 | "host": "${url1}", 6 | "alias": "kafka-node-1", 7 | "queries": [ 8 | { 9 | "outputWriters": [ 10 | { 11 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 12 | "url": "${influxUrl}", 13 | "database": "${influxDb}", 14 | "username": "${influxUser}", 15 | "password": "${influxPwd}" 16 | } 17 | ], 18 | "obj": "java.lang:type=Memory", 19 | "attr": [ 20 | "HeapMemoryUsage", 21 | "NonHeapMemoryUsage" 22 | ], 23 | "resultAlias": "jvmMemory" 24 | }, 25 | { 26 | "outputWriters": [ 27 | { 28 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 29 | "url": "${influxUrl}", 30 | "database": "${influxDb}", 31 | "username": "${influxUser}", 32 | "password": "${influxPwd}" 33 | } 34 | ], 35 | "obj": "java.lang:type=OperatingSystem", 36 | "attr": [ 37 | "FreePhysicalMemorySize", 38 | "ProcessCpuLoad", 39 | "SystemCpuLoad", 40 | "SystemLoadAverage" 41 | ], 42 | "resultAlias": "os" 43 | }, 44 | { 45 | "outputWriters": [ 46 | { 47 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 48 | "url": "${influxUrl}", 49 | "database": "${influxDb}", 50 | "username": "${influxUser}", 51 | "password": "${influxPwd}" 52 | } 53 | ], 54 | "obj": "kafka.server:type=ReplicaManager,name=UnderReplicatedPartitions", 55 | "attr": [ 56 | "Value" 57 | ], 58 | "resultAlias": "UnderReplicatedPartitions" 59 | }, 60 | { 61 | "outputWriters": [ 62 | { 63 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 64 | "url": "${influxUrl}", 65 | "database": "${influxDb}", 66 | "username": "${influxUser}", 67 | "password": "${influxPwd}" 68 | } 69 | ], 70 | "obj": "kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec", 71 | "attr": [ 72 | "Count", 73 | "MeanRate", 74 | "OneMinuteRate", 75 | "FiveMinuteRate", 76 | "FifteenMinuteRate" 77 | ], 78 | "resultAlias": "MessagesInPerSec" 79 | }, 80 | { 81 | "outputWriters": [ 82 | { 83 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 84 | "url": "${influxUrl}", 85 | "database": "${influxDb}", 86 | "username": "${influxUser}", 87 | "password": "${influxPwd}" 88 | } 89 | ], 90 | "obj": "kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec,topic=*", 91 | "attr": [ 92 | "Count", 93 | "MeanRate", 94 | "OneMinuteRate", 95 | "FiveMinuteRate", 96 | "FifteenMinuteRate" 97 | ], 98 | "resultAlias": "MessagesInPerSecPerTopic" 99 | }, 100 | { 101 | "outputWriters": [ 102 | { 103 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 104 | "url": "${influxUrl}", 105 | "database": "${influxDb}", 106 | "username": "${influxUser}", 107 | "password": "${influxPwd}" 108 | } 109 | ], 110 | "obj": "kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec", 111 | "attr": [ 112 | "Count", 113 | "MeanRate", 114 | "OneMinuteRate", 115 | "FiveMinuteRate", 116 | "FifteenMinuteRate" 117 | ], 118 | "resultAlias": "BytesInPerSec" 119 | }, 120 | { 121 | "outputWriters": [ 122 | { 123 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 124 | "url": "${influxUrl}", 125 | "database": "${influxDb}", 126 | "username": "${influxUser}", 127 | "password": "${influxPwd}" 128 | } 129 | ], 130 | "obj": "kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec,topic=*", 131 | "attr": [ 132 | "Count", 133 | "MeanRate", 134 | "OneMinuteRate", 135 | "FiveMinuteRate", 136 | "FifteenMinuteRate" 137 | ], 138 | "resultAlias": "BytesInPerSecPerTopic" 139 | }, 140 | { 141 | "outputWriters": [ 142 | { 143 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 144 | "url": "${influxUrl}", 145 | "database": "${influxDb}", 146 | "username": "${influxUser}", 147 | "password": "${influxPwd}" 148 | } 149 | ], 150 | "obj": "kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec", 151 | "attr": [ 152 | "Count", 153 | "MeanRate", 154 | "OneMinuteRate", 155 | "FiveMinuteRate", 156 | "FifteenMinuteRate" 157 | ], 158 | "resultAlias": "BytesOutPerSec" 159 | }, 160 | { 161 | "outputWriters": [ 162 | { 163 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 164 | "url": "${influxUrl}", 165 | "database": "${influxDb}", 166 | "username": "${influxUser}", 167 | "password": "${influxPwd}" 168 | } 169 | ], 170 | "obj": "kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec,topic=*", 171 | "attr": [ 172 | "Count", 173 | "MeanRate", 174 | "OneMinuteRate", 175 | "FiveMinuteRate", 176 | "FifteenMinuteRate" 177 | ], 178 | "resultAlias": "BytesOutPerSecPerTopic" 179 | }, 180 | { 181 | "outputWriters": [ 182 | { 183 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 184 | "url": "${influxUrl}", 185 | "database": "${influxDb}", 186 | "username": "${influxUser}", 187 | "password": "${influxPwd}" 188 | } 189 | ], 190 | "obj": "kafka.server:type=ReplicaManager,name=PartitionCount", 191 | "attr": [ 192 | "Value" 193 | ], 194 | "resultAlias": "PartitionCount" 195 | }, 196 | { 197 | "outputWriters": [ 198 | { 199 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 200 | "url": "${influxUrl}", 201 | "database": "${influxDb}", 202 | "username": "${influxUser}", 203 | "password": "${influxPwd}" 204 | } 205 | ], 206 | "obj": "kafka.server:type=ReplicaManager,name=LeaderCount", 207 | "attr": [ 208 | "Value" 209 | ], 210 | "resultAlias": "LeaderCount" 211 | }, 212 | { 213 | "outputWriters": [ 214 | { 215 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 216 | "url": "${influxUrl}", 217 | "database": "${influxDb}", 218 | "username": "${influxUser}", 219 | "password": "${influxPwd}" 220 | } 221 | ], 222 | "obj": "kafka.server:type=KafkaRequestHandlerPool,name=RequestHandlerAvgIdlePercent", 223 | "attr": [ 224 | "MeanRate" 225 | ], 226 | "resultAlias": "RequestHandlerAvgIdlePercent" 227 | }, 228 | { 229 | "outputWriters": [ 230 | { 231 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 232 | "url": "${influxUrl}", 233 | "database": "${influxDb}", 234 | "username": "${influxUser}", 235 | "password": "${influxPwd}" 236 | } 237 | ], 238 | "obj": "kafka.controller:type=KafkaController,name=OfflinePartitionsCount", 239 | "attr": [ 240 | "Value" 241 | ], 242 | "resultAlias": "OfflinePartitionsCount" 243 | }, 244 | { 245 | "outputWriters": [ 246 | { 247 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 248 | "url": "${influxUrl}", 249 | "database": "${influxDb}", 250 | "username": "${influxUser}", 251 | "password": "${influxPwd}" 252 | } 253 | ], 254 | "obj": "kafka.controller:type=KafkaController,name=ActiveControllerCount", 255 | "attr": [ 256 | "Value" 257 | ], 258 | "resultAlias": "ActiveControllerCount" 259 | }, 260 | { 261 | "outputWriters": [ 262 | { 263 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 264 | "url": "${influxUrl}", 265 | "database": "${influxDb}", 266 | "username": "${influxUser}", 267 | "password": "${influxPwd}" 268 | } 269 | ], 270 | "obj": "kafka.controller:type=ControllerStats,name=LeaderElectionRateAndTimeMs", 271 | "attr": [ 272 | "Count" 273 | ], 274 | "resultAlias": "LeaderElectionRateAndTimeMs" 275 | }, 276 | { 277 | "outputWriters": [ 278 | { 279 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 280 | "url": "${influxUrl}", 281 | "database": "${influxDb}", 282 | "username": "${influxUser}", 283 | "password": "${influxPwd}" 284 | } 285 | ], 286 | "obj": "kafka.controller:type=ControllerStats,name=UncleanLeaderElectionsPerSec", 287 | "attr": [ 288 | "Count" 289 | ], 290 | "resultAlias": "UncleanLeaderElectionsPerSec" 291 | }, 292 | { 293 | "outputWriters": [ 294 | { 295 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 296 | "url": "${influxUrl}", 297 | "database": "${influxDb}", 298 | "username": "${influxUser}", 299 | "password": "${influxPwd}" 300 | } 301 | ], 302 | "obj": "kafka.network:type=RequestMetrics,name=RequestsPerSec,request=Produce", 303 | "attr": [ 304 | "Count", 305 | "MeanRate", 306 | "OneMinuteRate", 307 | "FiveMinuteRate", 308 | "FifteenMinuteRate" 309 | ], 310 | "resultAlias": "ProduceRequestsPerSec" 311 | }, 312 | { 313 | "outputWriters": [ 314 | { 315 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 316 | "url": "${influxUrl}", 317 | "database": "${influxDb}", 318 | "username": "${influxUser}", 319 | "password": "${influxPwd}" 320 | } 321 | ], 322 | "obj": "kafka.network:type=RequestMetrics,name=RequestsPerSec,request=FetchConsumer", 323 | "attr": [ 324 | "Count", 325 | "MeanRate", 326 | "OneMinuteRate", 327 | "FiveMinuteRate", 328 | "FifteenMinuteRate" 329 | ], 330 | "resultAlias": "FetchConsumerRequestsPerSec" 331 | }, 332 | { 333 | "outputWriters": [ 334 | { 335 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 336 | "url": "${influxUrl}", 337 | "database": "${influxDb}", 338 | "username": "${influxUser}", 339 | "password": "${influxPwd}" 340 | } 341 | ], 342 | "obj": "kafka.network:type=RequestMetrics,name=RequestsPerSec,request=FetchFollower", 343 | "attr": [ 344 | "Count", 345 | "MeanRate", 346 | "OneMinuteRate", 347 | "FiveMinuteRate", 348 | "FifteenMinuteRate" 349 | ], 350 | "resultAlias": "FetchFollowerRequestsPerSec" 351 | }, 352 | { 353 | "outputWriters": [ 354 | { 355 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 356 | "url": "${influxUrl}", 357 | "database": "${influxDb}", 358 | "username": "${influxUser}", 359 | "password": "${influxPwd}" 360 | } 361 | ], 362 | "obj": "kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Produce", 363 | "attr": [ 364 | "Count", 365 | "Min", 366 | "Max", 367 | "Mean", 368 | "StdDev", 369 | "50thPercentile", 370 | "75thPercentile", 371 | "95thPercentile", 372 | "98thPercentile", 373 | "99thPercentile", 374 | "999thPercentile" 375 | ], 376 | "resultAlias": "ProduceTotalTimeMs" 377 | }, 378 | { 379 | "outputWriters": [ 380 | { 381 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 382 | "url": "${influxUrl}", 383 | "database": "${influxDb}", 384 | "username": "${influxUser}", 385 | "password": "${influxPwd}" 386 | } 387 | ], 388 | "obj": "kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchConsumer", 389 | "attr": [ 390 | "Count", 391 | "Min", 392 | "Max", 393 | "Mean", 394 | "StdDev", 395 | "50thPercentile", 396 | "75thPercentile", 397 | "95thPercentile", 398 | "98thPercentile", 399 | "99thPercentile", 400 | "999thPercentile" 401 | ], 402 | "resultAlias": "FetchConsumerTotalTimeMs" 403 | }, 404 | { 405 | "outputWriters": [ 406 | { 407 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 408 | "url": "${influxUrl}", 409 | "database": "${influxDb}", 410 | "username": "${influxUser}", 411 | "password": "${influxPwd}" 412 | } 413 | ], 414 | "obj": "kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchFollower", 415 | "attr": [ 416 | "Count", 417 | "Min", 418 | "Max", 419 | "Mean", 420 | "StdDev", 421 | "50thPercentile", 422 | "75thPercentile", 423 | "95thPercentile", 424 | "98thPercentile", 425 | "99thPercentile", 426 | "999thPercentile" 427 | ], 428 | "resultAlias": "FetchFollowerTotalTimeMs" 429 | }, 430 | { 431 | "outputWriters": [ 432 | { 433 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 434 | "url": "${influxUrl}", 435 | "database": "${influxDb}", 436 | "username": "${influxUser}", 437 | "password": "${influxPwd}" 438 | } 439 | ], 440 | "obj": "kafka.network:type=RequestMetrics,name=RequestQueueTimeMs,request=FetchFollower", 441 | "attr": [ 442 | "Count", 443 | "Min", 444 | "Max", 445 | "Mean", 446 | "StdDev", 447 | "50thPercentile", 448 | "75thPercentile", 449 | "95thPercentile", 450 | "98thPercentile", 451 | "99thPercentile", 452 | "999thPercentile" 453 | ], 454 | "resultAlias": "FetchFollowerRequestQueueTimeMs" 455 | }, 456 | { 457 | "outputWriters": [ 458 | { 459 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 460 | "url": "${influxUrl}", 461 | "database": "${influxDb}", 462 | "username": "${influxUser}", 463 | "password": "${influxPwd}" 464 | } 465 | ], 466 | "obj": "kafka.network:type=RequestMetrics,name=RequestQueueTimeMs,request=FetchConsumer", 467 | "attr": [ 468 | "Count", 469 | "Min", 470 | "Max", 471 | "Mean", 472 | "StdDev", 473 | "50thPercentile", 474 | "75thPercentile", 475 | "95thPercentile", 476 | "98thPercentile", 477 | "99thPercentile", 478 | "999thPercentile" 479 | ], 480 | "resultAlias": "FetchConsumerRequestQueueTimeMs" 481 | }, 482 | { 483 | "outputWriters": [ 484 | { 485 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 486 | "url": "${influxUrl}", 487 | "database": "${influxDb}", 488 | "username": "${influxUser}", 489 | "password": "${influxPwd}" 490 | } 491 | ], 492 | "obj": "kafka.network:type=RequestMetrics,name=RequestQueueTimeMs,request=Produce", 493 | "attr": [ 494 | "Count", 495 | "Min", 496 | "Max", 497 | "Mean", 498 | "StdDev", 499 | "50thPercentile", 500 | "75thPercentile", 501 | "95thPercentile", 502 | "98thPercentile", 503 | "99thPercentile", 504 | "999thPercentile" 505 | ], 506 | "resultAlias": "ProduceRequestQueueTimeMs" 507 | }, 508 | { 509 | "outputWriters": [ 510 | { 511 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 512 | "url": "${influxUrl}", 513 | "database": "${influxDb}", 514 | "username": "${influxUser}", 515 | "password": "${influxPwd}" 516 | } 517 | ], 518 | "obj": "kafka.network:type=SocketServer,name=NetworkProcessorAvgIdlePercent", 519 | "attr": [ 520 | "Value" 521 | ], 522 | "resultAlias": "NetworkProcessorAvgIdlePercent " 523 | }, 524 | { 525 | "outputWriters": [ 526 | { 527 | "@class": "com.googlecode.jmxtrans.model.output.InfluxDbWriterFactory", 528 | "url": "${influxUrl}", 529 | "database": "${influxDb}", 530 | "username": "${influxUser}", 531 | "password": "${influxPwd}" 532 | } 533 | ], 534 | "obj": "kafka.log:type=LogFlushStats,name=LogFlushRateAndTimeMs", 535 | "attr": [ 536 | "MeanRate", 537 | "OneMinuteRate", 538 | "FiveMinuteRate", 539 | "FifteenMinuteRate", 540 | "50thPercentile", 541 | "Min", 542 | "Mean", 543 | "StdDev", 544 | "75thPercentile", 545 | "95thPercentile", 546 | "98thPercentile", 547 | "99thPercentile", 548 | "999thPercentile", 549 | "Count", 550 | "Max" 551 | ], 552 | "resultAlias": "LogFlush" 553 | } 554 | ], 555 | "numQueryThreads": 2 556 | } 557 | ] 558 | } -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | keyar 8 | confluentio 9 | pom 10 | 1.0-SNAPSHOT 11 | 12 | 13 | UTF-8 14 | UTF-8 15 | 16 | 0.10.0.0-cp1 17 | 3.0.0 18 | 1.7.7 19 | 2.11.0 20 | 3.4.6 21 | 22 | 23 | 24 | 25 | 26 | log4j 27 | log4j 28 | 1.2.17 29 | 30 | 31 | 32 | 33 | 34 | avro-serialization 35 | 36 | 37 | 38 | 39 | confluent 40 | http://packages.confluent.io/maven/ 41 | 42 | 43 | 44 | 45 | --------------------------------------------------------------------------------