├── 01.Monitoring_with_ELK ├── docker-elk │ ├── extensions │ │ ├── README.md │ │ └── logspout │ │ │ ├── Dockerfile │ │ │ ├── modules.go │ │ │ ├── logspout-compose.yml │ │ │ ├── build.sh │ │ │ └── README.md │ ├── .gitattributes │ ├── kibana │ │ ├── Dockerfile │ │ └── config │ │ │ └── kibana.yml │ ├── logstash │ │ ├── Dockerfile │ │ ├── pipeline │ │ │ └── logstash.conf │ │ └── config │ │ │ └── logstash.yml │ ├── elasticsearch │ │ ├── Dockerfile │ │ └── config │ │ │ └── elasticsearch.yml │ ├── docker-compose.yml │ ├── LICENSE │ └── README.md ├── img_monitoing_stack.png ├── config │ ├── jmx_conf │ │ ├── jmx_kafka_consumer_lag.conf │ │ ├── jmx_zookeeper.conf │ │ ├── jmx_kafka_broker_resource_ds07.conf │ │ ├── jmx_kafka_broker_resource_ds08.conf │ │ ├── jmx_kafka_broker_resource_ds06.conf │ │ ├── jmx_kafka_consumer.conf │ │ ├── jmx_kafka_producer.conf │ │ ├── jmx_kafka_broker_ds06.conf │ │ ├── jmx_kafka_broker_ds07.conf │ │ └── jmx_kafka_broker_ds08.conf │ ├── logstash_producer.yml │ ├── logstash_consumer.yml │ ├── logstash_producer_multi_topic.yml │ ├── filebeat_producer.yml │ ├── logstash_producer_stream.yml │ └── logstash_jmx_kafka.yml ├── kibana │ ├── kibana_dashboard.json │ ├── kibana_visualizations.json │ └── kibana_export_everything_new.json ├── Kafka_Monitoring_using_docker.md └── README.md ├── 20.Monitoring_OS_with_metricbeats ├── metricbeat_conf │ └── metricbeat.yml └── README.md ├── 10.Monitoring_with_Datadog └── README.md ├── .gitignore ├── kafka_conf_default.md ├── Run_KafkaCluster_with_jmx.md └── README.md /01.Monitoring_with_ELK/docker-elk/extensions/README.md: -------------------------------------------------------------------------------- 1 | Third-party extensions that enable extra integrations with the ELK stack. 2 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/docker-elk/.gitattributes: -------------------------------------------------------------------------------- 1 | # Declare files that will always have LF line endings on checkout. 2 | *.sh text eol=lf -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/img_monitoing_stack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/freepsw/kafka-monitoring/HEAD/01.Monitoring_with_ELK/img_monitoing_stack.png -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/docker-elk/kibana/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/elastic/kibana-docker 2 | FROM docker.elastic.co/kibana/kibana:5.5.1 3 | 4 | # Add your kibana plugins setup here 5 | # Example: RUN kibana-plugin install 6 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/docker-elk/extensions/logspout/Dockerfile: -------------------------------------------------------------------------------- 1 | # uses ONBUILD instructions described here: 2 | # https://github.com/gliderlabs/logspout/tree/master/custom 3 | 4 | FROM gliderlabs/logspout:master 5 | ENV SYSLOG_FORMAT rfc3164 6 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/docker-elk/logstash/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/elastic/logstash-docker 2 | FROM docker.elastic.co/logstash/logstash:5.5.1 3 | 4 | # Add your logstash plugins setup here 5 | # Example: RUN logstash-plugin install logstash-filter-json 6 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/docker-elk/elasticsearch/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/elastic/elasticsearch-docker 2 | FROM docker.elastic.co/elasticsearch/elasticsearch:5.5.1 3 | 4 | # Add your elasticsearch plugins setup here 5 | # Example: RUN elasticsearch-plugin install analysis-icu 6 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/docker-elk/logstash/pipeline/logstash.conf: -------------------------------------------------------------------------------- 1 | input { 2 | tcp { 3 | port => 5000 4 | } 5 | } 6 | 7 | ## Add your filters / logstash plugins configuration here 8 | 9 | output { 10 | elasticsearch { 11 | hosts => "elasticsearch:9200" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/config/jmx_conf/jmx_kafka_consumer_lag.conf: -------------------------------------------------------------------------------- 1 | { 2 | "host" : "localhost", 3 | "port" : 9882, 4 | "alias" : "kafka-consumer", 5 | "queries" : [ 6 | { 7 | "object_name" : "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*", 8 | "attributes" : ["records-lag-max"], 9 | "object_alias" : "Consumer.RecordsLagMax" 10 | } 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/docker-elk/extensions/logspout/modules.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // installs the Logstash adapter for Logspout, and required dependencies 4 | // https://github.com/looplab/logspout-logstash 5 | import ( 6 | _ "github.com/looplab/logspout-logstash" 7 | _ "github.com/gliderlabs/logspout/transports/udp" 8 | _ "github.com/gliderlabs/logspout/transports/tcp" 9 | ) 10 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/config/jmx_conf/jmx_zookeeper.conf: -------------------------------------------------------------------------------- 1 | { 2 | "host" : "zookeper01", 3 | "port" : 9998, 4 | "alias" : "zookeeper3", 5 | "queries" : [ 6 | { 7 | "object_name": "org.apache.ZooKeeperService:name0=ReplicatedServer_id*,name1=replica.*,name2=Follower,*" 8 | }, 9 | { 10 | "object_name": "org.apache.ZooKeeperService:name0=ReplicatedServer_id*,name1=replica.*,name2=Leader,*" 11 | } 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/config/logstash_producer.yml: -------------------------------------------------------------------------------- 1 | input { 2 | file { 3 | path => "/home/poc/F1.dat" 4 | start_position => "beginning" 5 | } 6 | } 7 | 8 | output { 9 | kafka { 10 | codec => plain { format => "%{message}" } 11 | bootstrap_servers => "broker06:9092" 12 | topic_id => "F1" 13 | acks => "all" 14 | retries => 20 15 | batch_size => 49152 16 | linger_ms => 2000 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/config/logstash_consumer.yml: -------------------------------------------------------------------------------- 1 | input { 2 | kafka { 3 | type => "F1" 4 | bootstrap_servers => "broker06:9092,broker07:9092,broker08:9092" 5 | topics => "F1" 6 | group_id => "logstash" 7 | max_poll_records => "5" 8 | receive_buffer_bytes => "10240" 9 | } 10 | } 11 | 12 | output { 13 | file { 14 | path => "F1.dat" 15 | codec => line { format => "%{message}" } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/docker-elk/extensions/logspout/logspout-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | logspout: 5 | build: 6 | context: extensions/logspout 7 | volumes: 8 | - /var/run/docker.sock:/var/run/docker.sock:ro 9 | environment: 10 | ROUTE_URIS: logstash://logstash:5000 11 | LOGSTASH_TAGS: docker-elk 12 | networks: 13 | - elk 14 | depends_on: 15 | - logstash 16 | restart: on-failure 17 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/config/logstash_producer_multi_topic.yml: -------------------------------------------------------------------------------- 1 | input { 2 | file { 3 | type => "topic_001" 4 | path => "/home/rts/kafka-poc/data/input/prod_02.dat" 5 | } 6 | } 7 | 8 | output { 9 | if "002prod" in [message] { 10 | stdout { 11 | codec => rubydebug{ } 12 | } 13 | } 14 | 15 | 16 | kafka { 17 | codec => plain { 18 | format => "%{message}" 19 | } 20 | bootstrap_servers => "localhost:9092" 21 | topic_id => "test" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/config/filebeat_producer.yml: -------------------------------------------------------------------------------- 1 | filebeat.prospectors: 2 | - input_type: log 3 | document_type: F1 4 | paths: 5 | - /home/poc/data/F1.dat 6 | 7 | - input_type: log 8 | document_type: test2 9 | paths: 10 | - /home/poc/data/F2.dat 11 | 12 | output.kafka: 13 | # The Logstash hosts 14 | hosts: ["broker06:9092","broker07:9092","broker08:9092"] 15 | topic: '%{[type]}' 16 | codec.format: 17 | string: '%{[message]}' 18 | required_acks: -1 19 | max_retries: 3 20 | bulk_max_size: 2048 21 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/docker-elk/logstash/config/logstash.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Logstash configuration from logstash-docker. 3 | ## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash.yml 4 | # 5 | http.host: "0.0.0.0" 6 | path.config: /usr/share/logstash/pipeline 7 | 8 | ## Disable X-Pack 9 | ## see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html 10 | ## https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling 11 | # 12 | xpack.monitoring.enabled: false -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/config/jmx_conf/jmx_kafka_broker_resource_ds07.conf: -------------------------------------------------------------------------------- 1 | { 2 | "host" : "broker07", 3 | "port" : 9999, 4 | "alias" : "kafkabroker2", 5 | "queries" : [ 6 | { 7 | "object_name" : "java.lang:type=OperatingSystem", 8 | "attributes" : ["SystemCpuLoad", "ProcessCpuLoad", "OpenFileDescriptorCount","FreePhysicalMemorySize", "TotalPhysicalMemorySize" ], 9 | "object_alias" : "${type}" 10 | }, 11 | { 12 | "object_name" : "java.lang:type=Memory", 13 | "attributes" : ["HeapMemoryUsage", "NonHeapMemoryUsage"], 14 | "object_alias" : "${type}" 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/config/jmx_conf/jmx_kafka_broker_resource_ds08.conf: -------------------------------------------------------------------------------- 1 | { 2 | "host" : "broker08", 3 | "port" : 9999, 4 | "alias" : "kafkabroker3", 5 | "queries" : [ 6 | { 7 | "object_name" : "java.lang:type=OperatingSystem", 8 | "attributes" : ["SystemCpuLoad", "ProcessCpuLoad", "OpenFileDescriptorCount","FreePhysicalMemorySize", "TotalPhysicalMemorySize" ], 9 | "object_alias" : "${type}" 10 | }, 11 | { 12 | "object_name" : "java.lang:type=Memory", 13 | "attributes" : ["HeapMemoryUsage", "NonHeapMemoryUsage"], 14 | "object_alias" : "${type}" 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/config/jmx_conf/jmx_kafka_broker_resource_ds06.conf: -------------------------------------------------------------------------------- 1 | { 2 | "host" : "broker06", 3 | "port" : 9999, 4 | "alias" : "kafkabroker1", 5 | "queries" : [ 6 | { 7 | "object_name" : "java.lang:type=OperatingSystem", 8 | "attributes" : ["SystemCpuLoad", "ProcessCpuLoad", "OpenFileDescriptorCount","FreePhysicalMemorySize", "TotalPhysicalMemorySize" ], 9 | "object_alias" : "${type}" 10 | }, 11 | { 12 | "object_name" : "java.lang:type=Memory", 13 | "attributes" : ["HeapMemoryUsage", "NonHeapMemoryUsage"], 14 | "object_alias" : "${type}" 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/config/logstash_producer_stream.yml: -------------------------------------------------------------------------------- 1 | input { 2 | generator { 3 | lines => [ 4 | "line 1", 5 | "line 2", 6 | "line 3" 7 | ] 8 | # Emit all lines 3 times. 9 | count => 10000 10 | } 11 | } 12 | 13 | filter { 14 | sleep { 15 | time => "1" # Sleep 1 second 16 | every => 2 # on every 10th event 17 | } 18 | } 19 | 20 | output { 21 | stdout { 22 | codec => rubydebug{ } 23 | } 24 | 25 | kafka { 26 | codec => plain { 27 | format => "%{message}" 28 | } 29 | bootstrap_servers => "localhost:9092" 30 | topic_id => "test" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/docker-elk/extensions/logspout/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # unmodified from: 4 | # https://github.com/gliderlabs/logspout/blob/67ee3831cbd0594361bb3381380c65bdbeb3c20f/custom/build.sh 5 | 6 | set -e 7 | apk add --update go git mercurial build-base 8 | mkdir -p /go/src/github.com/gliderlabs 9 | cp -r /src /go/src/github.com/gliderlabs/logspout 10 | cd /go/src/github.com/gliderlabs/logspout 11 | export GOPATH=/go 12 | go get 13 | go build -ldflags "-X main.Version=$1" -o /bin/logspout 14 | apk del go git mercurial build-base 15 | rm -rf /go /var/cache/apk/* /root/.glide 16 | 17 | # backwards compatibility 18 | ln -fs /tmp/docker.sock /var/run/docker.sock 19 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/docker-elk/kibana/config/kibana.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Kibana configuration from kibana-docker. 3 | ## from https://github.com/elastic/kibana-docker/blob/master/build/kibana/config/kibana.yml 4 | # 5 | server.name: kibana 6 | server.host: "0" 7 | elasticsearch.url: http://elasticsearch:9200 8 | 9 | ## Disable X-Pack 10 | ## see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html 11 | ## https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling 12 | # 13 | xpack.security.enabled: false 14 | xpack.monitoring.enabled: false 15 | xpack.ml.enabled: false 16 | xpack.graph.enabled: false 17 | xpack.reporting.enabled: false 18 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/docker-elk/extensions/logspout/README.md: -------------------------------------------------------------------------------- 1 | # Logspout extension 2 | 3 | Logspout collects all Docker logs using the Docker logs API, and forwards them to Logstash without any additional 4 | configuration. 5 | 6 | ## Usage 7 | 8 | If you want to include the Logspout extension, run Docker Compose from the root of the repository with an additional 9 | command line argument referencing the `logspout-compose.yml` file: 10 | 11 | ```bash 12 | $ docker-compose -f docker-compose.yml -f extensions/logspout/logspout-compose.yml up 13 | ``` 14 | 15 | In your Logstash pipeline configuration, enable the `udp` input and set the input codec to `json`: 16 | 17 | ``` 18 | input { 19 | udp { 20 | port => 5000 21 | codec => json 22 | } 23 | } 24 | ``` 25 | 26 | ## Documentation 27 | 28 | https://github.com/looplab/logspout-logstash 29 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/config/jmx_conf/jmx_kafka_consumer.conf: -------------------------------------------------------------------------------- 1 | { 2 | "host" : "localhost", 3 | "port" : 9882, 4 | "alias" : "kafka-consumer", 5 | "queries" : [ 6 | { 7 | "object_name" : "kafka.consumer:type=consumer-metrics,client-id=*", 8 | "attributes" : ["incoming-byte-rate"], 9 | "object_alias" : "Consumer.InBytesRate" 10 | }, 11 | { 12 | "object_name" : "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=logstash-0", 13 | "attributes" : ["bytes-consumed-rate"], 14 | "object_alias" : "Consumer.ByteConsumed" 15 | }, 16 | { 17 | "object_name" : "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=logstash-0", 18 | "attributes" : ["records-consumed-rate"], 19 | "object_alias" : "Consumer.RecordsConsumed" 20 | }, 21 | { 22 | "object_name" : "java.lang:type=OperatingSystem", 23 | "object_alias" : "OperatingSystem", 24 | "attributes" : ["SystemCpuLoad"] 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/config/jmx_conf/jmx_kafka_producer.conf: -------------------------------------------------------------------------------- 1 | { 2 | "host" : "localhost", 3 | "port" : 9881, 4 | "alias" : "kafka-producer", 5 | "queries" : [ 6 | { 7 | "object_name" : "kafka.producer:type=producer-metrics,client-id=*", 8 | "attributes" : ["outgoing-byte-rate"], 9 | "object_alias" : "Producer.BytesRate" 10 | }, 11 | { 12 | "object_name" : "kafka.producer:type=producer-metrics,client-id=*", 13 | "attributes" : ["request-rate"], 14 | "object_alias" : "RequestRate" 15 | }, 16 | { 17 | "object_name" : "kafka.producer:type=producer-metrics,client-id=*", 18 | "attributes" : ["response-rate"] 19 | }, 20 | { 21 | "object_name" : "kafka.producer:type=producer-metrics,client-id=*", 22 | "attributes" : ["request-latency-avg"] 23 | }, 24 | { 25 | "object_name" : "java.lang:type=OperatingSystem", 26 | "object_alias" : "OperatingSystem", 27 | "attributes" : ["SystemCpuLoad"] 28 | } 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/docker-elk/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | 3 | services: 4 | 5 | elasticsearch: 6 | build: elasticsearch/ 7 | volumes: 8 | - ./elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml 9 | ports: 10 | - "9200:9200" 11 | - "9300:9300" 12 | environment: 13 | ES_JAVA_OPTS: "-Xmx256m -Xms256m" 14 | networks: 15 | - elk 16 | 17 | logstash: 18 | build: logstash/ 19 | volumes: 20 | - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml 21 | - ./logstash/pipeline:/usr/share/logstash/pipeline 22 | ports: 23 | - "5000:5000" 24 | environment: 25 | LS_JAVA_OPTS: "-Xmx256m -Xms256m" 26 | networks: 27 | - elk 28 | depends_on: 29 | - elasticsearch 30 | 31 | kibana: 32 | build: kibana/ 33 | volumes: 34 | - ./kibana/config/:/usr/share/kibana/config 35 | ports: 36 | - "5601:5601" 37 | networks: 38 | - elk 39 | depends_on: 40 | - elasticsearch 41 | 42 | networks: 43 | 44 | elk: 45 | driver: bridge 46 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/docker-elk/elasticsearch/config/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Elasticsearch configuration from elasticsearch-docker. 3 | ## from https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/elasticsearch.yml 4 | # 5 | cluster.name: "docker-cluster" 6 | network.host: 0.0.0.0 7 | 8 | # minimum_master_nodes need to be explicitly set when bound on a public IP 9 | # set to 1 to allow single node clusters 10 | # Details: https://github.com/elastic/elasticsearch/pull/17288 11 | discovery.zen.minimum_master_nodes: 1 12 | 13 | ## Use single node discovery in order to disable production mode and avoid bootstrap checks 14 | ## see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html 15 | # 16 | discovery.type: single-node 17 | 18 | ## Disable X-Pack 19 | ## see https://www.elastic.co/guide/en/x-pack/current/xpack-settings.html 20 | ## https://www.elastic.co/guide/en/x-pack/current/installing-xpack.html#xpack-enabling 21 | # 22 | xpack.security.enabled: false 23 | xpack.monitoring.enabled: false 24 | xpack.ml.enabled: false 25 | xpack.graph.enabled: false 26 | xpack.watcher.enabled: false 27 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/docker-elk/LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Anthony Lapenna 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /20.Monitoring_OS_with_metricbeats/metricbeat_conf/metricbeat.yml: -------------------------------------------------------------------------------- 1 | #========================== Modules configuration ============================ 2 | metricbeat.modules: 3 | 4 | #------------------------------- System Module ------------------------------- 5 | - module: system 6 | metricsets: 7 | # CPU stats 8 | - cpu 9 | 10 | # System Load stats 11 | - load 12 | 13 | # Per CPU core stats 14 | - core 15 | 16 | # IO stats 17 | - diskio 18 | 19 | # Per filesystem stats 20 | - filesystem 21 | 22 | # File system summary stats 23 | - fsstat 24 | 25 | # Memory stats 26 | - memory 27 | 28 | # Network stats 29 | - network 30 | 31 | # Per process stats 32 | - process 33 | 34 | # Sockets (linux only) 35 | #- socket 36 | enabled: true 37 | period: 10s 38 | processes: ['.*'] 39 | cpu_ticks: false 40 | 41 | #-------------------------- Elasticsearch output ------------------------------ 42 | output.elasticsearch: 43 | # Array of hosts to connect to. 44 | hosts: ["localhost:9200"] 45 | # index를 지정할 수도 있으나, Default값인 "metricbeat-*"으로 저장되어야 46 | # metricbeat에서 제공하는 kiban dashboard를 사용할 수 있다. 47 | # index: "broker1-metric" 48 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/config/logstash_jmx_kafka.yml: -------------------------------------------------------------------------------- 1 | input { 2 | jmx { 3 | path => "/home/poc/jmx_conf" 4 | polling_frequency => 2 5 | type => "jmx" 6 | } 7 | } 8 | 9 | filter { 10 | if [type] == "jmx" { 11 | if ("ProcessCpuLoad" in [metric_path] or "SystemCpuLoad" in [metric_path]) { 12 | ruby { 13 | code => "n = event.get('[metric_value_number]') 14 | event.set('[metric_value_number]', n*100) 15 | event.set('cpu_load',n) 16 | " 17 | } 18 | } 19 | if ("TotalPhysicalMemorySize" in [metric_path] or "FreePhysicalMemorySize" in [metric_path]) { 20 | ruby { 21 | code => "n = event.get('[metric_value_number]') 22 | event.set('[metric_value_number]', n/1024/1024) 23 | event.set('physical_memory',n) 24 | " 25 | } 26 | } 27 | if ("HeapMemoryUsage" in [metric_path] or "HeapMemoryUsage" in [metric_path]) { 28 | ruby { 29 | code => "n = event.get('[metric_value_number]') 30 | event.set('[metric_value_number]', n/1024/1024) 31 | " 32 | } 33 | } 34 | } 35 | } 36 | 37 | output{ 38 | stdout { 39 | codec => rubydebug 40 | } 41 | elasticsearch { 42 | hosts => "localhost:9200" 43 | index => "kafka_mon" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/config/jmx_conf/jmx_kafka_broker_ds06.conf: -------------------------------------------------------------------------------- 1 | { 2 | "host" : "broker06", 3 | "port" : 9999, 4 | "alias" : "kafkabroker1", 5 | "queries" : [ 6 | { 7 | "object_name" : "kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec", 8 | "attributes" : [ "OneMinuteRate" ], 9 | "object_alias" : "${type}.${name}" 10 | }, 11 | { 12 | "object_name" : "kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec", 13 | "attributes" : [ "OneMinuteRate" ], 14 | "object_alias" : "${type}.${name}" 15 | }, 16 | { 17 | "object_name" : "kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec", 18 | "attributes" : [ "OneMinuteRate" ], 19 | "object_alias" : "${type}.${name}" 20 | }, 21 | { 22 | "object_name" : "kafka.server:type=ReplicaManager,name=UnderReplicatedPartitions", 23 | "attributes" : [ "Value" ], 24 | "object_alias" : "${type}.${name}" 25 | }, 26 | { 27 | "object_name" : "kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Produce", 28 | "attributes" : [ "Mean" ], 29 | "object_alias" : "${type}.${name}" 30 | }, 31 | { 32 | "object_name" : "kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchConsumer", 33 | "attributes" : [ "Mean" ], 34 | "object_alias" : "${type}.${name}" 35 | } 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/config/jmx_conf/jmx_kafka_broker_ds07.conf: -------------------------------------------------------------------------------- 1 | { 2 | "host" : "broker07", 3 | "port" : 9999, 4 | "alias" : "kafkabroker2", 5 | "queries" : [ 6 | { 7 | "object_name" : "kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec", 8 | "attributes" : [ "OneMinuteRate" ], 9 | "object_alias" : "${type}.${name}" 10 | }, 11 | { 12 | "object_name" : "kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec", 13 | "attributes" : [ "OneMinuteRate" ], 14 | "object_alias" : "${type}.${name}" 15 | }, 16 | { 17 | "object_name" : "kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec", 18 | "attributes" : [ "OneMinuteRate" ], 19 | "object_alias" : "${type}.${name}" 20 | }, 21 | { 22 | "object_name" : "kafka.server:type=ReplicaManager,name=UnderReplicatedPartitions", 23 | "attributes" : [ "Value" ], 24 | "object_alias" : "${type}.${name}" 25 | }, 26 | { 27 | "object_name" : "kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Produce", 28 | "attributes" : [ "Mean" ], 29 | "object_alias" : "${type}.${name}" 30 | }, 31 | { 32 | "object_name" : "kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchConsumer", 33 | "attributes" : [ "Mean" ], 34 | "object_alias" : "${type}.${name}" 35 | } 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/config/jmx_conf/jmx_kafka_broker_ds08.conf: -------------------------------------------------------------------------------- 1 | { 2 | "host" : "broker08", 3 | "port" : 9999, 4 | "alias" : "kafkabroker3", 5 | "queries" : [ 6 | { 7 | "object_name" : "kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec", 8 | "attributes" : [ "OneMinuteRate" ], 9 | "object_alias" : "${type}.${name}" 10 | }, 11 | { 12 | "object_name" : "kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec", 13 | "attributes" : [ "OneMinuteRate" ], 14 | "object_alias" : "${type}.${name}" 15 | }, 16 | { 17 | "object_name" : "kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec", 18 | "attributes" : [ "OneMinuteRate" ], 19 | "object_alias" : "${type}.${name}" 20 | }, 21 | { 22 | "object_name" : "kafka.server:type=ReplicaManager,name=UnderReplicatedPartitions", 23 | "attributes" : [ "Value" ], 24 | "object_alias" : "${type}.${name}" 25 | }, 26 | { 27 | "object_name" : "kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Produce", 28 | "attributes" : [ "Mean" ], 29 | "object_alias" : "${type}.${name}" 30 | }, 31 | { 32 | "object_name" : "kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchConsumer", 33 | "attributes" : [ "Mean" ], 34 | "object_alias" : "${type}.${name}" 35 | } 36 | ] 37 | } 38 | -------------------------------------------------------------------------------- /10.Monitoring_with_Datadog/README.md: -------------------------------------------------------------------------------- 1 | ## PART 1. Monitoring using datadog 2 | - datadog는 cloud기반의 상용 모니터링 서비스 3 | - 즉, kafka broker가 구동하고 있는 서버에서, datadog 서버로 접속이 가능해야 한다. 4 | - https://www.datadoghq.com/blog/monitor-kafka-with-datadog/ 참고 5 | 6 | ### STEP 1. Start with free-trial (30 days) 7 | - connect to https://www.datadoghq.com/ 8 | - get API Key 9 | - copy printed command line on the web site 10 | - install data-dog agent 11 | 12 | ``` 13 | # centos 14 | > DD_API_KEY=xxxxxxxxx bash -c "$(curl -L https://raw.githubusercontent.com/DataDog/dd-agent/master/packaging/datadog-agent/source/install_agent.sh)" 15 | ``` 16 | 17 | ### STEP 2. Setting apache kafka integrations for Monitoring on Datadog web site 18 | - Menu > integrations 선택 19 | - Apache kafka click > install 20 | - Apache zookeeper click > install 21 | - monitoring을 위한 환경구성 까지 약 5분 정도 소요됨. 22 | 23 | 24 | ### STEP 3. Setting datadog configuration for apache kafka 25 | - 수집할 metrics 정보를 설정 (default 사용) 26 | ``` 27 | > cd /etc/dd-agent/conf.d 28 | > sudo cp kafka.yaml.example kafka.yaml 29 | 30 | instances: 31 | - host: localhost 32 | port: 9999 # This is the JMX port on which Kafka exposes its metrics (usually 9999) 33 | tags: 34 | kafka: broker 35 | 36 | > sudo cp kafka_consumer.yaml.example kafka_consumer.yaml 37 | 38 | instances: 39 | - kafka_connect_str: localhost:9092 40 | zk_connect_str: localhost:2181 41 | zk_prefix: /0.8 42 | consumer_groups: 43 | my_consumer: --> "consumer_group의 이름을 명시, console_consumer는 조회되지 않음" 44 | my_topic: [test] 45 | 46 | 47 | # datadog agent restart 48 | # http://docs.datadoghq.com/guides/basic_agent_usage/centos/ 참고 49 | > sudo /etc/init.d/datadog-agent restart 50 | 51 | # check datadog agent status 52 | > sudo /etc/init.d/datadog-agent info 53 | ``` 54 | 55 | 56 | ### STEP 4. Monitoring on datadog Dashboard 57 | - https://app.datadoghq.com/dash/list 접속 58 | - Custom Kafka 를 클릭히면, 상세 metrics 정보가 제공됨 59 | -------------------------------------------------------------------------------- /20.Monitoring_OS_with_metricbeats/README.md: -------------------------------------------------------------------------------- 1 | # Monitoring Server resource status using metricbeats 2 | - https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-getting-started.html 3 | 4 | ## STEP 1. Install on the server which you want to monitor 5 | ### Download metricbeats 6 | - 설치파일을 다운받고 압축을 해제한다. (설치완료!) 7 | ``` 8 | > cd ~ 9 | > wget https://artifacts.elastic.co/downloads/beats/metricbeat/metricbeat-5.4.3-linux-x86_64.tar.gz 10 | > tar xvf metricbeat-5.4.3-linux-x86_64.tar.gz 11 | ``` 12 | 13 | ## STEP 2. Configuring metricbeats 14 | ```yml 15 | metricbeat.modules: 16 | #------------------------------- System Module ------------------------------- 17 | - module: system 18 | metricsets: 19 | # CPU stats 20 | - cpu 21 | 22 | # System Load stats 23 | - load 24 | 25 | # Per CPU core stats 26 | - core 27 | 28 | # IO stats 29 | - diskio 30 | 31 | # Per filesystem stats 32 | - filesystem 33 | 34 | # File system summary stats 35 | - fsstat 36 | 37 | # Memory stats 38 | - memory 39 | 40 | # Network stats 41 | - network 42 | 43 | # Per process stats 44 | - process 45 | 46 | # Sockets (linux only) 47 | enabled: true 48 | period: 10s 49 | processes: ['.*'] 50 | cpu_ticks: false 51 | 52 | #-------------------------- Elasticsearch output ------------------------------ 53 | output.elasticsearch: 54 | # Array of hosts to connect to. 55 | hosts: ["localhost:9200"] 56 | # index를 지정할 수도 있으나, Default값인 "metricbeat-*"으로 저장되어야 57 | # metricbeat에서 제공하는 kiban dashboard를 사용할 수 있다. 58 | # index: "broker1-metric" 59 | ``` 60 | 61 | 62 | ## STEP 3. Run metricbeats 63 | ``` 64 | > ./metricbeat -e -c metricbeat.yml -d "publish" 65 | ``` 66 | 67 | ### STEP 4. Import Metricbeat dashboard to elasticsearch 68 | - metricbeat에서 기본으로 제공하는 dashboard를 이용하여 Kibana에서 시각화 가능 69 | ``` 70 | > ./scripts/import_dashboards -es http://169.56.124.19:9200 71 | ``` 72 | 73 | ### STEP 4. Visualiza metricbeat dashboard on the kibana dashboard ui 74 | - Dashboard를 클릭하면, import된 다양한 dashboard가 목록으로 표시된다. 75 | - 전체를 개략적으로 보려면 "Metricbeat system overview"를 선택 76 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/kibana/kibana_dashboard.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_id": "57d87d40-5a3d-11e7-a06b-bfa32df73ed4", 4 | "_type": "dashboard", 5 | "_source": { 6 | "title": "KafkaMon-Dash(throughput)", 7 | "hits": 0, 8 | "description": "", 9 | "panelsJSON": "[{\"col\":1,\"id\":\"09564bf0-5a30-11e7-a06b-bfa32df73ed4\",\"panelIndex\":2,\"row\":1,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"f1e4eb80-5a29-11e7-a406-63b0e3ccc267\",\"panelIndex\":3,\"row\":17,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"1beb2240-5a44-11e7-a06b-bfa32df73ed4\",\"panelIndex\":5,\"row\":7,\"size_x\":12,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"1c9d7880-5ad7-11e7-a406-63b0e3ccc267\",\"panelIndex\":6,\"row\":4,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"e4a325a0-5b09-11e7-afa4-79fb9d9351b8\",\"panelIndex\":7,\"row\":11,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"68c1fc30-5b0f-11e7-afa4-79fb9d9351b8\",\"panelIndex\":8,\"row\":14,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"}]", 10 | "optionsJSON": "{\"darkTheme\":false}", 11 | "uiStateJSON": "{\"P-2\":{\"spy\":{\"mode\":{\"fill\":false,\"name\":null}},\"vis\":{\"colors\":{\"kafka-consumer.Consumer.RecordsConsumed.records-consumed-rate\":\"#629E51\",\"kafkabroker1.FetchConsumer.Mean\":\"#3F6833\",\"kafkabroker1.Procude.Mean\":\"#E5AC0E\"},\"legendOpen\":true}},\"P-3\":{\"vis\":{\"legendOpen\":false}},\"P-5\":{\"vis\":{\"legendOpen\":true}},\"P-6\":{\"vis\":{\"colors\":{\"kafkabroker1.BrokerTopicMetrics.BytesInPerSec.OneMinuteRate\":\"#E24D42\"},\"legendOpen\":true}},\"P-7\":{\"spy\":{\"mode\":{\"fill\":false,\"name\":null}}}}", 12 | "version": 1, 13 | "timeRestore": false, 14 | "kibanaSavedObjectMeta": { 15 | "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" 16 | } 17 | } 18 | }, 19 | { 20 | "_id": "58f3bde0-5bd5-11e7-be67-49b67c124db6", 21 | "_type": "dashboard", 22 | "_source": { 23 | "title": "KafkaMon-Dash(Resource)", 24 | "hits": 0, 25 | "description": "", 26 | "panelsJSON": "[{\"col\":1,\"id\":\"41dc5590-5bd5-11e7-be67-49b67c124db6\",\"panelIndex\":1,\"row\":10,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"0c96b8b0-5bd7-11e7-be67-49b67c124db6\",\"panelIndex\":2,\"row\":7,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"f7849b30-5c63-11e7-844c-bbc872ee5c30\",\"panelIndex\":3,\"row\":13,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"065b3d90-5c63-11e7-8b85-6d355270351a\",\"panelIndex\":4,\"row\":4,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":3,\"panelIndex\":5,\"type\":\"visualization\",\"id\":\"7399a5c0-6057-11e7-87bb-1dcc594557a4\",\"col\":1,\"row\":1}]", 27 | "optionsJSON": "{\"darkTheme\":false}", 28 | "uiStateJSON": "{}", 29 | "version": 1, 30 | "timeRestore": false, 31 | "kibanaSavedObjectMeta": { 32 | "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" 33 | } 34 | } 35 | } 36 | ] -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Directories # 2 | /build/ 3 | /bin/ 4 | target/ 5 | 6 | # OS Files # 7 | .DS_Store 8 | 9 | *.class 10 | 11 | # Package Files # 12 | *.jar 13 | *.war 14 | *.ear 15 | *.db 16 | 17 | ###################### 18 | # Windows 19 | ###################### 20 | 21 | # Windows image file caches 22 | Thumbs.db 23 | 24 | # Folder config file 25 | Desktop.ini 26 | 27 | ###################### 28 | # OSX 29 | ###################### 30 | 31 | .DS_Store 32 | .svn 33 | 34 | # Thumbnails 35 | ._* 36 | 37 | # Files that might appear on external disk 38 | .Spotlight-V100 39 | .Trashes 40 | 41 | ###################### 42 | # Eclipse 43 | ###################### 44 | 45 | *.pydevproject 46 | .project 47 | .metadata 48 | bin/** 49 | tmp/** 50 | tmp/**/* 51 | *.tmp 52 | *.bak 53 | *.swp 54 | *~.nib 55 | local.properties 56 | .classpath 57 | .settings/ 58 | .loadpath 59 | /src/main/resources/rebel.xml 60 | # External tool builders 61 | .externalToolBuilders/ 62 | 63 | # Locally stored "Eclipse launch configurations" 64 | *.launch 65 | 66 | # CDT-specific 67 | .cproject 68 | 69 | # PDT-specific 70 | .buildpath 71 | 72 | 73 | ###################### 74 | # Spark 75 | ###################### 76 | 77 | *#*# 78 | *.#* 79 | *.iml 80 | *.ipr 81 | *.iws 82 | *.pyc 83 | *.pyo 84 | *.swp 85 | *~ 86 | .DS_Store 87 | .cache 88 | .cache-main 89 | .cache-tests 90 | .classpath 91 | .ensime 92 | .ensime_cache/ 93 | .ensime_lucene 94 | .generated-mima* 95 | .idea/ 96 | .idea_modules/ 97 | .project 98 | .pydevproject 99 | .scala_dependencies 100 | .settings 101 | /lib/ 102 | R-unit-tests.log 103 | R/unit-tests.out 104 | R/cran-check.out 105 | R/pkg/vignettes/sparkr-vignettes.html 106 | build/*.jar 107 | build/apache-maven* 108 | build/scala* 109 | build/zinc* 110 | cache 111 | checkpoint 112 | conf/*.cmd 113 | conf/*.conf 114 | conf/*.properties 115 | conf/*.sh 116 | conf/*.xml 117 | conf/java-opts 118 | conf/slaves 119 | dependency-reduced-pom.xml 120 | derby.log 121 | dev/create-release/*final 122 | dev/create-release/*txt 123 | dist/ 124 | docs/_site 125 | docs/api 126 | lib_managed/ 127 | lint-r-report.log 128 | log/ 129 | logs/ 130 | out/ 131 | project/boot/ 132 | project/build/target/ 133 | project/plugins/lib_managed/ 134 | project/plugins/project/build.properties 135 | project/plugins/src_managed/ 136 | project/plugins/target/ 137 | python/lib/pyspark.zip 138 | reports/ 139 | scalastyle-on-compile.generated.xml 140 | scalastyle-output.xml 141 | scalastyle.txt 142 | spark-*-bin-*.tgz 143 | spark-tests.log 144 | src_managed/ 145 | streaming-tests.log 146 | target/ 147 | unit-tests.log 148 | work/ 149 | 150 | # For Hive 151 | TempStatsStore/ 152 | metastore/ 153 | metastore_db/ 154 | sql/hive-thriftserver/test_warehouses 155 | warehouse/ 156 | spark-warehouse/ 157 | 158 | # For R session data 159 | .RData 160 | .RHistory 161 | .Rhistory 162 | *.Rproj 163 | *.Rproj.* 164 | 165 | .Rproj.user 166 | 167 | 168 | ###################### 169 | # Etc 170 | ###################### 171 | # Logs 172 | logs 173 | *.log 174 | *.log.* 175 | 176 | # Runtime data 177 | pids 178 | *.pid 179 | *.seed 180 | 181 | # Directory for instrumented libs generated by jscoverage/JSCover 182 | lib-cov 183 | 184 | # Coverage directory used by tools like istanbul 185 | coverage 186 | 187 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 188 | .grunt 189 | 190 | # nexe 191 | build 192 | nexe_node 193 | meta.json 194 | 195 | # Compiled binary addons (http://nodejs.org/api/addons.html) 196 | build/Release 197 | 198 | # Dependency directory 199 | # Commenting this out is preferred by some people, see 200 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- 201 | node_modules 202 | npm-shrinkwrap.json 203 | 204 | # Users Environment Variables 205 | .lock-wscript 206 | 207 | # IDE configuration 208 | .idea 209 | 210 | licenses.json 211 | 212 | # Temp file 213 | *.*~ -------------------------------------------------------------------------------- /kafka_conf_default.md: -------------------------------------------------------------------------------- 1 | ``` 2 | > env JMX_PORT=9999 ./kafka-server-start.sh ../config/server.properties 3 | INFO KafkaConfig values: 4 | advertised.host.name = null 5 | advertised.listeners = null 6 | advertised.port = null 7 | authorizer.class.name = 8 | auto.create.topics.enable = true 9 | auto.leader.rebalance.enable = true 10 | background.threads = 10 11 | broker.id = 0 12 | broker.id.generation.enable = true 13 | broker.rack = null 14 | compression.type = producer 15 | connections.max.idle.ms = 600000 16 | controlled.shutdown.enable = true 17 | controlled.shutdown.max.retries = 3 18 | controlled.shutdown.retry.backoff.ms = 5000 19 | controller.socket.timeout.ms = 30000 20 | create.topic.policy.class.name = null 21 | default.replication.factor = 1 22 | delete.topic.enable = true 23 | fetch.purgatory.purge.interval.requests = 1000 24 | group.max.session.timeout.ms = 300000 25 | group.min.session.timeout.ms = 6000 26 | host.name = 27 | inter.broker.listener.name = null 28 | inter.broker.protocol.version = 0.10.2-IV0 29 | leader.imbalance.check.interval.seconds = 300 30 | leader.imbalance.per.broker.percentage = 10 31 | listener.security.protocol.map = SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,TRACE:TRACE,SASL_SSL:SASL_SSL,PLAINTEXT:PLAINTEXT 32 | listeners = null 33 | log.cleaner.backoff.ms = 15000 34 | log.cleaner.dedupe.buffer.size = 134217728 35 | log.cleaner.delete.retention.ms = 86400000 36 | log.cleaner.enable = true 37 | log.cleaner.io.buffer.load.factor = 0.9 38 | log.cleaner.io.buffer.size = 524288 39 | log.cleaner.io.max.bytes.per.second = 1.7976931348623157E308 40 | log.cleaner.min.cleanable.ratio = 0.5 41 | log.cleaner.min.compaction.lag.ms = 0 42 | log.cleaner.threads = 1 43 | log.cleanup.policy = [delete] 44 | log.dir = /tmp/kafka-logs 45 | log.dirs = /tmp/kafka-logs 46 | log.flush.interval.messages = 9223372036854775807 47 | log.flush.interval.ms = null 48 | log.flush.offset.checkpoint.interval.ms = 60000 49 | log.flush.scheduler.interval.ms = 9223372036854775807 50 | log.index.interval.bytes = 4096 51 | log.index.size.max.bytes = 10485760 52 | log.message.format.version = 0.10.2-IV0 53 | log.message.timestamp.difference.max.ms = 9223372036854775807 54 | log.message.timestamp.type = CreateTime 55 | log.preallocate = false 56 | log.retention.bytes = -1 57 | log.retention.check.interval.ms = 300000 58 | log.retention.hours = 168 59 | log.retention.minutes = null 60 | log.retention.ms = null 61 | log.roll.hours = 168 62 | log.roll.jitter.hours = 0 63 | log.roll.jitter.ms = null 64 | log.roll.ms = null 65 | log.segment.bytes = 1073741824 66 | log.segment.delete.delay.ms = 60000 67 | max.connections.per.ip = 2147483647 68 | max.connections.per.ip.overrides = 69 | message.max.bytes = 1000012 70 | metric.reporters = [] 71 | metrics.num.samples = 2 72 | metrics.recording.level = INFO 73 | metrics.sample.window.ms = 30000 74 | min.insync.replicas = 1 75 | num.io.threads = 8 76 | num.network.threads = 3 77 | num.partitions = 1 78 | num.recovery.threads.per.data.dir = 1 79 | num.replica.fetchers = 1 80 | offset.metadata.max.bytes = 4096 81 | offsets.commit.required.acks = -1 82 | offsets.commit.timeout.ms = 5000 83 | offsets.load.buffer.size = 5242880 84 | offsets.retention.check.interval.ms = 600000 85 | offsets.retention.minutes = 1440 86 | offsets.topic.compression.codec = 0 87 | offsets.topic.num.partitions = 50 88 | offsets.topic.replication.factor = 3 89 | offsets.topic.segment.bytes = 104857600 90 | port = 9092 91 | principal.builder.class = class org.apache.kafka.common.security.auth.DefaultPrincipalBuilder 92 | producer.purgatory.purge.interval.requests = 1000 93 | queued.max.requests = 500 94 | quota.consumer.default = 9223372036854775807 95 | quota.producer.default = 9223372036854775807 96 | quota.window.num = 11 97 | quota.window.size.seconds = 1 98 | replica.fetch.backoff.ms = 1000 99 | replica.fetch.max.bytes = 1048576 100 | replica.fetch.min.bytes = 1 101 | replica.fetch.response.max.bytes = 10485760 102 | replica.fetch.wait.max.ms = 500 103 | replica.high.watermark.checkpoint.interval.ms = 5000 104 | replica.lag.time.max.ms = 10000 105 | replica.socket.receive.buffer.bytes = 65536 106 | replica.socket.timeout.ms = 30000 107 | replication.quota.window.num = 11 108 | replication.quota.window.size.seconds = 1 109 | request.timeout.ms = 30000 110 | reserved.broker.max.id = 1000 111 | sasl.enabled.mechanisms = [GSSAPI] 112 | sasl.kerberos.kinit.cmd = /usr/bin/kinit 113 | sasl.kerberos.min.time.before.relogin = 60000 114 | sasl.kerberos.principal.to.local.rules = [DEFAULT] 115 | sasl.kerberos.service.name = null 116 | sasl.kerberos.ticket.renew.jitter = 0.05 117 | sasl.kerberos.ticket.renew.window.factor = 0.8 118 | sasl.mechanism.inter.broker.protocol = GSSAPI 119 | security.inter.broker.protocol = PLAINTEXT 120 | socket.receive.buffer.bytes = 102400 121 | socket.request.max.bytes = 104857600 122 | socket.send.buffer.bytes = 102400 123 | ssl.cipher.suites = null 124 | ssl.client.auth = none 125 | ssl.enabled.protocols = [TLSv1.2, TLSv1.1, TLSv1] 126 | ssl.endpoint.identification.algorithm = null 127 | ssl.key.password = null 128 | ssl.keymanager.algorithm = SunX509 129 | ssl.keystore.location = null 130 | ssl.keystore.password = null 131 | ssl.keystore.type = JKS 132 | ssl.protocol = TLS 133 | ssl.provider = null 134 | ssl.secure.random.implementation = null 135 | ssl.trustmanager.algorithm = PKIX 136 | ssl.truststore.location = null 137 | ssl.truststore.password = null 138 | ssl.truststore.type = JKS 139 | unclean.leader.election.enable = true 140 | zookeeper.connect = localhost:2181 141 | zookeeper.connection.timeout.ms = 6000 142 | zookeeper.session.timeout.ms = 6000 143 | zookeeper.set.acl = false 144 | zookeeper.sync.time.ms = 2000 145 | ``` 146 | -------------------------------------------------------------------------------- /Run_KafkaCluster_with_jmx.md: -------------------------------------------------------------------------------- 1 | # Run Kafka Cluster & Zookeeper & Producer & Consumer with jmx option 2 | - Apache Kafka와 관련된 컴퍼넌트들의 jmx metrics를 수집하기 위한 설정 3 | 4 | ## [PART 0] Run Apache Kafka & Zookeeper Cluster with jmx option 5 | 6 | ### STEP 1. Run Apache Zookeeper with JMX_PORT environment 7 | 8 | #### Setting Zookeeper configuration 9 | - conf/zoo.cfg 10 | 11 | ``` 12 | tickTime=2000 13 | initLimit=10 14 | syncLimit=5 15 | dataDir=/home/poc/zookeeper-3.4.10/data 16 | clientPort=2181 17 | server.1=192.178.50.1:2888:3888 18 | server.2=192.178.50.2:2888:3888 19 | server.3=192.178.50.3:2888:3888 20 | ``` 21 | 22 | - data/myid 23 | 24 | - conf/java.env 수정 25 | ``` 26 | export JVMFLAGS="-Xmx2048m" 27 | ``` 28 | 29 | - bin/zkServer.sh 수정 30 | ``` 31 | ZOOMAIN="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9998 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false org.apache.zookeeper.server.quorum.QuorumPeerMain" 32 | ``` 33 | 34 | #### Run Zookeeper cluster (Cluster) 35 | ``` 36 | > bin/zkServer.sh start 37 | ``` 38 | 39 | #### Run Zookeeper cluster (Standalone) 40 | 41 | ``` 42 | > env JMX_PORT=9998 bin/zookeeper-server-start.sh config/zookeeper.properties 43 | ``` 44 | 45 | 46 | #### check zookeeper status 47 | ``` 48 | > echo ruok | nc localhost 2181 49 | imok (정상) 50 | ``` 51 | 52 | ### STEP 2. Run Apache Kafka with JMX_PORT environment 53 | 54 | #### Setting Kafka Configuration 55 | - conf/server.properties 56 | ``` 57 | broker.id=1 58 | log.dirs=/logdata/data/kafka 59 | zookeeper.connect=192.168.100.135:2181,192.xxx.xxx.136:2181,192.xxx.xxx.137:2181 60 | ``` 61 | 62 | #### Config JMS configuration 63 | - bin/kafka-run-class.sh 64 | - 외부 jmx client(jconsole 등)에서 접속 할 수 있도록, IP/hostname 정보를 등록한다. 65 | ``` 66 | # JMX settings 67 | if [ -z "$KAFKA_JMX_OPTS" ]; then 68 | KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=34.85.37.182(kafka broker ip or hostname) -Djava.net.preferIPv4Stack=true" 69 | 70 | fi 71 | ``` 72 | 73 | 74 | #### Run Kafka Cluster 75 | 76 | ``` 77 | > env JMX_PORT=9999 bin/kafka-server-start.sh config/server.properties 78 | ``` 79 | 80 | #### create topic (test) 81 | ``` 82 | > bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test 83 | ``` 84 | 85 | 86 | ## [PART 2] Run Kafka Producer 87 | - file에서 메시지를 읽어와서, kafka topic으로 전송한다. 88 | - 모니터링이 가능하도록 jmx port 설정을 추가하여 실행한다. 89 | 90 | ### Case 1. Run producer using logstash 91 | #### logstash jmx configuration 92 | ``` 93 | > vi bin/logstash.lib.sh 94 | -- 아래 JAVA_OPTS 항목을 추가 95 | if [ "$LS_JAVA_OPTS" ] ; then 96 | # The client set the variable LS_JAVA_OPTS, choosing his own 97 | # set of java opts. 98 | JAVA_OPTS="$JAVA_OPTS $LS_JAVA_OPTS" 99 | JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote" 100 | JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote.port=9881" 101 | JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote.local.only=false" 102 | JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote.authenticate=false" 103 | JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote.ssl=false" 104 | fi 105 | 106 | 또는 (logstash 7.4에서는 아래 옵션 필요 ) 107 | LS_JAVA_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=9881 -Dcom.sun.management.jmxremote.rmi.port=18080 -Djava.rmi.server.hostname=34.84.47.120 -Dcom.sun.management.jmxremote.local.only=false" 108 | ``` 109 | 110 | - 또는 export를 이용 111 | ``` 112 | > export LS_JAVA_OPTS=" -Dcom.sun.management.jmxremote.authenticate=false \ 113 | -Dcom.sun.management.jmxremote.port=9010 \ 114 | -Dcom.sun.management.jmxremote.ssl=false \ 115 | -Dcom.sun.management.jmxremote.authenticate=false" 116 | ``` 117 | 118 | - 또는 jmxremote.port를 제외한 값은 logstash.lib.sh에 추가하고, 119 | - jmxremote.port 만 export로 실행 120 | - logstash를 하나의 서버에서 121 | ``` 122 | > export LS_JAVA_OPTS=" -Dcom.sun.management.jmxremote.port=9881 " 123 | ``` 124 | 125 | #### logstash config 126 | - file을 읽어와서 kafka topic으로 메시지를 전송한다. 127 | - config/logstash_producer.yml 128 | 129 | ```yml 130 | 131 | input { 132 | file { 133 | path => ["$1/F1.dat"] 134 | start_position => "beginning" 135 | } 136 | } 137 | 138 | output { 139 | kafka { 140 | codec => plain { format => "%{message}" } 141 | bootstrap_servers => "broker06:9092" 142 | topic_id => "F1" 143 | acks => "all" 144 | retries => 20 145 | batch_size => 49152 146 | linger_ms => 2000 147 | } 148 | } 149 | ``` 150 | 151 | #### Run logstash 152 | ``` 153 | > logstash -f config/logstash_producer.conf 154 | ``` 155 | 156 | ### Case 2. Run producer using filebeat 157 | - logstash의 경우 여러개 process를 구동하면 서버의 자원을 너무 많이 소모한다. 158 | - 따라서 성능 테스트와 같이 여러개의 producer를 구동해야 할 경우에는 159 | filebeat과 같은 경량화된 도구를 사용하는 것이 효과적이다 160 | 161 | #### filebeat producer config file 162 | 163 | - filebeat_producer.yml 164 | ```yml 165 | filebeat.prospectors: 166 | - input_type: log 167 | document_type: F1 168 | paths: 169 | - /home/poc/data/F1.dat 170 | 171 | - input_type: log 172 | document_type: test2 173 | paths: 174 | - /home/poc/data/F2.dat 175 | 176 | output.kafka: 177 | # The Logstash hosts 178 | hosts: ["broker06:9092","broker07:9092","broker08:9092"] 179 | topic: '%{[type]}' 180 | codec.format: 181 | string: '%{[message]}' 182 | required_acks: -1 183 | max_retries: 3 184 | bulk_max_size: 2048 185 | ``` 186 | 187 | #### run flebeat 188 | ``` 189 | > filebeat -c conf/filebeat_producer.yml -d "publish" & 190 | ``` 191 | 192 | ### Case 3. Run producer using kafka command line 193 | - filebeat와 logstash의 경우 lz4 압축 옵션을 지원하지 못한다. 194 | - 따라서 해당 압축 옵션을 이용하여 성능을 측정하는 경우 직접 producer를 구현하거나, kafka에서 제공하는 tool을 활용한다. 195 | ``` 196 | > kafka-console-producer.sh —broker-list broker06:9092,broker07:9092,broker08:9092 —topic F1 —sync —compression-codec 'lz4' < F1.dat 197 | ``` 198 | 199 | 200 | ## [PART 3] Run Kafka Consumer 201 | - Kafka Producer와 마찬가지로 jmx 설정 202 | 203 | #### setting logstash config file 204 | 205 | - logstash_consumer.conf 206 | ```yml 207 | input { 208 | kafka { 209 | type => "F1" 210 | bootstrap_servers => "broker06:9092,broker07:9092,broker08:9092" 211 | topics => "F1" 212 | group_id => "logstash" 213 | max_poll_records => "5" 214 | receive_buffer_bytes => "10240" 215 | } 216 | } 217 | 218 | output { 219 | file { 220 | path => "F1.dat" 221 | codec => line { format => "%{message}" } 222 | } 223 | } 224 | 225 | ``` 226 | 227 | #### Run logstash 228 | ``` 229 | > logstash -f config/logstash_consumer.conf 230 | ``` 231 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/Kafka_Monitoring_using_docker.md: -------------------------------------------------------------------------------- 1 | # Implementing Kafka Monitoring System using docker_compose 2 | - docker_compose를 이용하여 kafka broker, zookeeper, producer, consumer에 대한 정보를 수집하는 시스템 구성 3 | - 사용자가 별도의 오픈소스 설치없이, logstash config 파일만 수정하여 모니터링 가능한 구조 4 | - kibana의 시각화는 미리 정의된 visualization & dashboard file을 import하여 한번에 구성 가능 5 | 6 | 7 | 8 | # [PART 0]. Prerequisite 9 | - Kafka Monitoring 시스템 구축을 위해 사전에 필요한 소프트웨어 및 설정 10 | - Software environment 11 | ``` 12 | Apache Kafka : 0.11.0 13 | elasticsearch : 5.5.1 14 | kibana : 5.5.1 15 | logstash : 5.5.1 16 | ``` 17 | 18 | ## STEP 1. Software 설치 및 실행 (docker-compose 실행에 필요) 19 | - docker-ce 설치 및 실행 20 | - docker-compose 설치 및 실행 21 | 22 | - https://github.com/freepsw/simple-log-monitoring-using-docker/tree/master/04.elastic#step-01-install-necessary-sw 참고 23 | 24 | ## STEP 2. Check Kafka Environment (IP, JMX Port) 25 | - 모니터링할 서버의 IP 목록과 JMX Port를 사전에 확인 26 | - Kafka Broker 27 | - Zookeeper 28 | - Kafka Producer 29 | - Kafka Consumer 30 | 31 | - 만약 JMX port가 설정되지 않았다면, 모니터링할 jmx metrics를 수집할 수 없다. 32 | 33 | 34 | # [PART 1]. Configure docker-dompose for Kafka Monitoring 35 | - 아래 링크에서 elk stack을 docker-compse로 구성하기 위한 가이드를 자세히 제공한다. 36 | - https://github.com/deviantony/docker-elk 참고 37 | 38 | ## STEP 1. Download ELK project 39 | - logstash -> elasticsearch --> kibana 실행을 위한 기본 docker-compose를 다운로드 40 | - github에 공개된 docker-compose project를 활용한다. 41 | - https://github.com/deviantony/docker-elk 42 | 43 | ``` 44 | > cd ~ 45 | > git clone https://github.com/deviantony/docker-elk.git 46 | > cd dokeer-elk 47 | ``` 48 | 49 | ## STEP 2. Configure logstash pipeline setting 50 | - elasticsearch에서 제공하는 기본 Dockerfile에 logstash 실행과 관련된 내용이 작성됨. 51 | - 또한 docker-compose에서 logstash pipiline을 정의한 config파일의 경로를 52 | - docker run 시에 volume을 이용하여 참고하도록 설정하였다 53 | - docker-compose.yml 54 | ``` 55 | logstash: 56 | build: logstash/ 57 | volumes: 58 | - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml 59 | - ./logstash/pipeline:/usr/share/logstash/pipeline 60 | ports: 61 | - "5000:5000" 62 | environment: 63 | LS_JAVA_OPTS: "-Xmx256m -Xms256m" 64 | networks: 65 | - elk 66 | depends_on: 67 | - elasticsearch 68 | ``` 69 | ### pipeline 디렉토리 아래에 jmx 수집을 위한 설정 파일 복사 70 | ``` 71 | > cp config/logstash_jmx_kafka.yml docker-elk/logstash/pipeline 72 | > cp -R config/jmx_conf docker-elk/logstash/pipeline 73 | ``` 74 | ### logstash에서 jmx를 사용할 수 있도록 plugin을 설치 75 | - 아래의 plugin 설치 스크립트 추가 76 | - vi docker-elk/logstash/Dockerfile 77 | ``` 78 | RUN logstash-plugin install logstash-input-jmx 79 | ``` 80 | - docker image 가 새롭게 build 되어야 하므로, 기존 image가 존재한다면 docker rmi로 삭제한 후 실행 81 | 82 | 83 | ## STEP 3. Run ELK Stack using docker-compose 84 | - 아래의 명령어 만으로 logstash -> elasticsearch --> kibana 프로세스 구동이 완료된다. 85 | ``` 86 | > cd ~ 87 | > git clone https://github.com/deviantony/docker-elk.git 88 | > cd dokeer-elk 89 | > docker-compose up 90 | ``` 91 | 92 | - 정상적으로 jmx metrics를 수집한다면, 아래와 같은 로그가 출력되어야 함. 93 | ``` 94 | logstash_1 | { 95 | logstash_1 | "metric_value_number" => -1, 96 | logstash_1 | "path" => "/usr/share/logstash/pipeline/jmx", 97 | logstash_1 | "@timestamp" => 2017-08-10T01:26:36.021Z, 98 | logstash_1 | "@version" => "1", 99 | logstash_1 | "host" => "10.178.50.105", 100 | logstash_1 | "metric_path" => "kafkabroker2.Memory.NonHeapMemoryUsage.max", 101 | logstash_1 | "type" => "jmx" 102 | logstash_1 | } 103 | ``` 104 | 105 | 106 | # [PART 3]. Monitoring Kafka jmx metrics 107 | ## STEP 1. Open Kibana Web UI 108 | - http://:5601 로 kibana로 접속 109 | 110 | ## STEP 2. Create Kibana index 111 | - Management > Index Patterns > Create Index Pattern 클릭 112 | - "Index name or pattern"의 텍스트 박스에 "kafka_mon"을 입력하고, 113 | - "Time Filter field name"의 select box에서 "@timestamp"를 선택 114 | - "Create" 버튼 클릭 115 | 116 | ## STEP 3. Import Kibana dashboard object 117 | - Management > Saved Objects > Import 클릭 118 | - 현재 폴더 아래의 "kibana/kibana_export_everything.json" 파일 선택 119 | 120 | ## STEP 4. View imported dashboard 121 | - Dashboard > KafkaMon-Dash(Resource) 클릭 122 | - Kafka Broker 프로세스의 cpu, memory 사용량을 모니터링 가능 123 | 124 | 125 | 126 | # [ETC] 127 | 128 | ## Create logstash Dockerfile using elastic official repository 129 | - elastic 공식 github에서 제공하는 코드에는 Dockerfile이 없다. 130 | - 대신 Makefile을 이용해서 Dockerfile을 생성하고, 131 | - 이를 기준으로 image를 생성하도록 코드를 제공한다. 132 | - 그래서 Dockerfile을 보려면 Makefile을 아래와 같이 실행해야 한다. 133 | 134 | ### Install python3.5 with virtualenv 135 | - 기존 python2.x에 영향을 주지 않고, 별도의 virtualenv로 python3.5 환경을 구성 136 | - https://www.atlantic.net/community/howto/installing-python-3/ 137 | 138 | ``` 139 | wget https://www.python.org/ftp/python/3.5.4/Python-3.5.4.tgz 140 | tar xvf Python-3.5.4.tgz 141 | sudo yum groupinstall "Development Tools" "Development Libraries" 142 | sudo yum -y install openssl-devel 143 | sudo yum -y install readline-devel 144 | 145 | cd Python-3.5.4 146 | ./configure --prefix=/home/rts/apps/Python-3.5.4/ 147 | make && make install 148 | 149 | # virtualenv로 python3.5 환경 생성 150 | virtualenv venv --python=/home/rts/apps/Python-3.5.4/bin/python3.5 151 | source venv/bin/activate 152 | python -V 153 | ``` 154 | 155 | ### Create Dockerfile and build image 156 | ``` 157 | git clone https://github.com/elastic/logstash-docker.git 158 | cd logstash-docker 159 | 160 | ``` 161 | #### - virtualenv로 설치한 python3.5의 경로를 지정해 준다. 162 | - 기존 코드에서 virtualenv --python="python3.5.4경로지정" 163 | ``` 164 | vi Makefile 165 | # The tests are written in Python. Make a virtualenv to handle the dependencies. 166 | venv: requirements.txt 167 | test -d venv || virtualenv --python=/home/rts/apps/Python-3.5.4/bin/python3.5 venv 168 | pip install -r requirements.txt 169 | touch venv 170 | ``` 171 | 172 | #### - logstash 용 Dockerfile 확인하기 173 | 174 | ``` 175 | make test 176 | ``` 177 | - Dockerfile은 정상적으로 생성됨 (build/logstash/Dockerfile) 178 | - docker build 시에 아래와 같은 오류가 발생함. (나중에 확인 필요) 179 | ``` 180 | docker run --rm -i \ 181 | -v /home/rts/apps/logstash-docker/build/logstash/env2yaml:/usr/local/src/env2yaml \ 182 | golang:env2yaml 183 | /usr/bin/env: python3: No such file or directory 184 | docker build --pull -t docker.elastic.co/logstash/logstash: build/logstash 185 | invalid argument "docker.elastic.co/logstash/logstash:" for t: Error parsing reference: "docker.elastic.co/logstash/logstash:" is not a valid repository/tag 186 | See 'docker build --help'. 187 | make: *** [build] Error 125 188 | ``` 189 | 190 | ### - 생성된 Dockerfile 참고 191 | ``` 192 | # This Dockerfile was generated from templates/Dockerfile.j2 193 | FROM centos:7 194 | LABEL maintainer "Elastic Docker Team " 195 | 196 | # Install Java and the "which" command, which is needed by Logstash's shell 197 | # scripts. 198 | RUN yum update -y && yum install -y java-1.8.0-openjdk-devel which && \ 199 | yum clean all 200 | 201 | # Provide a non-root user to run the process. 202 | RUN groupadd --gid 1000 logstash && \ 203 | adduser --uid 1000 --gid 1000 \ 204 | --home-dir /usr/share/logstash --no-create-home \ 205 | logstash 206 | 207 | # Add Logstash itself. 208 | RUN curl -Lo - https://artifacts.elastic.co/downloads/logstash/logstash-.tar.gz | \ 209 | tar zxf - -C /usr/share && \ 210 | mv /usr/share/logstash- /usr/share/logstash && \ 211 | chown --recursive logstash:logstash /usr/share/logstash/ && \ 212 | ln -s /usr/share/logstash /opt/logstash 213 | 214 | ENV ELASTIC_CONTAINER true 215 | ENV PATH=/usr/share/logstash/bin:$PATH 216 | 217 | # Provide a minimal configuration, so that simple invocations will provide 218 | # a good experience. 219 | ADD config/logstash.yml config/log4j2.properties /usr/share/logstash/config/ 220 | ADD pipeline/default.conf /usr/share/logstash/pipeline/logstash.conf 221 | RUN chown --recursive logstash:logstash /usr/share/logstash/config/ /usr/share/logstash/pipeline/ 222 | 223 | # Ensure Logstash gets a UTF-8 locale by default. 224 | ENV LANG='en_US.UTF-8' LC_ALL='en_US.UTF-8' 225 | 226 | # Place the startup wrapper script. 227 | ADD bin/docker-entrypoint /usr/local/bin/ 228 | RUN chmod 0755 /usr/local/bin/docker-entrypoint 229 | 230 | USER logstash 231 | 232 | RUN cd /usr/share/logstash && LOGSTASH_PACK_URL=https://artifacts.elastic.co/downloads/logstash-plugins logstash-plugin install x-pack 233 | 234 | ADD env2yaml/env2yaml /usr/local/bin/ 235 | 236 | EXPOSE 9600 5044 237 | 238 | ENTRYPOINT ["/usr/local/bin/docker-entrypoint"] 239 | ``` 240 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Monitoring apache kafka using monitoring tools 2 | - Apache Kafka를 구성하는 Broker, Producer, Consumer, Zookeeper를 통합 모니터링하기 위한 방안 검토 3 | - Producer -> Kafka Clustr & Zookeeper Cluster -> Consumer 4 | 5 | ## [PART 0] Run Apache Kafka & Zookeeper Cluster with jmx option 6 | 7 | 8 | ### STEP 1. Run Apache Zookeeper with JMX_PORT environment 9 | 10 | #### Setting Zookeeper configuration 11 | - conf/zoo.cfg 12 | 13 | ``` 14 | tickTime=2000 15 | initLimit=10 16 | syncLimit=5 17 | dataDir=/home/poc/zookeeper-3.4.10/data 18 | clientPort=2181 19 | server.1=192.178.50.1:2888:3888 20 | server.2=192.178.50.2:2888:3888 21 | server.3=192.178.50.3:2888:3888 22 | ``` 23 | 24 | - data/myid 25 | 26 | - conf/java.env 수정 27 | ``` 28 | export JVMFLAGS="-Xmx2048m" 29 | ``` 30 | 31 | - bin/zkServer.sh 수정 32 | ``` 33 | ZOOMAIN="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9998 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false org.apache.zookeeper.server.quorum.QuorumPeerMain" 34 | ``` 35 | 36 | #### Run Zookeeper cluster (Cluster) 37 | ``` 38 | > bin/zkServer.sh start 39 | ``` 40 | 41 | #### Run Zookeeper cluster (Standalone) 42 | 43 | ``` 44 | > env JMX_PORT=9998 bin/zookeeper-server-start.sh config/zookeeper.properties 45 | ``` 46 | 47 | 48 | #### check zookeeper status 49 | ``` 50 | > echo ruok | nc localhost 2181 51 | imok (정상) 52 | ``` 53 | 54 | ### STEP 2. Run Apache Kafka with JMX_PORT environment 55 | 56 | #### Setting Kafka Configuration 57 | - conf/server.properties 58 | ``` 59 | broker.id=1 60 | log.dirs=/logdata/data/kafka 61 | zookeeper.connect=192.168.100.135:2181,192.xxx.xxx.136:2181,192.xxx.xxx.137:2181 62 | ``` 63 | 64 | #### Run Kafka Cluster 65 | 66 | ``` 67 | > env JMX_PORT=9999 bin/kafka-server-start.sh config/server.properties 68 | ``` 69 | 70 | #### create topic (test) 71 | ``` 72 | > bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test 73 | ``` 74 | 75 | 76 | ## [PART 2] Run Kafka Producer 77 | - file에서 메시지를 읽어와서, kafka topic으로 전송한다. 78 | - 모니터링이 가능하도록 jmx port 설정을 추가하여 실행한다. 79 | 80 | ### Case 1. Run producer using logstash 81 | #### logstash jmx configuration 82 | ``` 83 | > vi bin/logstash.lib.sh 84 | -- 아래 JAVA_OPTS 항목을 추가 85 | if [ "$LS_JAVA_OPTS" ] ; then 86 | # The client set the variable LS_JAVA_OPTS, choosing his own 87 | # set of java opts. 88 | JAVA_OPTS="$JAVA_OPTS $LS_JAVA_OPTS" 89 | JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote" 90 | JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote.port=9010" 91 | JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote.local.only=false" 92 | JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote.authenticate=false" 93 | JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote.ssl=false" 94 | fi 95 | ``` 96 | 97 | - 또는 export를 이용 98 | ``` 99 | > export LS_JAVA_OPTS=" -Dcom.sun.management.jmxremote.authenticate=false \ 100 | -Dcom.sun.management.jmxremote.port=9010 \ 101 | -Dcom.sun.management.jmxremote.ssl=false \ 102 | -Dcom.sun.management.jmxremote.authenticate=false" 103 | ``` 104 | 105 | - 또는 jmxremote.port를 제외한 값은 logstash.lib.sh에 추가하고, 106 | - jmxremote.port 만 export로 실행 107 | - logstash를 하나의 서버에서 108 | ``` 109 | > export LS_JAVA_OPTS=" -Dcom.sun.management.jmxremote.port=9881 " 110 | ``` 111 | 112 | #### logstash config 113 | - file을 읽어와서 kafka topic으로 메시지를 전송한다. 114 | - config/logstash_producer.yml 115 | 116 | ```yml 117 | 118 | input { 119 | file { 120 | path => ["$1/F1.dat"] 121 | start_position => "beginning" 122 | } 123 | } 124 | 125 | output { 126 | kafka { 127 | codec => plain { format => "%{message}" } 128 | bootstrap_servers => "broker06:9092" 129 | topic_id => "F1" 130 | acks => "all" 131 | retries => 20 132 | batch_size => 49152 133 | linger_ms => 2000 134 | } 135 | } 136 | ``` 137 | 138 | #### Run logstash 139 | ``` 140 | > logstash -f config/logstash_producer.conf 141 | ``` 142 | 143 | ### Case 2. Run producer using filebeat 144 | - logstash의 경우 여러개 process를 구동하면 서버의 자원을 너무 많이 소모한다. 145 | - 따라서 성능 테스트와 같이 여러개의 producer를 구동해야 할 경우에는 146 | filebeat과 같은 경량화된 도구를 사용하는 것이 효과적이다 147 | 148 | #### filebeat producer config file 149 | 150 | - filebeat_producer.yml 151 | ```yml 152 | filebeat.prospectors: 153 | - input_type: log 154 | document_type: F1 155 | paths: 156 | - /home/poc/data/F1.dat 157 | 158 | - input_type: log 159 | document_type: test2 160 | paths: 161 | - /home/poc/data/F2.dat 162 | 163 | output.kafka: 164 | # The Logstash hosts 165 | hosts: ["broker06:9092","broker07:9092","broker08:9092"] 166 | topic: '%{[type]}' 167 | codec.format: 168 | string: '%{[message]}' 169 | required_acks: -1 170 | max_retries: 3 171 | bulk_max_size: 2048 172 | ``` 173 | 174 | #### run flebeat 175 | ``` 176 | > filebeat -c conf/filebeat_producer.yml -d "publish" & 177 | ``` 178 | 179 | ### Case 3. Run producer using kafka command line 180 | - filebeat와 logstash의 경우 lz4 압축 옵션을 지원하지 못한다. 181 | - 따라서 해당 압축 옵션을 이용하여 성능을 측정하는 경우 직접 producer를 구현하거나, kafka에서 제공하는 tool을 활용한다. 182 | ``` 183 | > kafka-console-producer.sh —broker-list broker06:9092,broker07:9092,broker08:9092 —topic F1 —sync —compression-codec 'lz4' < F1.dat 184 | ``` 185 | 186 | 187 | ## [PART 3] Run Kafka Consumer 188 | - Kafka Producer와 마찬가지로 jmx 설정 189 | 190 | #### setting logstash config file 191 | 192 | - logstash_consumer.conf 193 | ```yml 194 | input { 195 | kafka { 196 | type => "F1" 197 | bootstrap_servers => "broker06:9092,broker07:9092,broker08:9092" 198 | topics => "F1" 199 | group_id => "logstash" 200 | max_poll_records => "5" 201 | receive_buffer_bytes => "10240" 202 | } 203 | } 204 | 205 | output { 206 | file { 207 | path => "F1.dat" 208 | codec => line { format => "%{message}" } 209 | } 210 | } 211 | 212 | ``` 213 | 214 | #### Run logstash 215 | ``` 216 | > logstash -f config/logstash_consumer.conf 217 | ``` 218 | 219 | 220 | ## [PART 1] Monitoring Kafka using ELK stack 221 | - link (https://github.com/freepsw/kafka-monitoring/tree/master/01.Monitoring_with_ELK) 222 | 223 | ## [PART 2] Monitoring Kafka using datadog service 224 | - link 225 | 226 | 227 | 228 | # [ETC] 229 | 230 | ### 1. Clean system cache memory 231 | ``` 232 | sync 233 | echo 1 > /proc/sys/vm/drop_caches 234 | ``` 235 | 236 | 237 | ### 2. Test environment using logstash 238 | #### run kafka_producer 239 | 240 | ``` 241 | > bin/logstash -e "input { stdin {} } output { kafka { bootstrap_servers => ':9092' topic_id => 'test' } }" 242 | ``` 243 | 244 | ### 3. logstash 실행시 고려사항 245 | - path.data 옵션은 동일한 directory에서 logstash를 실행하면, 246 | - logstash instance별로 사용하는 data directory를 별도로 지정해야 함 247 | 248 | ``` 249 | > bin/logstash --path.data data1 -f config/kafka_consumer.yml 250 | ``` 251 | 252 | ### 4. kafka-console-consumer 253 | 254 | ``` 255 | > bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic test --from-beginning 256 | ``` 257 | 258 | 259 | ### 4. Kafka ERROR filebeat logs에서 "dropping too large message of size" 해결 260 | - https://discuss.elastic.co/t/filebeat-5-0beta1-with-kafka-output-multiplication-of-log-lines/62513/12 참고 261 | ### [Problem] 262 | - compression: none 으로 압축을 하지 않고, 3,000 byte 메세지를 보낼때 263 | - filebeat log에서 해당 오류가 발생함. 264 | - 특이한 부분은 이 오류가 발생하면 retry를 시도하지 않고, 메세지가 유실됨. 265 | 266 | ### [Cause] 267 | - 한번에 보내는 메세지 크기를 제한하는 Kafka Broker의 "message.max.bytes=1,000,000"을 초과하는 경우 268 | #### 이상한 점 1. 269 | - 1개 메시지 크기는 3,000 byte인데, 270 | - 1,000,000 byte(1M)를 넘을 수 있을까? 271 | - 그렇다면 1개 메시지의 크기가 아니라 어떤 크기를 의미할까? 272 | 273 | ### [Solve] 274 | - 아래 3개 설정 값을 함꼐 수정해야 정상적으로 메시지 처리가 가능하다. 275 | 276 | - Broker (server.properties) 277 | 278 | ``` 279 | message.max.bytes=10000000 (10배 증가) 280 | replica.fetch.max.bytes (이 부분은 변경하지 않았는데, 정상 동작) 281 | ``` 282 | 283 | - Producer 284 | 285 | ``` 286 | max.request.size=10000000 287 | ``` 288 | 289 | - Consumer 290 | 291 | ``` 292 | max.partition.fetch.bytes=10000000 293 | ``` 294 | 295 | #### 이상한 점 2. 296 | - producer의 설정은 변경하지 않고, 1,000,000으로 유지하고, 297 | - broker, consumer의 설정 값만 변경하였다. 298 | - 그런데 메시지가 정상적으로 처리되었다는 것은.. 299 | - 결국 보낼때는 1,000,000 이하로 보내는데, 300 | - Broker에서 1,000,000 이상으로 인식한다는 의미가 되는데... 301 | - 음.... filebeat kafka output의 버그인가? (이상하다...) 302 | 303 | #### [정확한 원인] 304 | - 0.11.0.0 부터 max.message.bytes의 의미가 batch에 포함된 전체 메시지 사이즈로 변경됨!!! 305 | - https://kafka.apache.org/documentation/#upgrade_1100_notable 306 | - The broker configuration max.message.bytes now applies to the total size of a batch of messages 307 | 308 | 309 | ### 5. Kafka Error (Error: Executing consumer group command failed due to The consumer group command timed out while waiting for group to initialize) 310 | 311 | #### [Problem] 312 | ##### kafka-consumer-groups.sh --describe 실행시 오류 발생 313 | 314 | ``` 315 | > /bin/kafka-topics.sh --describe --zookeeper localhost:2181 --topic __consumer_offsets 316 | Error: Executing consumer group command failed due to The consumer group command timed out while waiting for group to initialize: 317 | ``` 318 | 319 | - ConsumerGroupCommand throws GroupCoordinatorNotAvailableException when describing a non-existent group before the offset topic is created 320 | - http://mail-archives.apache.org/mod_mbox/kafka-dev/201606.mbox/%3CJIRA.12914176.1447867021000.57097.1465587741031@Atlassian.JIRA%3E 321 | 322 | - The offsets.topic.replication.factor broker config is now enforced upon auto topic creation. Internal auto topic creation will fail with a GROUP_COORDINATOR_NOT_AVAILABLE error until the cluster size meets this replication factor requirement. 323 | - https://kafka.apache.org/documentation/#upgrade_1100_notable 324 | 325 | 326 | 327 | #### [Solve] 328 | 329 | - 카프카 내부 토픽(consumer_offsets)을 여러 노드에 복제하도록 리밸런싱을 실행한다. 330 | - http://www.chidoo.me/index.php/2017/05/30/kafka-monitoring-and-administration 331 | - http://blog.leedohyun.pe.kr/2016/08/kafka-topic-replication-factor.html 참고 332 | 333 | ``` 334 | > sudo pip install kafka-tools 335 | > /bin/kafka-assigner -z localhost:2181 --tools-path /home/poc/kafka/bin -e set-replication-factor --topic __consumer_offsets --replication-factor 3 336 | ``` 337 | 338 | ### 6. kafka & zookeeper restart 339 | 340 | ``` 341 | /home/poc/kafka/bin/kafka-server-stop.sh 342 | /home/poc/zookeeper-3.4.10/bin/zkServer.sh stop 343 | 344 | /home/poc/zookeeper-3.4.10/bin/zkServer.sh start 345 | env JMX_PORT=9999 /home/poc/kafka/bin/kafka-server-start.sh config/server.properties & 346 | ``` 347 | 348 | 349 | #### 7. 추가 성능 테스트 350 | ##### 1) 1개 broker, topic 1개(AAA,) producer 10, lz4 처리 성능 : 100,000건 / 초 351 | ##### 2) 1개 broker, topic 3개(AAA, BBB, CCC), producer 10 * 3, lz4 성능 : 260,000건 / 초 352 | - 7/18 09:44 353 | - producer가 1대의 서버에서 구동되어 자원 부족현항으로 인하어 데이터 전송 속도가 늦음 (이로 인한 처리 건수가 낮아짐.) 354 | - 이후 producer를 3개의 다른서버에서 구동하면, 300,000건 /초 성능이 나올 것임. 355 | 356 | ``` 357 | >bin/kafka-console-producer.sh —broker-list broker01:9092,broker02:9092,broker03:9092 —topic DDD —sync —compression-codec 'lz4' < 100m.dat 358 | ``` 359 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/docker-elk/README.md: -------------------------------------------------------------------------------- 1 | # Docker ELK stack 2 | 3 | [![Join the chat at https://gitter.im/deviantony/docker-elk](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/deviantony/docker-elk?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Elastic Stack version](https://img.shields.io/badge/ELK-5.5.1-blue.svg?style=flat)](https://github.com/deviantony/docker-elk/issues/158) 4 | 5 | Run the latest version of the ELK (Elasticsearch, Logstash, Kibana) stack with Docker and Docker Compose. 6 | 7 | It will give you the ability to analyze any data set by using the searching/aggregation capabilities of Elasticsearch 8 | and the visualization power of Kibana. 9 | 10 | Based on the official Docker images: 11 | 12 | * [elasticsearch](https://github.com/elastic/elasticsearch-docker) 13 | * [logstash](https://github.com/elastic/logstash-docker) 14 | * [kibana](https://github.com/elastic/kibana-docker) 15 | 16 | **Note**: Other branches in this project are available: 17 | 18 | * ELK 5 with X-Pack support: https://github.com/deviantony/docker-elk/tree/x-pack 19 | * ELK 5 in Vagrant: https://github.com/deviantony/docker-elk/tree/vagrant 20 | * ELK 5 with Search Guard: https://github.com/deviantony/docker-elk/tree/searchguard 21 | 22 | ## Contents 23 | 24 | 1. [Requirements](#requirements) 25 | * [Host setup](#host-setup) 26 | * [SELinux](#selinux) 27 | 2. [Getting started](#getting-started) 28 | * [Bringing up the stack](#bringing-up-the-stack) 29 | * [Initial setup](#initial-setup) 30 | 3. [Configuration](#configuration) 31 | * [How can I tune the Kibana configuration?](#how-can-i-tune-the-kibana-configuration) 32 | * [How can I tune the Logstash configuration?](#how-can-i-tune-the-logstash-configuration) 33 | * [How can I tune the Elasticsearch configuration?](#how-can-i-tune-the-elasticsearch-configuration) 34 | * [How can I scale out the Elasticsearch cluster?](#how-can-i-scale-up-the-elasticsearch-cluster) 35 | 4. [Storage](#storage) 36 | * [How can I persist Elasticsearch data?](#how-can-i-persist-elasticsearch-data) 37 | 5. [Extensibility](#extensibility) 38 | * [How can I add plugins?](#how-can-i-add-plugins) 39 | * [How can I enable the provided extensions?](#how-can-i-enable-the-provided-extensions) 40 | 6. [JVM tuning](#jvm-tuning) 41 | * [How can I specify the amount of memory used by a service?](#how-can-i-specify-the-amount-of-memory-used-by-a-service) 42 | * [How can I enable a remote JMX connection to a service?](#how-can-i-enable-a-remote-jmx-connection-to-a-service) 43 | 44 | ## Requirements 45 | 46 | ### Host setup 47 | 48 | 1. Install [Docker](https://www.docker.com/community-edition#/download) version **1.10.0+** 49 | 2. Install [Docker Compose](https://docs.docker.com/compose/install/) version **1.6.0+** 50 | 3. Clone this repository 51 | 52 | ### SELinux 53 | 54 | On distributions which have SELinux enabled out-of-the-box you will need to either re-context the files or set SELinux 55 | into Permissive mode in order for docker-elk to start properly. For example on Redhat and CentOS, the following will 56 | apply the proper context: 57 | 58 | ```bash 59 | $ chcon -R system_u:object_r:admin_home_t:s0 docker-elk/ 60 | ``` 61 | 62 | ## Usage 63 | 64 | ### Bringing up the stack 65 | 66 | Start the ELK stack using `docker-compose`: 67 | 68 | ```bash 69 | $ docker-compose up 70 | ``` 71 | 72 | You can also choose to run it in background (detached mode): 73 | 74 | ```bash 75 | $ docker-compose up -d 76 | ``` 77 | 78 | Give Kibana about 2 minutes to initialize, then access the Kibana web UI by hitting 79 | [http://localhost:5601](http://localhost:5601) with a web browser. 80 | 81 | By default, the stack exposes the following ports: 82 | * 5000: Logstash TCP input. 83 | * 9200: Elasticsearch HTTP 84 | * 9300: Elasticsearch TCP transport 85 | * 5601: Kibana 86 | 87 | **WARNING**: If you're using `boot2docker`, you must access it via the `boot2docker` IP address instead of `localhost`. 88 | 89 | **WARNING**: If you're using *Docker Toolbox*, you must access it via the `docker-machine` IP address instead of 90 | `localhost`. 91 | 92 | Now that the stack is running, you will want to inject some log entries. The shipped Logstash configuration allows you 93 | to send content via TCP: 94 | 95 | ```bash 96 | $ nc localhost 5000 < /path/to/logfile.log 97 | ``` 98 | 99 | ## Initial setup 100 | 101 | ### Default Kibana index pattern creation 102 | 103 | When Kibana launches for the first time, it is not configured with any index pattern. 104 | 105 | #### Via the Kibana web UI 106 | 107 | **NOTE**: You need to inject data into Logstash before being able to configure a Logstash index pattern via the Kibana web 108 | UI. Then all you have to do is hit the *Create* button. 109 | 110 | Refer to [Connect Kibana with 111 | Elasticsearch](https://www.elastic.co/guide/en/kibana/current/connect-to-elasticsearch.html) for detailed instructions 112 | about the index pattern configuration. 113 | 114 | #### On the command line 115 | 116 | Run this command to create a Logstash index pattern: 117 | 118 | ```bash 119 | $ curl -XPUT -D- 'http://localhost:9200/.kibana/index-pattern/logstash-*' \ 120 | -H 'Content-Type: application/json' \ 121 | -d '{"title" : "logstash-*", "timeFieldName": "@timestamp", "notExpandable": true}' 122 | ``` 123 | 124 | This command will mark the Logstash index pattern as the default index pattern: 125 | 126 | ```bash 127 | $ curl -XPUT -D- 'http://localhost:9200/.kibana/config/5.5.1' \ 128 | -H 'Content-Type: application/json' \ 129 | -d '{"defaultIndex": "logstash-*"}' 130 | ``` 131 | 132 | ## Configuration 133 | 134 | **NOTE**: Configuration is not dynamically reloaded, you will need to restart the stack after any change in the 135 | configuration of a component. 136 | 137 | ### How can I tune the Kibana configuration? 138 | 139 | The Kibana default configuration is stored in `kibana/config/kibana.yml`. 140 | 141 | It is also possible to map the entire `config` directory instead of a single file. 142 | 143 | ### How can I tune the Logstash configuration? 144 | 145 | The Logstash configuration is stored in `logstash/config/logstash.yml`. 146 | 147 | It is also possible to map the entire `config` directory instead of a single file, however you must be aware that 148 | Logstash will be expecting a 149 | [`log4j2.properties`](https://github.com/elastic/logstash-docker/tree/master/build/logstash/config) file for its own 150 | logging. 151 | 152 | ### How can I tune the Elasticsearch configuration? 153 | 154 | The Elasticsearch configuration is stored in `elasticsearch/config/elasticsearch.yml`. 155 | 156 | You can also specify the options you want to override directly via environment variables: 157 | 158 | ```yml 159 | elasticsearch: 160 | 161 | environment: 162 | network.host: "_non_loopback_" 163 | cluster.name: "my-cluster" 164 | ``` 165 | 166 | ### How can I scale out the Elasticsearch cluster? 167 | 168 | Follow the instructions from the Wiki: [Scaling out 169 | Elasticsearch](https://github.com/deviantony/docker-elk/wiki/Elasticsearch-cluster) 170 | 171 | ## Storage 172 | 173 | ### How can I persist Elasticsearch data? 174 | 175 | The data stored in Elasticsearch will be persisted after container reboot but not after container removal. 176 | 177 | In order to persist Elasticsearch data even after removing the Elasticsearch container, you'll have to mount a volume on 178 | your Docker host. Update the `elasticsearch` service declaration to: 179 | 180 | ```yml 181 | elasticsearch: 182 | 183 | volumes: 184 | - /path/to/storage:/usr/share/elasticsearch/data 185 | ``` 186 | 187 | This will store Elasticsearch data inside `/path/to/storage`. 188 | 189 | **NOTE:** beware of these OS-specific considerations: 190 | * **Linux:** the [unprivileged `elasticsearch` user][esuser] is used within the Elasticsearch image, therefore the 191 | mounted data directory must be owned by the uid `1000`. 192 | * **macOS:** the default Docker for Mac configuration allows mounting files from `/Users/`, `/Volumes/`, `/private/`, 193 | and `/tmp` exclusively. Follow the instructions from the [documentation][macmounts] to add more locations. 194 | 195 | [esuser]: https://github.com/elastic/elasticsearch-docker/blob/016bcc9db1dd97ecd0ff60c1290e7fa9142f8ddd/templates/Dockerfile.j2#L22 196 | [macmounts]: https://docs.docker.com/docker-for-mac/osxfs/ 197 | 198 | ## Extensibility 199 | 200 | ### How can I add plugins? 201 | 202 | To add plugins to any ELK component you have to: 203 | 204 | 1. Add a `RUN` statement to the corresponding `Dockerfile` (eg. `RUN logstash-plugin install logstash-filter-json`) 205 | 2. Add the associated plugin code configuration to the service configuration (eg. Logstash input/output) 206 | 3. Rebuild the images using the `docker-compose build` command 207 | 208 | ### How can I enable the provided extensions? 209 | 210 | A few extensions are available inside the [`extensions`](extensions) directory. These extensions provide features which 211 | are not part of the standard Elastic stack, but can be used to enrich it with extra integrations. 212 | 213 | The documentation for these extensions is provided inside each individual subdirectory, on a per-extension basis. Some 214 | of them require manual changes to the default ELK configuration. 215 | 216 | ## JVM tuning 217 | 218 | ### How can I specify the amount of memory used by a service? 219 | 220 | By default, both Elasticsearch and Logstash start with [1/4 of the total host 221 | memory](https://docs.oracle.com/javase/8/docs/technotes/guides/vm/gctuning/parallel.html#default_heap_size) allocated to 222 | the JVM Heap Size. 223 | 224 | The startup scripts for Elasticsearch and Logstash can append extra JVM options from the value of an environment 225 | variable, allowing the user to adjust the amount of memory that can be used by each component: 226 | 227 | | Service | Environment variable | 228 | |---------------|----------------------| 229 | | Elasticsearch | ES_JAVA_OPTS | 230 | | Logstash | LS_JAVA_OPTS | 231 | 232 | To accomodate environments where memory is scarce (Docker for Mac has only 2 GB available by default), the Heap Size 233 | allocation is capped by default to 256MB per service in the `docker-compose.yml` file. If you want to override the 234 | default JVM configuration, edit the matching environment variable(s) in the `docker-compose.yml` file. 235 | 236 | For example, to increase the maximum JVM Heap Size for Logstash: 237 | 238 | ```yml 239 | logstash: 240 | 241 | environment: 242 | LS_JAVA_OPTS: "-Xmx1g -Xms1g" 243 | ``` 244 | 245 | ### How can I enable a remote JMX connection to a service? 246 | 247 | As for the Java Heap memory (see above), you can specify JVM options to enable JMX and map the JMX port on the docker 248 | host. 249 | 250 | Update the `{ES,LS}_JAVA_OPTS` environment variable with the following content (I've mapped the JMX service on the port 251 | 18080, you can change that). Do not forget to update the `-Djava.rmi.server.hostname` option with the IP address of your 252 | Docker host (replace **DOCKER_HOST_IP**): 253 | 254 | ```yml 255 | logstash: 256 | 257 | environment: 258 | LS_JAVA_OPTS: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=18080 -Dcom.sun.management.jmxremote.rmi.port=18080 -Djava.rmi.server.hostname=DOCKER_HOST_IP -Dcom.sun.management.jmxremote.local.only=false" 259 | ``` 260 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/README.md: -------------------------------------------------------------------------------- 1 | # Monitoring Kafka JMX metrics using ELK stack 2 | - Apache Kafka를 구성하는 Broker, Producer, Consumer, Zookeeper를 통합 모니터링하기 위한 방안 검토 3 | - JMX metrics 기반으로, ELK stack을 이용하여 통합 모니터링 dashboard를 구성한다. 4 | 5 | # Kafka 모니터링 아키텍처 6 | - 우리가 구축할 Kafka 모니터링 아키텍처 7 | - 8 | ![architecture evtns](https://github.com/freepsw/kafka-monitoring/blob/master/01.Monitoring_with_ELK/img_monitoing_stack.png?raw=true) 9 | 10 | # [PART 1] Install and run Elasticsearch & Kibana 11 | ## STEP 1. Elasticsearch-5.4.3 Install 12 | ``` 13 | > mkdir ~/apps 14 | > cd ~/apps 15 | > wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.6.1.tar.gz 16 | > tar xvf elasticsearch-6.6.1.tar.gz 17 | > cd elasticsearch-6.6.1 18 | > vi config/elasticsearch.yml (network.host: 0.0.0.0) 19 | > bin/elasticsearch 20 | ``` 21 | 22 | ### [Error 해결] max file descriptors [4096] for elasticsearch process is too low 오류 발생시 23 | - file descriptors 수를 증가 24 | - https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-system-settings.html#limits.conf 참고 25 | ``` 26 | sudo vi /etc/security/limits.conf 27 | 28 | username - nofile 65536 29 | username - nproc 262144 30 | ``` 31 | 32 | ### [Error 해결] max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144] 33 | 34 | ``` 35 | > sudo vi /etc/sysctl.conf 36 | > sudo sysctl -a # 설정값 확인 37 | > sudo sysctl -w vm.max_map_count=262144 38 | ``` 39 | 40 | 41 | ## STEP 2. Kibana Install 42 | 43 | ``` 44 | > mkdir ~/apps 45 | > cd ~/apps 46 | > wget https://artifacts.elastic.co/downloads/kibana/kibana-6.6.1-linux-x86_64.tar.gz 47 | > tar xvf kibana-6.6.1-linux-x86_64.tar.gz 48 | > cd kibana-6.6.1-linux-x86_64 49 | > vi config/kibana.yml (server.host: "0.0.0.0") 50 | > cd kibana-5.4.3-linux-x86_64 51 | > bin/kibana 52 | 53 | # backgroud 실행 54 | > nohup bin/kibana & 55 | > ps -ef | grep node 56 | ``` 57 | 58 | 59 | # [PART 2]. Collect Kafka JMX Metrics using logstash 60 | 61 | ## STEP 1. Collect Kafka Broker/Producer/Consumer jmx metrics using logstash (jmx_port 9999) 62 | - https://www.datadoghq.com/blog/monitoring-kafka-performance-metrics/ 참고 63 | - http://docs.confluent.io/current/kafka/monitoring.html#new-consumer-metrics 참고 64 | 65 | ``` 66 | > cd ~/apps 67 | > wget https://artifacts.elastic.co/downloads/logstash/logstash-6.6.1.tar.gz 68 | > tar xvf logstash-6.6.1.tar.gz 69 | > cd logstash-6.6.1 70 | > bin/logstash-plugin install logstash-input-jmx 71 | ``` 72 | 73 | ### Logstash 수집을 위한 config 설정 74 | - jmx input plugin을 이용하여 jmx metrics를 수집한다. 75 | - jmx path 아래에 있는 jmx 설정파일 들을 수집한다. (broker, producer, consumer, zookeeper) 76 | - logstash_jmx_kafka.yml 77 | 78 | ```yml 79 | input { 80 | jmx { 81 | path => "/home/poc/jmx_conf" 82 | polling_frequency => 2 83 | type => "jmx" 84 | } 85 | } 86 | 87 | filter { 88 | if [type] == "jmx" { 89 | if ("ProcessCpuLoad" in [metric_path] or "SystemCpuLoad" in [metric_path]) { 90 | ruby { 91 | code => "n = event.get('[metric_value_number]') 92 | event.set('[metric_value_number]', n*100) 93 | event.set('cpu_load',n) 94 | " 95 | } 96 | } 97 | if ("TotalPhysicalMemorySize" in [metric_path] or "FreePhysicalMemorySize" in [metric_path]) { 98 | ruby { 99 | code => "n = event.get('[metric_value_number]') 100 | event.set('[metric_value_number]', n/1024/1024) 101 | event.set('physical_memory',n) 102 | " 103 | } 104 | } 105 | if ("HeapMemoryUsage" in [metric_path] or "HeapMemoryUsage" in [metric_path]) { 106 | ruby { 107 | code => "n = event.get('[metric_value_number]') 108 | event.set('[metric_value_number]', n/1024/1024) 109 | " 110 | } 111 | } 112 | } 113 | } 114 | 115 | output{ 116 | stdout { 117 | codec => rubydebug 118 | } 119 | elasticsearch { 120 | hosts => "localhost:9200" 121 | index => "kafka_mon" 122 | } 123 | } 124 | ``` 125 | - 각 event의 메시지 별로 별도의 로직을 추가하거나, 필드를 추가할 수 있다. 126 | - 위의 예시에서 event api에서 제공하는 event.set을 통하여 값을 변경하거나, 필드를 추가함. 127 | - logstash를 실행해 보면, 화면에 cpu_load라는 필드가 추가된 것을 볼 수 있다. 128 | - https://www.elastic.co/guide/en/logstash/current/event-api.html 참고 129 | 130 | ### Xpack monitoring 을 위한 설정 131 | - Kibana monitoring에서 logstash의 성능을 모니터링 하기 위한 설정 132 | - config/logstash.yml 에 xpack 모니터링을 활성화 하기 위한 설정 추가 133 | ```yml 134 | xpack.monitoring.enabled: true 135 | xpack.monitoring.elasticsearch.username: logstash_system 136 | xpack.monitoring.elasticsearch.password: password 137 | xpack.monitoring.elasticsearch.url: ["http://localhost:9200"] 138 | ``` 139 | 140 | ```yml 141 | { 142 | "metric_value_number" => 0.020206998521439132, 143 | "path" => "/home/rts/apps/logstash-5.4.1/jmx", 144 | "@timestamp" => 2017-06-22T06:26:49.297Z, 145 | "@version" => "1", 146 | "host" => "localhost", 147 | "metric_path" => "kafkabroker1.OperatingSystem.SystemCpuLoad", 148 | "type" => "jmx", 149 | "cpu_load" => 0.020206998521439132 150 | } 151 | ``` 152 | 153 | ### JMX Configuration for Broker (jmx_conf/jmx_kafka_broker.conf) 154 | - https://developers.lightbend.com/docs/opsclarity/latest/articles/Kafka-Brokers.html 참고 155 | 156 | ```shell 157 | { 158 | "host" : "broker06", 159 | "port" : 9999, 160 | "alias" : "kafkabroker1", 161 | "queries" : [ 162 | { 163 | "object_name" : "kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec", 164 | "attributes" : [ "OneMinuteRate" ], 165 | "object_alias" : "${type}.${name}" 166 | }, 167 | { 168 | "object_name" : "kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec", 169 | "attributes" : [ "OneMinuteRate" ], 170 | "object_alias" : "${type}.${name}" 171 | }, 172 | { 173 | "object_name" : "kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec", 174 | "attributes" : [ "OneMinuteRate" ], 175 | "object_alias" : "${type}.${name}" 176 | }, 177 | { 178 | "object_name" : "kafka.server:type=ReplicaManager,name=UnderReplicatedPartitions", 179 | "attributes" : [ "Value" ], 180 | "object_alias" : "${type}.${name}" 181 | }, 182 | { 183 | "object_name" : "kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Produce", 184 | "attributes" : [ "Mean" ], 185 | "object_alias" : "${type}.${name}" 186 | }, 187 | { 188 | "object_name" : "kafka.network:type=RequestMetrics,name=TotalTimeMs,request=FetchConsumer", 189 | "attributes" : [ "Mean" ], 190 | "object_alias" : "${type}.${name}" 191 | } 192 | ] 193 | } 194 | ``` 195 | 196 | #### - Broker Kafka metric 이해 (평균 값) 197 | - MeanRate, OneMinuteRate 등의 Attributes가 있는데, 어떤 차이가 있는지 이해가 필요함. (모니터링 시) 198 | - 예를 들어 kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec, BytesOutPerSec가 있는 경우 199 | - BytesOutPerSec 별로 다양한 Attributes가 존재한다. 200 | - MeanRate : producer가 구동한 이후부터 broker에 전송된 bytes/sec (이 경우, 왜곡된 값이 보일 수 있다.) 201 | * 예를 들어, producer가 매일 1시간 만 전송하는 경우, 202 | * 1시간 동안 전송된 bytes의 초당 평균 전송량을 봐야 하는데, 203 | * MeanRate를 사용하면, producer가 구동된 이후에 전송된 모든 bytes를 204 | * producer가 구동된 시간으로 평균을 구하게 됨. (따라서 낮은 값이 보임) 205 | - OneMinuteRate : 이전 60초 동안 전송된 평균 bytes 206 | - https://stackoverflow.com/questions/43677710/kafka-jmx-metric-current-rate-of-messages-in/43679512 참고 207 | 208 | 209 | 210 | ### JMX Configuration for Broker Resource (jmx_conf/jmx_kafka_broekr_resource.conf) 211 | - http://help.boomi.com/atomsphere/GUID-48F37A93-5BF9-488D-82C3-38E4E9D45A22.html 참고 212 | - CPU 부하 213 | * "SystemCpuLoad", "ProcessCpuLoad" 214 | * SystemLoadAverage : 지난 1분간 서버의 CPU 사용량 (0: 사용하지 않음. 1: 1개의 CPU가 100% 사용, 2: 2개의 CPU가 100% 사용) 215 | - Open File 현황 216 | * "OpenFileDescriptorCount" : Kafka Broker를 구동한 User의 open file 갯수 217 | - Memory 사용량 218 | * TotalPhysicalMemorySize : 전체 물리 메모리 크기 219 | * FreePhysicalMemorySize : 사용가능한 물리 메모리 크기 220 | - JVM Heap Memory 사용량 221 | 222 | ```shell 223 | { 224 | "host" : "broker06", 225 | "port" : 9999, 226 | "alias" : "kafkabroker1", 227 | "queries" : [ 228 | { 229 | "object_name" : "java.lang:type=OperatingSystem", 230 | "attributes" : ["SystemCpuLoad", "ProcessCpuLoad", "OpenFileDescriptorCount","FreePhysicalMemorySize", "TotalPhysicalMemorySize" ], 231 | "object_alias" : "${type}" 232 | }, 233 | { 234 | "object_name" : "java.lang:type=Memory", 235 | "attributes" : ["HeapMemoryUsage", "NonHeapMemoryUsage"], 236 | "object_alias" : "${type}" 237 | } 238 | ] 239 | } 240 | ``` 241 | 242 | ### JMX Configuration for Producer (jmx_conf/jmx_kafka_producer.conf) 243 | 244 | ```json 245 | { 246 | "host" : "localhost", 247 | "port" : 9881, 248 | "alias" : "kafka-producer", 249 | "queries" : [ 250 | { 251 | "object_name" : "kafka.producer:type=producer-metrics,client-id=*", 252 | "attributes" : ["outgoing-byte-rate"], 253 | "object_alias" : "Producer.BytesRate" 254 | }, 255 | { 256 | "object_name" : "kafka.producer:type=producer-metrics,client-id=*", 257 | "attributes" : ["request-rate"], 258 | "object_alias" : "RequestRate" 259 | }, 260 | { 261 | "object_name" : "kafka.producer:type=producer-metrics,client-id=*", 262 | "attributes" : ["response-rate"], 263 | }, 264 | { 265 | "object_name" : "kafka.producer:type=producer-metrics,client-id=*", 266 | "attributes" : ["request-latency-avg"], 267 | }, 268 | { 269 | "object_name" : "java.lang:type=OperatingSystem", 270 | "object_alias" : "OperatingSystem", 271 | "attributes" : ["SystemCpuLoad"] 272 | } 273 | ] 274 | } 275 | ``` 276 | 277 | ### JMX Configuration for Consumer (jmx_conf/jmx_kafka_consumer.conf) 278 | ```json 279 | { 280 | "host" : "localhost", 281 | "port" : 9882, 282 | "alias" : "kafka-consumer", 283 | "queries" : [ 284 | { 285 | "object_name" : "kafka.consumer:type=consumer-metrics,client-id=*", 286 | "attributes" : ["incoming-byte-rate"], 287 | "object_alias" : "Consumer.InBytesRate" 288 | }, 289 | { 290 | "object_name" : "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=logstash-0", 291 | "attributes" : ["bytes-consumed-rate"], 292 | "object_alias" : "Consumer.ByteConsumed" 293 | }, 294 | { 295 | "object_name" : "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=logstash-0", 296 | "attributes" : ["records-consumed-rate"], 297 | "object_alias" : "Consumer.RecordsConsumed" 298 | }, 299 | { 300 | "object_name" : "java.lang:type=OperatingSystem", 301 | "object_alias" : "OperatingSystem", 302 | "attributes" : ["SystemCpuLoad"] 303 | } 304 | ] 305 | } 306 | ``` 307 | 308 | ### JMX Configuration for Consumer (jmx_conf/jmx_kafka_consumer_lag.conf) 309 | - records-lag-max의 경우, 값이 없을 때 "-Infinity"라는 값이 저장된다 310 | - 그런데, 이 값은 string이므로 jmx 값을 파싱할 때 오류가 발생하면서, 311 | - 나머지 metrics도 수집못하게 되는 문제가 발생한다. (값이 있을 때는 정상동작) 312 | - 그래서 일단 jmx config를 분리하여 처리함. (좀 더 근본적으로는 logstash event api로 처리 가능한지 확인 필요) 313 | ```json 314 | { 315 | "host" : "localhost", 316 | "port" : 9882, 317 | "alias" : "kafka-consumer", 318 | "queries" : [ 319 | { 320 | "object_name" : "kafka.consumer:type=consumer-fetch-manager-metrics,client-id=*", 321 | "attributes" : ["records-lag-max"], 322 | "object_alias" : "Consumer.RecordsLagMax" 323 | } 324 | ] 325 | } 326 | ``` 327 | 328 | 329 | ### [참고] kafka 관련 jmx metrics 참고 330 | - https://github.com/wikimedia/puppet-kafka/blob/master/manifests/server/jmxtrans.pp 331 | 332 | 333 | ## STEP 2. Run logstash 334 | - lgstash를 실행하면, 335 | - 3개의 jmx설정에 따라서 각각의 jmx metrics를 수집하게 된다. 336 | 337 | ``` 338 | bin/logstash -f config/jmx_kafka.yml 339 | ``` 340 | - 만약 jmx plugin이 없다는 오류가 발생하면, 직접 설치한다. 341 | 342 | ``` 343 | # online install 344 | > bin/logstash-plugin install logstash-input-jmx 345 | 346 | # offline install 347 | # export plugins to zip 348 | > bin/logstash-plugin prepare-offline-pack logstash-input-jmx 349 | 350 | # import zip to logstash plugins 351 | > bin/logstash-plugin install file:///home/rts/logstash-offline-plugins-5.4.1.zip 352 | ``` 353 | 354 | 355 | ### 수집된 jmx metrics 예시 356 | 357 | ```yml 358 | { 359 | "metric_value_number" => 0.1642061100531988, 360 | "path" => "/home/rts/apps/logstash-5.4.1/jmx", 361 | "@timestamp" => 2017-06-20T09:08:19.968Z, 362 | "@version" => "1", 363 | "host" => "localhost", 364 | "metric_path" => "kafkabroker1.kafka.server:type=BrokerTopicMetrics,name=TotalFetchRequestsPerSec.FiveMinuteRate", 365 | "type" => "jmx" 366 | } 367 | ``` 368 | 369 | 370 | # [PART 3]. Monitorig JMX metrics using kibana 371 | 372 | ### STEP 1. Kibana Index 생성 373 | - Menu : Management > Index Pattenrs > "+" 버튼 클릭 > "index name" 필드에 "kafka-mon" 입력 374 | 375 | ### STEP 2. Import visualizations & dashboard object 376 | - Menu : Management > Saved Object > "import" 버튼 클릭 > 아래 2개 파일 import 377 | - kibana/kibana_dashboard.json 378 | - kibana/kibana_visualizations.json 379 | 380 | ### STEP 3. Monitoring Kafka metrics using kibana dashboard 381 | -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/kibana/kibana_visualizations.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_id": "1beb2240-5a44-11e7-a06b-bfa32df73ed4", 4 | "_type": "visualization", 5 | "_source": { 6 | "title": "KafkaMon-Producer(out-going bytes) vs Consumer(in-coming bytes) per sec", 7 | "visState": "{\"title\":\"KafkaMon-Producer(out-going bytes) vs Consumer(in-coming bytes) per sec\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"rotate\":0,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Request rate vs Response rate\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"metric_value_number\",\"customLabel\":\"Count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"custom\",\"customInterval\":\"1s\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Request rate vs Response rate\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"metric_path.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"Producer.BytesRate.outgoing-byte-rate\"}}],\"listeners\":{}}", 8 | "uiStateJSON": "{\"vis\":{\"colors\":{\"kafkabroker1.FetchConsumer.Mean\":\"#3F6833\",\"kafkabroker1.Procude.Mean\":\"#E5AC0E\"},\"legendOpen\":true}}", 9 | "description": "", 10 | "version": 1, 11 | "kibanaSavedObjectMeta": { 12 | "searchSourceJSON": "{\"index\":\"kafka_mon\",\"query\":{\"query_string\":{\"query\":\"\\\"kafka-producer.Producer.BytesRate.outgoing-byte-rate\\\" or \\\"kafka-consumer.Consumer.InBytesRate.incoming-byte-rate\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" 13 | } 14 | } 15 | }, 16 | { 17 | "_id": "68c1fc30-5b0f-11e7-afa4-79fb9d9351b8", 18 | "_type": "visualization", 19 | "_source": { 20 | "title": "KafkaMon-Producer(request vs response rate)", 21 | "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Count\",\"field\":\"metric_value_number\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"2h\",\"customLabel\":\"Consumer MaxLag\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"s\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"field\":\"metric_path.keyword\",\"order\":\"desc\",\"orderBy\":\"_term\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Consumer MaxLag\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":false,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"title\":\"KafkaMon-Producer(request vs response rate)\",\"type\":\"line\"}", 22 | "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", 23 | "description": "", 24 | "version": 1, 25 | "kibanaSavedObjectMeta": { 26 | "searchSourceJSON": "{\"index\":\"kafka_mon\",\"query\":{\"query_string\":{\"query\":\"\\\"kafka-producer.ResponseRate.response-rate\\\" or \\\"kafka-producer.RequestRate.request-rate\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" 27 | } 28 | } 29 | }, 30 | { 31 | "_id": "f1e4eb80-5a29-11e7-a406-63b0e3ccc267", 32 | "_type": "visualization", 33 | "_source": { 34 | "title": "KafkaMon-Broker(FecthConsumer)", 35 | "visState": "{\"title\":\"KafkaMon-Broker(FecthConsumer)\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Broker.FetchConsumer\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"TotalTimeMs\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":false,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"TotalTimeMs\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"metric_value_number\",\"customLabel\":\"TotalTimeMs\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"s\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Broker.FetchConsumer\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"metric_path.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\"}}],\"listeners\":{}}", 36 | "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", 37 | "description": "", 38 | "version": 1, 39 | "kibanaSavedObjectMeta": { 40 | "searchSourceJSON": "{\"index\":\"kafka_mon\",\"query\":{\"query_string\":{\"query\":\"\\\"kafkabroker1.FetchConsumer.Mean\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" 41 | } 42 | } 43 | }, 44 | { 45 | "_id": "e4a325a0-5b09-11e7-afa4-79fb9d9351b8", 46 | "_type": "visualization", 47 | "_source": { 48 | "title": "KafkaMon-Consumer(MaxLag)", 49 | "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Count\",\"field\":\"metric_value_number\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"1s\",\"customLabel\":\"Consumer MaxLag\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"custom\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"field\":\"metric_path.keyword\",\"order\":\"desc\",\"orderBy\":\"_term\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Consumer MaxLag\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":false,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"title\":\"KafkaMon-Consumer(MaxLag)\",\"type\":\"line\"}", 50 | "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", 51 | "description": "", 52 | "version": 1, 53 | "kibanaSavedObjectMeta": { 54 | "searchSourceJSON": "{\"index\":\"kafka_mon\",\"query\":{\"query_string\":{\"query\":\"\\\"kafka-consumer-lag.Consumer.RecordsLagMax.records-lag-max\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" 55 | } 56 | } 57 | }, 58 | { 59 | "_id": "1c9d7880-5ad7-11e7-a406-63b0e3ccc267", 60 | "_type": "visualization", 61 | "_source": { 62 | "title": "KafkaMon-Broker(InOut Bytes)", 63 | "visState": "{\"title\":\"KafkaMon-Broker(InOut Bytes)\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Broker in vs out Bytes\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Bytes\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Bytes\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"metric_value_number\",\"customLabel\":\"Bytes\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"custom\",\"customInterval\":\"1s\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Broker in vs out Bytes\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"metric_path.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", 64 | "uiStateJSON": "{}", 65 | "description": "", 66 | "version": 1, 67 | "kibanaSavedObjectMeta": { 68 | "searchSourceJSON": "{\"index\":\"kafka_mon\",\"query\":{\"query_string\":{\"query\":\"\\\"BrokerTopicMetrics.BytesInPerSec.OneMinuteRate\\\" or \\\"BrokerTopicMetrics.BytesOutPerSec.OneMinuteRate\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" 69 | } 70 | } 71 | }, 72 | { 73 | "_id": "09564bf0-5a30-11e7-a06b-bfa32df73ed4", 74 | "_type": "visualization", 75 | "_source": { 76 | "title": "KafkaMon-Broker vs Consumer(Message/sec)", 77 | "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Messages\",\"field\":\"metric_value_number\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"1s\",\"customLabel\":\"Broker & Consumer (Messages/Sec)\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"custom\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"field\":\"metric_path.keyword\",\"order\":\"desc\",\"orderBy\":\"_term\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"rotate\":0,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Broker & Consumer (Messages/Sec)\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Messages\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Messages\"},\"type\":\"value\"}]},\"title\":\"KafkaMon-Broker vs Consumer(Message/sec)\",\"type\":\"line\"}", 78 | "uiStateJSON": "{\"vis\":{\"colors\":{\"kafkabroker1.FetchConsumer.Mean\":\"#3F6833\",\"kafkabroker1.Procude.Mean\":\"#E5AC0E\"},\"legendOpen\":true}}", 79 | "description": "", 80 | "version": 1, 81 | "kibanaSavedObjectMeta": { 82 | "searchSourceJSON": "{\"index\":\"kafka_mon\",\"query\":{\"query_string\":{\"query\":\"\\\"kafkabroker1.BrokerTopicMetrics.MessagesInPerSec.OneMinuteRate\\\" or \\\"kafkabroker2.BrokerTopicMetrics.MessagesInPerSec.OneMinuteRate\\\", \\\"kafkabroker3.BrokerTopicMetrics.MessagesInPerSec.OneMinuteRate\\\", \\\"kafka-consumer.Consumer.RecordsConsumed.records-consumed-rate\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" 83 | } 84 | } 85 | }, 86 | { 87 | "_id": "b812adb0-5a43-11e7-a406-63b0e3ccc267", 88 | "_type": "visualization", 89 | "_source": { 90 | "title": "KafkaMon-Consumer(incoming-byte)", 91 | "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Bytes\",\"field\":\"metric_value_number\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"2h\",\"customLabel\":\"Consumer.outgoing-byte-rate/Sec\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"s\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"Producer.BytesRate.outgoing-byte-rate\",\"field\":\"metric_path.keyword\",\"order\":\"desc\",\"orderBy\":\"_term\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"rotate\":90,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Consumer.incoming-byte/Sec\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Bytes\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Bytes\"},\"type\":\"value\"}]},\"title\":\"KafkaMon-Consumer(incoming-byte)\",\"type\":\"line\"}", 92 | "uiStateJSON": "{\"vis\":{\"colors\":{\"kafkabroker1.FetchConsumer.Mean\":\"#3F6833\",\"kafkabroker1.Procude.Mean\":\"#E5AC0E\"},\"legendOpen\":false}}", 93 | "description": "", 94 | "version": 1, 95 | "kibanaSavedObjectMeta": { 96 | "searchSourceJSON": "{\"index\":\"kafka_mon\",\"query\":{\"query_string\":{\"query\":\"\\\"kafka-consumer.Consumer.InBytesRate.incoming-byte-rate\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" 97 | } 98 | } 99 | }, 100 | { 101 | "_id": "7399a5c0-6057-11e7-87bb-1dcc594557a4", 102 | "_type": "visualization", 103 | "_source": { 104 | "title": "KafkaMon-Broker(Process cpu load)", 105 | "visState": "{\"title\":\"KafkaMon-Broker(Process cpu load)\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Broker CPU Load( Process)\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"%(Load)\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"%(Load)\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"metric_value_number\",\"customLabel\":\"%(Load)\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"custom\",\"customInterval\":\"1s\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Broker CPU Load( Process)\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"metric_path.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", 106 | "uiStateJSON": "{\"vis\":{\"legendOpen\":true},\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}}", 107 | "description": "", 108 | "version": 1, 109 | "kibanaSavedObjectMeta": { 110 | "searchSourceJSON": "{\"index\":\"kafka_mon\",\"query\":{\"query_string\":{\"query\":\"\\\"OperatingSystem.ProcessCpuLoad\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" 111 | } 112 | } 113 | }, 114 | { 115 | "_id": "f7849b30-5c63-11e7-844c-bbc872ee5c30", 116 | "_type": "visualization", 117 | "_source": { 118 | "title": "KafkaMon-Broker_Server(OpenFileDescriptor Count)", 119 | "visState": "{\"title\":\"KafkaMon-Broker_Server(OpenFileDescriptor Count)\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Broker Server (Open File Descriptor Count)\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"metric_value_number\",\"customLabel\":\"Count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"custom\",\"customInterval\":\"1s\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Broker Server (Open File Descriptor Count)\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"metric_path.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", 120 | "uiStateJSON": "{}", 121 | "description": "", 122 | "version": 1, 123 | "kibanaSavedObjectMeta": { 124 | "searchSourceJSON": "{\"index\":\"kafka_mon\",\"query\":{\"query_string\":{\"query\":\"\\\"OperatingSystem.OpenFileDescriptorCount\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" 125 | } 126 | } 127 | }, 128 | { 129 | "_id": "065b3d90-5c63-11e7-8b85-6d355270351a", 130 | "_type": "visualization", 131 | "_source": { 132 | "title": "KafkaMon-Broker (Memory Heap vs NonHeap)", 133 | "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Used (MB)\",\"field\":\"metric_value_number\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"1s\",\"customLabel\":\"Broker Memory (Heap vs NonHeap)\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"custom\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"field\":\"metric_path.keyword\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Broker Memory (Heap vs NonHeap)\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Used (MB)\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Used (MB)\"},\"type\":\"value\"}]},\"title\":\"KafkaMon-Broker (Memory Heap vs NonHeap)\",\"type\":\"line\"}", 134 | "uiStateJSON": "{}", 135 | "description": "", 136 | "version": 1, 137 | "kibanaSavedObjectMeta": { 138 | "searchSourceJSON": "{\"index\":\"kafka_mon\",\"query\":{\"query_string\":{\"query\":\"\\\"Memory.HeapMemoryUsage.used\\\" or \\\"Memory.NonHeapMemoryUsage.used\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" 139 | } 140 | } 141 | }, 142 | { 143 | "_id": "0c96b8b0-5bd7-11e7-be67-49b67c124db6", 144 | "_type": "visualization", 145 | "_source": { 146 | "title": "KafkaMon-Broker Server (Memory Usage)", 147 | "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Usage (MB)\",\"field\":\"metric_value_number\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"1s\",\"customLabel\":\"Broker Server Memory Usage(Total vs Free)\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"custom\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"field\":\"metric_path.keyword\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Broker Server Memory Usage(Total vs Free)\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Usage (MB)\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Usage (MB)\"},\"type\":\"value\"}]},\"title\":\"KafkaMon-Broker Server (Memory Usage)\",\"type\":\"line\"}", 148 | "uiStateJSON": "{}", 149 | "description": "", 150 | "version": 1, 151 | "kibanaSavedObjectMeta": { 152 | "searchSourceJSON": "{\"index\":\"kafka_mon\",\"query\":{\"query_string\":{\"query\":\"\\\"OperatingSystem.FreePhysicalMemorySize\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" 153 | } 154 | } 155 | }, 156 | { 157 | "_id": "41dc5590-5bd5-11e7-be67-49b67c124db6", 158 | "_type": "visualization", 159 | "_source": { 160 | "title": "KafkaMon-Broker(CPU Load)", 161 | "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"%(Load)\",\"field\":\"metric_value_number\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"1s\",\"customLabel\":\"Broker CPU Load( System vs Process)\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"custom\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"field\":\"metric_path.keyword\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Broker CPU Load( System vs Process)\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"%(Load)\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"%(Load)\"},\"type\":\"value\"}]},\"title\":\"KafkaMon-Broker(CPU Load)\",\"type\":\"line\"}", 162 | "uiStateJSON": "{\"vis\":{\"legendOpen\":true},\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}}", 163 | "description": "", 164 | "version": 1, 165 | "kibanaSavedObjectMeta": { 166 | "searchSourceJSON": "{\"index\":\"kafka_mon\",\"query\":{\"query_string\":{\"query\":\"\\\"OperatingSystem.SystemCpuLoad\\\" or \\\"OperatingSystem.ProcessCpuLoad\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" 167 | } 168 | } 169 | }, 170 | { 171 | "_id": "2fe80850-5a3d-11e7-a06b-bfa32df73ed4", 172 | "_type": "visualization", 173 | "_source": { 174 | "title": "KafkaMon-Producer(outgoing-byte)", 175 | "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Bytes\",\"field\":\"metric_value_number\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"2h\",\"customLabel\":\"Producer.BytesRate.outgoing-byte-rate\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"s\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"Producer.BytesRate.outgoing-byte-rate\",\"field\":\"metric_path.keyword\",\"order\":\"desc\",\"orderBy\":\"_term\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"rotate\":90,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Producer.BytesRate.outgoing-byte-rate\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Bytes\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Bytes\"},\"type\":\"value\"}]},\"title\":\"KafkaMon-Producer(outgoing-byte)\",\"type\":\"line\"}", 176 | "uiStateJSON": "{\"vis\":{\"colors\":{\"kafkabroker1.FetchConsumer.Mean\":\"#3F6833\",\"kafkabroker1.Procude.Mean\":\"#E5AC0E\"},\"legendOpen\":false}}", 177 | "description": "", 178 | "version": 1, 179 | "kibanaSavedObjectMeta": { 180 | "searchSourceJSON": "{\"index\":\"kafka_mon\",\"query\":{\"query_string\":{\"query\":\"\\\"kafka-producer.Producer.BytesRate.outgoing-byte-rate\\\"\",\"analyze_wildcard\":true}},\"filter\":[]}" 181 | } 182 | } 183 | } 184 | ] -------------------------------------------------------------------------------- /01.Monitoring_with_ELK/kibana/kibana_export_everything_new.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_id": "57d87d40-5a3d-11e7-a06b-bfa32df73ed4", 4 | "_type": "dashboard", 5 | "_source": { 6 | "title": "KafkaMon-Dash(throughput)", 7 | "hits": 0, 8 | "description": "", 9 | "panelsJSON": "[{\"col\":1,\"id\":\"09564bf0-5a30-11e7-a06b-bfa32df73ed4\",\"panelIndex\":2,\"row\":1,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"f1e4eb80-5a29-11e7-a406-63b0e3ccc267\",\"panelIndex\":3,\"row\":17,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"1beb2240-5a44-11e7-a06b-bfa32df73ed4\",\"panelIndex\":5,\"row\":7,\"size_x\":12,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"1c9d7880-5ad7-11e7-a406-63b0e3ccc267\",\"panelIndex\":6,\"row\":4,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"e4a325a0-5b09-11e7-afa4-79fb9d9351b8\",\"panelIndex\":7,\"row\":11,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"68c1fc30-5b0f-11e7-afa4-79fb9d9351b8\",\"panelIndex\":8,\"row\":14,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"}]", 10 | "optionsJSON": "{\"darkTheme\":false}", 11 | "uiStateJSON": "{\"P-2\":{\"spy\":{\"mode\":{\"fill\":false,\"name\":null}},\"vis\":{\"colors\":{\"kafka-consumer.Consumer.RecordsConsumed.records-consumed-rate\":\"#629E51\",\"kafkabroker1.FetchConsumer.Mean\":\"#3F6833\",\"kafkabroker1.Procude.Mean\":\"#E5AC0E\"},\"legendOpen\":true}},\"P-3\":{\"vis\":{\"legendOpen\":false}},\"P-5\":{\"vis\":{\"legendOpen\":true}},\"P-6\":{\"vis\":{\"colors\":{\"kafkabroker1.BrokerTopicMetrics.BytesInPerSec.OneMinuteRate\":\"#E24D42\"},\"legendOpen\":true}},\"P-7\":{\"spy\":{\"mode\":{\"fill\":false,\"name\":null}}}}", 12 | "version": 1, 13 | "timeRestore": false, 14 | "kibanaSavedObjectMeta": { 15 | "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" 16 | } 17 | } 18 | }, 19 | { 20 | "_id": "58f3bde0-5bd5-11e7-be67-49b67c124db6", 21 | "_type": "dashboard", 22 | "_source": { 23 | "title": "KafkaMon-Dash(Resource)", 24 | "hits": 0, 25 | "description": "", 26 | "panelsJSON": "[{\"col\":1,\"id\":\"41dc5590-5bd5-11e7-be67-49b67c124db6\",\"panelIndex\":1,\"row\":10,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"0c96b8b0-5bd7-11e7-be67-49b67c124db6\",\"panelIndex\":2,\"row\":7,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"f7849b30-5c63-11e7-844c-bbc872ee5c30\",\"panelIndex\":3,\"row\":13,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"065b3d90-5c63-11e7-8b85-6d355270351a\",\"panelIndex\":4,\"row\":4,\"size_x\":12,\"size_y\":3,\"type\":\"visualization\"},{\"size_x\":12,\"size_y\":3,\"panelIndex\":5,\"type\":\"visualization\",\"id\":\"7399a5c0-6057-11e7-87bb-1dcc594557a4\",\"col\":1,\"row\":1}]", 27 | "optionsJSON": "{\"darkTheme\":false}", 28 | "uiStateJSON": "{}", 29 | "version": 1, 30 | "timeRestore": false, 31 | "kibanaSavedObjectMeta": { 32 | "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}],\"highlightAll\":true,\"version\":true}" 33 | } 34 | } 35 | }, 36 | { 37 | "_id": "57bdfee0-124a-11e9-a01c-dde110f73204", 38 | "_type": "index-pattern", 39 | "_source": { 40 | "title": "kafka_mon", 41 | "timeFieldName": "@timestamp", 42 | "fields": "[{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"@version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"@version.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"name\":\"cpu_load\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"host.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"metric_path\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"metric_path.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"metric_value_number\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"path\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"path.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"physical_memory\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"name\":\"type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"name\":\"type.keyword\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]" 43 | }, 44 | "_migrationVersion": { 45 | "index-pattern": "6.5.0" 46 | } 47 | }, 48 | { 49 | "_id": "1beb2240-5a44-11e7-a06b-bfa32df73ed4", 50 | "_type": "visualization", 51 | "_source": { 52 | "title": "KafkaMon-Producer(out-going bytes) vs Consumer(in-coming bytes) per sec", 53 | "visState": "{\"title\":\"KafkaMon-Producer(out-going bytes) vs Consumer(in-coming bytes) per sec\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"rotate\":0,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Request rate vs Response rate\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"metric_value_number\",\"customLabel\":\"Count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"custom\",\"customInterval\":\"1s\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Request rate vs Response rate\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"metric_path.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\",\"customLabel\":\"Producer.BytesRate.outgoing-byte-rate\"}}],\"listeners\":{}}", 54 | "uiStateJSON": "{\"vis\":{\"colors\":{\"kafkabroker1.FetchConsumer.Mean\":\"#3F6833\",\"kafkabroker1.Procude.Mean\":\"#E5AC0E\"},\"legendOpen\":true}}", 55 | "description": "", 56 | "version": 1, 57 | "kibanaSavedObjectMeta": { 58 | "searchSourceJSON": "{\"index\":\"57bdfee0-124a-11e9-a01c-dde110f73204\",\"query\":{\"query\":{\"query_string\":{\"query\":\"\\\"kafka-producer.Producer.BytesRate.outgoing-byte-rate\\\" or \\\"kafka-consumer.Consumer.InBytesRate.incoming-byte-rate\\\"\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" 59 | } 60 | } 61 | }, 62 | { 63 | "_id": "68c1fc30-5b0f-11e7-afa4-79fb9d9351b8", 64 | "_type": "visualization", 65 | "_source": { 66 | "title": "KafkaMon-Producer(request vs response rate)", 67 | "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Count\",\"field\":\"metric_value_number\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"2h\",\"customLabel\":\"Consumer MaxLag\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"s\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"field\":\"metric_path.keyword\",\"order\":\"desc\",\"orderBy\":\"_term\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Consumer MaxLag\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":false,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"title\":\"KafkaMon-Producer(request vs response rate)\",\"type\":\"line\"}", 68 | "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", 69 | "description": "", 70 | "version": 1, 71 | "kibanaSavedObjectMeta": { 72 | "searchSourceJSON": "{\"index\":\"57bdfee0-124a-11e9-a01c-dde110f73204\",\"query\":{\"query\":{\"query_string\":{\"query\":\"\\\"kafka-producer.ResponseRate.response-rate\\\" or \\\"kafka-producer.RequestRate.request-rate\\\"\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" 73 | } 74 | } 75 | }, 76 | { 77 | "_id": "f1e4eb80-5a29-11e7-a406-63b0e3ccc267", 78 | "_type": "visualization", 79 | "_source": { 80 | "title": "KafkaMon-Broker(FecthConsumer)", 81 | "visState": "{\"title\":\"KafkaMon-Broker(FecthConsumer)\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Broker.FetchConsumer\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"TotalTimeMs\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":false,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"TotalTimeMs\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"metric_value_number\",\"customLabel\":\"TotalTimeMs\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"s\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Broker.FetchConsumer\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"metric_path.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"_term\"}}],\"listeners\":{}}", 82 | "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", 83 | "description": "", 84 | "version": 1, 85 | "kibanaSavedObjectMeta": { 86 | "searchSourceJSON": "{\"index\":\"57bdfee0-124a-11e9-a01c-dde110f73204\",\"query\":{\"query\":{\"query_string\":{\"query\":\"\\\"kafkabroker1.FetchConsumer.Mean\\\"\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" 87 | } 88 | } 89 | }, 90 | { 91 | "_id": "e4a325a0-5b09-11e7-afa4-79fb9d9351b8", 92 | "_type": "visualization", 93 | "_source": { 94 | "title": "KafkaMon-Consumer(MaxLag)", 95 | "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Count\",\"field\":\"metric_value_number\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"1s\",\"customLabel\":\"Consumer MaxLag\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"custom\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"field\":\"metric_path.keyword\",\"order\":\"desc\",\"orderBy\":\"_term\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Consumer MaxLag\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":false,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"title\":\"KafkaMon-Consumer(MaxLag)\",\"type\":\"line\"}", 96 | "uiStateJSON": "{\"vis\":{\"legendOpen\":true}}", 97 | "description": "", 98 | "version": 1, 99 | "kibanaSavedObjectMeta": { 100 | "searchSourceJSON": "{\"index\":\"57bdfee0-124a-11e9-a01c-dde110f73204\",\"query\":{\"query\":{\"query_string\":{\"query\":\"\\\"kafka-consumer-lag.Consumer.RecordsLagMax.records-lag-max\\\"\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" 101 | } 102 | } 103 | }, 104 | { 105 | "_id": "1c9d7880-5ad7-11e7-a406-63b0e3ccc267", 106 | "_type": "visualization", 107 | "_source": { 108 | "title": "KafkaMon-Broker(InOut Bytes)", 109 | "visState": "{\"title\":\"KafkaMon-Broker(InOut Bytes)\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Broker in vs out Bytes\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Bytes\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Bytes\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"metric_value_number\",\"customLabel\":\"Bytes\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"custom\",\"customInterval\":\"1s\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Broker in vs out Bytes\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"metric_path.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", 110 | "uiStateJSON": "{}", 111 | "description": "", 112 | "version": 1, 113 | "kibanaSavedObjectMeta": { 114 | "searchSourceJSON": "{\"index\":\"57bdfee0-124a-11e9-a01c-dde110f73204\",\"query\":{\"query\":{\"query_string\":{\"query\":\"\\\"BrokerTopicMetrics.BytesInPerSec.OneMinuteRate\\\" or \\\"BrokerTopicMetrics.BytesOutPerSec.OneMinuteRate\\\"\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" 115 | } 116 | } 117 | }, 118 | { 119 | "_id": "09564bf0-5a30-11e7-a06b-bfa32df73ed4", 120 | "_type": "visualization", 121 | "_source": { 122 | "title": "KafkaMon-Broker vs Consumer(Message/sec)", 123 | "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Messages\",\"field\":\"metric_value_number\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"1s\",\"customLabel\":\"Broker & Consumer (Messages/Sec)\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"custom\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"field\":\"metric_path.keyword\",\"order\":\"desc\",\"orderBy\":\"_term\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"rotate\":0,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Broker & Consumer (Messages/Sec)\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Messages\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Messages\"},\"type\":\"value\"}]},\"title\":\"KafkaMon-Broker vs Consumer(Message/sec)\",\"type\":\"line\"}", 124 | "uiStateJSON": "{\"vis\":{\"colors\":{\"kafkabroker1.FetchConsumer.Mean\":\"#3F6833\",\"kafkabroker1.Procude.Mean\":\"#E5AC0E\"},\"legendOpen\":true}}", 125 | "description": "", 126 | "version": 1, 127 | "kibanaSavedObjectMeta": { 128 | "searchSourceJSON": "{\"index\":\"57bdfee0-124a-11e9-a01c-dde110f73204\",\"query\":{\"query\":{\"query_string\":{\"query\":\"\\\"kafkabroker1.BrokerTopicMetrics.MessagesInPerSec.OneMinuteRate\\\" or \\\"kafkabroker2.BrokerTopicMetrics.MessagesInPerSec.OneMinuteRate\\\", \\\"kafkabroker3.BrokerTopicMetrics.MessagesInPerSec.OneMinuteRate\\\", \\\"kafka-consumer.Consumer.RecordsConsumed.records-consumed-rate\\\"\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" 129 | } 130 | } 131 | }, 132 | { 133 | "_id": "b812adb0-5a43-11e7-a406-63b0e3ccc267", 134 | "_type": "visualization", 135 | "_source": { 136 | "title": "KafkaMon-Consumer(incoming-byte)", 137 | "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Bytes\",\"field\":\"metric_value_number\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"2h\",\"customLabel\":\"Consumer.outgoing-byte-rate/Sec\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"s\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"Producer.BytesRate.outgoing-byte-rate\",\"field\":\"metric_path.keyword\",\"order\":\"desc\",\"orderBy\":\"_term\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"rotate\":90,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Consumer.incoming-byte/Sec\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Bytes\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Bytes\"},\"type\":\"value\"}]},\"title\":\"KafkaMon-Consumer(incoming-byte)\",\"type\":\"line\"}", 138 | "uiStateJSON": "{\"vis\":{\"colors\":{\"kafkabroker1.FetchConsumer.Mean\":\"#3F6833\",\"kafkabroker1.Procude.Mean\":\"#E5AC0E\"},\"legendOpen\":false}}", 139 | "description": "", 140 | "version": 1, 141 | "kibanaSavedObjectMeta": { 142 | "searchSourceJSON": "{\"index\":\"57bdfee0-124a-11e9-a01c-dde110f73204\",\"query\":{\"query\":{\"query_string\":{\"query\":\"\\\"kafka-consumer.Consumer.InBytesRate.incoming-byte-rate\\\"\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" 143 | } 144 | } 145 | }, 146 | { 147 | "_id": "7399a5c0-6057-11e7-87bb-1dcc594557a4", 148 | "_type": "visualization", 149 | "_source": { 150 | "title": "KafkaMon-Broker(Process cpu load)", 151 | "visState": "{\"title\":\"KafkaMon-Broker(Process cpu load)\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Broker CPU Load( Process)\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"%(Load)\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"%(Load)\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"metric_value_number\",\"customLabel\":\"%(Load)\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"custom\",\"customInterval\":\"1s\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Broker CPU Load( Process)\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"metric_path.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", 152 | "uiStateJSON": "{\"vis\":{\"legendOpen\":true},\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}}", 153 | "description": "", 154 | "version": 1, 155 | "kibanaSavedObjectMeta": { 156 | "searchSourceJSON": "{\"index\":\"57bdfee0-124a-11e9-a01c-dde110f73204\",\"query\":{\"query\":{\"query_string\":{\"query\":\"\\\"OperatingSystem.ProcessCpuLoad\\\"\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" 157 | } 158 | } 159 | }, 160 | { 161 | "_id": "f7849b30-5c63-11e7-844c-bbc872ee5c30", 162 | "_type": "visualization", 163 | "_source": { 164 | "title": "KafkaMon-Broker_Server(OpenFileDescriptor Count)", 165 | "visState": "{\"title\":\"KafkaMon-Broker_Server(OpenFileDescriptor Count)\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Broker Server (Open File Descriptor Count)\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"schema\":\"metric\",\"params\":{\"field\":\"metric_value_number\",\"customLabel\":\"Count\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"custom\",\"customInterval\":\"1s\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Broker Server (Open File Descriptor Count)\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"metric_path.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", 166 | "uiStateJSON": "{}", 167 | "description": "", 168 | "version": 1, 169 | "kibanaSavedObjectMeta": { 170 | "searchSourceJSON": "{\"index\":\"57bdfee0-124a-11e9-a01c-dde110f73204\",\"query\":{\"query\":{\"query_string\":{\"query\":\"\\\"OperatingSystem.OpenFileDescriptorCount\\\"\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" 171 | } 172 | } 173 | }, 174 | { 175 | "_id": "065b3d90-5c63-11e7-8b85-6d355270351a", 176 | "_type": "visualization", 177 | "_source": { 178 | "title": "KafkaMon-Broker (Memory Heap vs NonHeap)", 179 | "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Used (MB)\",\"field\":\"metric_value_number\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"1s\",\"customLabel\":\"Broker Memory (Heap vs NonHeap)\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"custom\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"field\":\"metric_path.keyword\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Broker Memory (Heap vs NonHeap)\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Used (MB)\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Used (MB)\"},\"type\":\"value\"}]},\"title\":\"KafkaMon-Broker (Memory Heap vs NonHeap)\",\"type\":\"line\"}", 180 | "uiStateJSON": "{}", 181 | "description": "", 182 | "version": 1, 183 | "kibanaSavedObjectMeta": { 184 | "searchSourceJSON": "{\"index\":\"57bdfee0-124a-11e9-a01c-dde110f73204\",\"query\":{\"query\":{\"query_string\":{\"query\":\"\\\"Memory.HeapMemoryUsage.used\\\" or \\\"Memory.NonHeapMemoryUsage.used\\\"\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" 185 | } 186 | } 187 | }, 188 | { 189 | "_id": "41dc5590-5bd5-11e7-be67-49b67c124db6", 190 | "_type": "visualization", 191 | "_source": { 192 | "title": "KafkaMon-Broker(CPU Load)", 193 | "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"%(Load)\",\"field\":\"metric_value_number\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"1s\",\"customLabel\":\"Broker CPU Load( System vs Process)\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"custom\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"field\":\"metric_path.keyword\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Broker CPU Load( System vs Process)\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"%(Load)\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"%(Load)\"},\"type\":\"value\"}]},\"title\":\"KafkaMon-Broker(CPU Load)\",\"type\":\"line\"}", 194 | "uiStateJSON": "{\"vis\":{\"legendOpen\":true},\"spy\":{\"mode\":{\"name\":null,\"fill\":false}}}", 195 | "description": "", 196 | "version": 1, 197 | "kibanaSavedObjectMeta": { 198 | "searchSourceJSON": "{\"index\":\"57bdfee0-124a-11e9-a01c-dde110f73204\",\"query\":{\"query\":{\"query_string\":{\"query\":\"\\\"OperatingSystem.SystemCpuLoad\\\" or \\\"OperatingSystem.ProcessCpuLoad\\\"\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" 199 | } 200 | } 201 | }, 202 | { 203 | "_id": "0c96b8b0-5bd7-11e7-be67-49b67c124db6", 204 | "_type": "visualization", 205 | "_source": { 206 | "title": "KafkaMon-Broker Server (Memory Usage)", 207 | "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Usage (MB)\",\"field\":\"metric_value_number\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"1s\",\"customLabel\":\"Broker Server Memory Usage(Total vs Free)\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"custom\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"field\":\"metric_path.keyword\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Broker Server Memory Usage(Total vs Free)\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":true,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"bottom\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Usage (MB)\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Usage (MB)\"},\"type\":\"value\"}]},\"title\":\"KafkaMon-Broker Server (Memory Usage)\",\"type\":\"line\"}", 208 | "uiStateJSON": "{}", 209 | "description": "", 210 | "version": 1, 211 | "kibanaSavedObjectMeta": { 212 | "searchSourceJSON": "{\"index\":\"57bdfee0-124a-11e9-a01c-dde110f73204\",\"query\":{\"query\":{\"query_string\":{\"query\":\"\\\"OperatingSystem.FreePhysicalMemorySize\\\"\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" 213 | } 214 | } 215 | }, 216 | { 217 | "_id": "2fe80850-5a3d-11e7-a06b-bfa32df73ed4", 218 | "_type": "visualization", 219 | "_source": { 220 | "title": "KafkaMon-Producer(outgoing-byte)", 221 | "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{\"customLabel\":\"Bytes\",\"field\":\"metric_value_number\"},\"schema\":\"metric\",\"type\":\"avg\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customInterval\":\"2h\",\"customLabel\":\"Producer.BytesRate.outgoing-byte-rate\",\"extended_bounds\":{},\"field\":\"@timestamp\",\"interval\":\"s\",\"min_doc_count\":1},\"schema\":\"segment\",\"type\":\"date_histogram\"},{\"enabled\":true,\"id\":\"3\",\"params\":{\"customLabel\":\"Producer.BytesRate.outgoing-byte-rate\",\"field\":\"metric_path.keyword\",\"order\":\"desc\",\"orderBy\":\"_term\",\"size\":5},\"schema\":\"group\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"rotate\":90,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Producer.BytesRate.outgoing-byte-rate\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Bytes\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"normal\",\"show\":true,\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":true,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"defaultYExtents\":true,\"mode\":\"normal\",\"setYExtents\":false,\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Bytes\"},\"type\":\"value\"}]},\"title\":\"KafkaMon-Producer(outgoing-byte)\",\"type\":\"line\"}", 222 | "uiStateJSON": "{\"vis\":{\"colors\":{\"kafkabroker1.FetchConsumer.Mean\":\"#3F6833\",\"kafkabroker1.Procude.Mean\":\"#E5AC0E\"},\"legendOpen\":false}}", 223 | "description": "", 224 | "version": 1, 225 | "kibanaSavedObjectMeta": { 226 | "searchSourceJSON": "{\"index\":\"57bdfee0-124a-11e9-a01c-dde110f73204\",\"query\":{\"query\":{\"query_string\":{\"query\":\"\\\"kafka-producer.Producer.BytesRate.outgoing-byte-rate\\\"\",\"analyze_wildcard\":true}},\"language\":\"lucene\"},\"filter\":[]}" 227 | } 228 | } 229 | } 230 | ] --------------------------------------------------------------------------------