├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── Vagrantfile ├── docker-compose.yaml ├── img ├── dashboard.png ├── image01.png ├── image02.png ├── image03.png ├── image04.png └── image05.png ├── import_zabbix_clickhouse_template.php ├── zbx_clickhouse_agent.example.conf ├── zbx_clickhouse_config.xml ├── zbx_clickhouse_monitor.sh └── zbx_clickhouse_template.xml /.gitignore: -------------------------------------------------------------------------------- 1 | composer.* 2 | vendor/ 3 | .idea/ 4 | .vagrant/ 5 | *.log 6 | *.pcap -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # zabbix-agent + php + composer which allow connect to clickhouse server 2 | FROM composer AS composer 3 | FROM php:7.3-cli-stretch 4 | 5 | ENV DOCKERIZE_VERSION v0.6.1 6 | RUN curl -o dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz -sL https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ 7 | && tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz \ 8 | && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz 9 | 10 | ENV ZBX_AGENT_VERSION=3.4 11 | ENV ZBX_AGENT_DISTRO=stretch 12 | RUN curl -o zabbix-release_${ZBX_AGENT_VERSION}-1+${ZBX_AGENT_DISTRO}_all.deb -sL https://repo.zabbix.com/zabbix/${ZBX_AGENT_VERSION}/debian/pool/main/z/zabbix-release/zabbix-release_${ZBX_AGENT_VERSION}-1+${ZBX_AGENT_DISTRO}_all.deb \ 13 | && apt-get update \ 14 | && apt-get install --no-install-recommends -y apt-transport-https ca-certificates software-properties-common curl unzip git libxml2-utils gnupg2 default-mysql-client inetutils-telnet inetutils-ping iproute2 less tcpdump dirmngr\ 15 | && dpkg -i zabbix-release_${ZBX_AGENT_VERSION}-1+${ZBX_AGENT_DISTRO}_all.deb \ 16 | && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E0C56BD4 \ 17 | && add-apt-repository "deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" \ 18 | && apt-get update \ 19 | && apt-get install --no-install-recommends -y zabbix-agent/stretch \ 20 | && apt-get install --no-install-recommends -y clickhouse-client \ 21 | && rm -rf zabbix-release_${ZBX_AGENT_VERSION}-1+${ZBX_AGENT_DISTRO}_all.deb \ 22 | && apt-get clean && apt-get auto-remove \ 23 | && rm -rf /var/lib/apt/lists/* 24 | 25 | ENV ZBX_SERVER=zabbix 26 | ENV ZBX_API_URL=http://zabbix/api_jsonrpc.php 27 | ENV ZBX_API_USER=Admin 28 | ENV ZBX_API_PASS=zabbix 29 | ENV ZBX_HTTP_AUTH_UZER="" 30 | ENV ZBX_HTTP_AUTH_PASS="" 31 | ENV CLICKHOUSE_SERVER=clickhouse 32 | ENV ZBX_TEMPLATES=/etc/zabbix/templates/zbx_clickhouse_template.xml 33 | 34 | RUN mkdir -p /var/run/zabbix 35 | RUN mkdir -p /etc/zabbix/php-zabbix-api 36 | WORKDIR /etc/zabbix/php-zabbix-api 37 | COPY --from=composer /usr/bin/composer /usr/bin/composer 38 | RUN composer require confirm-it-solutions/php-zabbix-api 39 | COPY import_zabbix_clickhouse_template.php ./ 40 | 41 | RUN echo "Server=${ZBX_SERVER}" > /etc/zabbix/zabbix_agentd.conf 42 | RUN echo "ServerActive=${ZBX_SERVER}" >> /etc/zabbix/zabbix_agentd.conf 43 | RUN echo "Hostname=zabbix-agent" >> /etc/zabbix/zabbix_agentd.conf 44 | RUN echo "DebugLevel=3" >> /etc/zabbix/zabbix_agentd.conf 45 | RUN echo "AllowRoot=1" >> /etc/zabbix/zabbix_agentd.conf 46 | RUN echo "LogFile=/var/log/zabbix_agentd.log" >> /etc/zabbix/zabbix_agentd.conf 47 | RUN echo "Include=/etc/zabbix/zabbix_agentd.d/*.conf" >> /etc/zabbix/zabbix_agentd.conf 48 | 49 | CMD dockerize -timeout 120s -wait tcp://${ZBX_SERVER}:10051 -wait http://${CLICKHOUSE_SERVER}:8123/ping -- bash -xc "php -f /etc/zabbix/php-zabbix-api/import_zabbix_clickhouse_template.php && zabbix_agentd -f -c /etc/zabbix/zabbix_agentd.conf" -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Altinity 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Zabbix template for ClickHouse 2 | Required Zabbix Server version - 3.4+ 3 | For zabbix 5.0+ please use [official Zabbix ClickHouse integration used http_agent](https://www.zabbix.com/integrations/clickhouse) 4 | 5 | How to set up server with Zabbix agent: 6 | * Ensure `xmllint` is installed 7 | - for Debian\Ubuntu `apt-get install -y libxml2-utils` 8 | - for CentOS / Fedora `yum install -y libxml2` 9 | - for OpenSUSE `yum install -y libxml2-tools` 10 | * Ensure `clickhouse-client` is installed and allow access to monitored ClickHouse server 11 | - https://clickhouse.tech/docs/en/getting_started/install/ 12 | * Clone this repo. 13 | * Edit `/etc/zabbix/zabbix_agentd.conf`. Add the following line: 14 | ```bash 15 | UserParameter=ch_params[*],bash /PATH/TO/zbx_clickhouse_monitor.sh "$1" "HOST_WHERE_CH_IS_RUNNING" "ADDITIONAL CLICKHOUSE-CLIENT PARAMS" 16 | ``` 17 | where: 18 | * `/PATH/TO/zbx_clickhouse_monitor.sh` depends on where you've cloned this repo. 19 | * `HOST_WHERE_CH_IS_RUNNING` is optional parameter, in case none specified `localhost` would be used 20 | * `ADDITIONAL CLICKHOUSE-CLIENT PARAMS` is optional string appended to any clickhouse-client call. Useful if authorization or ssl required 21 | 22 | And finally 23 | * Import `zbx_clickhouse_template.xml` in zabbix (**zabbix -> Configuration -> Templates -> Import**). 24 | 25 | ![image01](img/image01.png) 26 | ![image02](img/image02.png) 27 | ![image03](img/image03.png) 28 | ![image04](img/image04.png) 29 | ![image05](img/image05.png) 30 | ![dashboard](img/dashboard.png) 31 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | Vagrant.configure(2) do |config| 4 | config.vm.box = "ubuntu/bionic64" 5 | config.vm.box_check_update = false 6 | 7 | config.hostmanager.enabled = true 8 | config.hostmanager.manage_host = true 9 | config.hostmanager.ignore_private_ip = false 10 | config.hostmanager.include_offline = false 11 | 12 | if Vagrant.has_plugin?("vagrant-vbguest") 13 | config.vbguest.auto_update = false 14 | end 15 | 16 | config.vm.define :clickhouse_zabbix do |clickhouse_zabbix| 17 | clickhouse_zabbix.vm.network "private_network", ip: "172.16.2.2", nic_type: "virtio" 18 | clickhouse_zabbix.vm.host_name = "local-altinity-clickhouse-zabbix" 19 | clickhouse_zabbix.hostmanager.aliases = ["clickhouse-zabbix.local"] 20 | end 21 | 22 | config.vm.provider "virtualbox" do |vb| 23 | vb.gui = false 24 | vb.memory = "2048" 25 | end 26 | 27 | config.vm.provision "shell", inline: <<-SHELL 28 | set -xeuo pipefail 29 | 30 | apt-get update 31 | apt-get install --no-install-recommends -y apt-transport-https ca-certificates software-properties-common curl 32 | apt-get install --no-install-recommends -y htop ethtool mc curl wget jq socat git mysql-client 33 | 34 | # yq 35 | apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 36 | add-apt-repository ppa:rmescandon/yq 37 | apt-get install --no-install-recommends -y yq 38 | 39 | # clickhouse 40 | apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E0C56BD4 41 | add-apt-repository "deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" 42 | apt-get install --no-install-recommends -y clickhouse-client 43 | 44 | # docker 45 | apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8D81803C0EBFCD88 46 | add-apt-repository "deb https://download.docker.com/linux/ubuntu bionic edge" 47 | apt-get install --no-install-recommends -y docker-ce 48 | 49 | # docker compose 50 | apt-get install --no-install-recommends -y python3-pip 51 | python3 -m pip install -U pip 52 | rm -rf /usr/bin/pip3 53 | pip3 install -U setuptools 54 | pip3 install -U docker-compose 55 | cd /vagrant 56 | docker-compose pull 57 | docker-compose build 58 | docker-compose up -d 59 | SHELL 60 | end -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | zabbix: 4 | image: zabbix/zabbix-appliance:ubuntu-3.4-latest 5 | ports: 6 | - 80:80 7 | networks: 8 | main: 9 | aliases: 10 | - zabbix 11 | clickhouse: 12 | image: yandex/clickhouse-server:latest 13 | ports: 14 | - 8123:8123 15 | - 9000:9000 16 | - 33306:33306 17 | volumes: 18 | - clickhouse_data:/var/lib/clickhouse 19 | - ./zbx_clickhouse_config.xml:/etc/clickhouse-server/config.xml 20 | networks: 21 | main: 22 | aliases: 23 | - clickhouse 24 | zabbix-agent: 25 | build: 26 | dockerfile: Dockerfile 27 | context: ./ 28 | volumes: 29 | - clickhouse_data:/var/lib/clickhouse 30 | - ./zbx_clickhouse_config.xml:/etc/clickhouse-server/config.xml 31 | - ./zbx_clickhouse_monitor.sh:/etc/zabbix/scripts/zbx_clickhouse_monitor.sh 32 | - ./zbx_clickhouse_template.xml:/etc/zabbix/templates/zbx_clickhouse_template.xml 33 | - ./zbx_clickhouse_agent.example.conf:/etc/zabbix/zabbix_agentd.d/zbx_clickhouse_agent.conf 34 | - ./import_zabbix_clickhouse_template.php:/etc/zabbix/php-zabbix-api/import_zabbix_clickhouse_template.php 35 | networks: 36 | main: 37 | aliases: 38 | - zabbix-agent 39 | depends_on: 40 | - zabbix 41 | - clickhouse 42 | 43 | networks: 44 | main: 45 | 46 | volumes: 47 | clickhouse_data: -------------------------------------------------------------------------------- /img/dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Altinity/clickhouse-zabbix-template/197455b63195156d3a8adc98b91b090b578acc25/img/dashboard.png -------------------------------------------------------------------------------- /img/image01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Altinity/clickhouse-zabbix-template/197455b63195156d3a8adc98b91b090b578acc25/img/image01.png -------------------------------------------------------------------------------- /img/image02.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Altinity/clickhouse-zabbix-template/197455b63195156d3a8adc98b91b090b578acc25/img/image02.png -------------------------------------------------------------------------------- /img/image03.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Altinity/clickhouse-zabbix-template/197455b63195156d3a8adc98b91b090b578acc25/img/image03.png -------------------------------------------------------------------------------- /img/image04.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Altinity/clickhouse-zabbix-template/197455b63195156d3a8adc98b91b090b578acc25/img/image04.png -------------------------------------------------------------------------------- /img/image05.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Altinity/clickhouse-zabbix-template/197455b63195156d3a8adc98b91b090b578acc25/img/image05.png -------------------------------------------------------------------------------- /import_zabbix_clickhouse_template.php: -------------------------------------------------------------------------------- 1 | configurationImport([ 22 | 'format'=>'xml', 23 | 'rules'=>[ 24 | 'applications'=>[ 25 | 'createMissing'=>true, 26 | ], 27 | 'discoveryRules'=>[ 28 | 'createMissing'=>true, 29 | 'updateExisting'=>true, 30 | ], 31 | 'groups'=>[ 32 | 'createMissing'=>true, 33 | ], 34 | 'graphs'=>[ 35 | 'createMissing'=>true, 36 | 'updateExisting'=>true, 37 | ], 38 | 'items'=>[ 39 | 'createMissing'=>true, 40 | 'updateExisting'=>true, 41 | ], 42 | 'screens'=>[ 43 | 'createMissing'=>true, 44 | 'updateExisting'=>true, 45 | ], 46 | 'templateLinkage'=>[ 47 | 'createMissing'=>true, 48 | ], 49 | 'templates'=>[ 50 | 'createMissing'=>true, 51 | 'updateExisting'=>true, 52 | ], 53 | 'templateScreens'=>[ 54 | 'createMissing'=>true, 55 | 'updateExisting'=>true, 56 | ], 57 | 'triggers'=>[ 58 | 'createMissing'=>true, 59 | 'updateExisting'=>true, 60 | ], 61 | 'valueMaps'=>[ 62 | 'createMissing'=>true, 63 | 'updateExisting'=>true, 64 | ], 65 | ], 66 | 'source'=>file_get_contents($template), 67 | ]); 68 | echo "$template put to $url OK\n"; 69 | } 70 | -------------------------------------------------------------------------------- /zbx_clickhouse_agent.example.conf: -------------------------------------------------------------------------------- 1 | UserParameter=ch_params[*],bash /etc/zabbix/scripts/zbx_clickhouse_monitor.sh "$1" "${CLICKHOUSE_SERVER}" --user default --connect_timeout 5 --receive_timeout 5 2 | -------------------------------------------------------------------------------- /zbx_clickhouse_config.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 9 | 10 | 18 | 19 | 23 | 24 | 25 | 26 | 27 | trace 28 | /var/log/clickhouse-server/clickhouse-server.log 29 | /var/log/clickhouse-server/clickhouse-server.err.log 30 | 1000M 31 | 10 32 | 33 | 34 | 35 | 8123 36 | 9000 37 | 33306 38 | 39 | 43 | 44 | 45 | 46 | 47 | 48 | /etc/clickhouse-server/server.crt 49 | /etc/clickhouse-server/server.key 50 | 51 | /etc/clickhouse-server/dhparam.pem 52 | none 53 | true 54 | true 55 | sslv2,sslv3 56 | true 57 | 58 | 59 | 60 | true 61 | true 62 | sslv2,sslv3 63 | true 64 | 65 | 66 | 67 | RejectCertificateHandler 68 | 69 | 70 | 71 | 72 | 73 | 76 | 77 | 78 | 9009 79 | 80 | 84 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 4096 107 | 3 108 | 109 | 110 | 100 111 | 112 | 114 | 115 | 116 | 121 | 8589934592 122 | 123 | 127 | 5368709120 128 | 129 | 130 | 131 | /var/lib/clickhouse/ 132 | 133 | 134 | /var/lib/clickhouse/tmp/ 135 | 136 | 137 | /var/lib/clickhouse/user_files/ 138 | 139 | 140 | users.xml 141 | 142 | 143 | default 144 | 145 | 146 | 147 | 148 | 149 | default 150 | 151 | 163 | 164 | 165 | 168 | 169 | 170 | 174 | false 175 | 176 | 179 | 180 | 181 | 182 | 183 | 184 | localhost 185 | 9000 186 | 187 | 188 | 189 | 190 | 191 | 192 | localhost 193 | 9000 194 | 195 | 196 | 197 | 198 | localhost 199 | 9000 200 | 201 | 202 | 203 | 204 | 205 | 206 | 127.0.0.1 207 | 9000 208 | 209 | 210 | 211 | 212 | 127.0.0.2 213 | 9000 214 | 215 | 216 | 217 | 218 | 219 | 220 | localhost 221 | 9440 222 | 1 223 | 224 | 225 | 226 | 227 | 228 | 229 | localhost 230 | 9000 231 | 232 | 233 | 234 | 235 | localhost 236 | 1 237 | 238 | 239 | 240 | 241 | 242 | 243 | 247 | 248 | 253 | 254 | 255 | 256 | 261 | 262 | 263 | 264 | 265 | 3600 266 | 267 | 268 | 269 | 3600 270 | 271 | 272 | 60 273 | 274 | 275 | 283 | 310 | 311 | 312 | 319 | 329 | 330 | 331 | 332 | 336 | system 337 | query_log
338 | 346 | toYYYYMM(event_date) 347 | 348 | 7500 349 | 350 | 351 | 353 | 354 | system 355 | trace_log
356 | 357 | toYYYYMM(event_date) 358 | 7500 359 | 360 | 361 | 363 | 364 | system 365 | query_thread_log
366 | toYYYYMM(event_date) 367 | 7500 368 | 369 | 370 | 373 | 374 | system 375 | part_log
376 | 7500 377 | 378 | 379 | 387 | 388 | 391 | 392 | system 393 | metric_log
394 | 7500 395 | 1000 396 | 397 | 398 | 401 | 402 | 403 | 404 | 405 | 406 | 407 | 408 | 409 | 412 | *_dictionary.xml 413 | 414 | 417 | 418 | 430 | 431 | 432 | 434 | 435 | 436 | /clickhouse/task_queue/ddl 437 | 438 | 439 | 440 | 441 | 442 | 443 | 448 | 449 | 456 | 457 | 458 | 459 | 460 | 461 | 462 | click_cost 463 | any 464 | 465 | 0 466 | 3600 467 | 468 | 469 | 86400 470 | 60 471 | 472 | 473 | 474 | max 475 | 476 | 0 477 | 60 478 | 479 | 480 | 3600 481 | 300 482 | 483 | 484 | 86400 485 | 3600 486 | 487 | 488 | 489 | 490 | 493 | /var/lib/clickhouse/format_schemas/ 494 | 495 | 496 | 508 | 509 | 510 | 511 | 512 | -------------------------------------------------------------------------------- /zbx_clickhouse_monitor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Yandex ClickHouse Zabbix template 4 | # 5 | # Copyright (C) Ivinco Ltd 6 | # Copyright (C) Altinity Ltd 7 | 8 | 9 | # Default host where ClickHouse is expected to be available 10 | # You may want to change this for your installation 11 | CH_HOST="${2:-localhost}" 12 | 13 | ## 14 | ## Write usage info 15 | ## 16 | function usage() 17 | { 18 | echo "Usage: $(basename "$0") Command [ClickHouse Host to Connect] [Additional ClickHouse-Client flags]" 19 | } 20 | 21 | # Command to execute 22 | ITEM=$1 23 | if [ -z "$ITEM" ]; then 24 | echo "Provide command to run" 25 | usage 26 | exit 1 27 | fi 28 | 29 | # Collect additional parameters if available. Get last argument if args count > 2. 30 | # IMPORTANT Middle agruments are skipped for simplicity 31 | if [ $# -gt 2 ]; then 32 | ADD_FLAGS="${*:3}" 33 | else 34 | ADD_FLAGS="" 35 | fi 36 | 37 | # Ensure xmllint is available 38 | if ! command -v xmllint >/dev/null 2>&1; then 39 | echo "Looks like xmllint is not available. Please install it." 40 | exit 1 41 | fi 42 | 43 | # Extract ClickHouse data directory 44 | # Usually it is /var/lib/clickhouse 45 | CH_PATH="$(xmllint --xpath 'string(/yandex/path)' /etc/clickhouse-server/config.xml)" 46 | if [ "$?" -ne 0 ]; then 47 | echo "Something went wrong with parsing ClickHouse config. Is xmllint installed? Is ClickHouse config available?" 48 | exit 1 49 | fi 50 | 51 | ## 52 | ## Run ClickHouse monitoring query 53 | ## 54 | function run_ch_query() 55 | { 56 | # Table where to look into 57 | TABLE=$1 58 | # Column where to look into 59 | COLUMN=$2 60 | # Metric name to fetch 61 | METRIC=$3 62 | # System DB to look into 63 | DATABASE="system" 64 | 65 | SQL="SELECT value FROM ${DATABASE}.${TABLE} WHERE $COLUMN = '$METRIC'" 66 | clickhouse-client -h "$CH_HOST" -d "$DATABASE" -q "$SQL" $ADD_FLAGS 67 | } 68 | 69 | ## 70 | ## Fetch metric by name from ClickHouse 71 | ## 72 | function run_ch_metric_command() 73 | { 74 | # $1 - metric name to fetch 75 | run_ch_query 'metrics' 'metric' $1 76 | } 77 | 78 | ## 79 | ## Fetch asynchronous metric by name from ClickHouse 80 | ## 81 | function run_ch_async_metric_command() 82 | { 83 | # $1 - metric name to fetch 84 | run_ch_query 'asynchronous_metrics' 'metric' $1 85 | } 86 | 87 | ## 88 | ## Fetch event by name from ClickHouse 89 | ## 90 | function run_ch_event_command() 91 | { 92 | # $1 - metric name to fetch 93 | run_ch_query 'events' 'event' $1 94 | } 95 | 96 | ## 97 | ## Fetch processes info from ClickHouse 98 | ## 99 | function run_ch_process_command() 100 | { 101 | DATABASE="system" 102 | SQL="SELECT elapsed FROM processes" 103 | clickhouse-client -h "$CH_HOST" -d "$DATABASE" -q "$SQL" $ADD_FLAGS 104 | } 105 | 106 | ## 107 | ## Fetch event by name from ClickHouse 108 | ## 109 | function run_ch_event_command_zeropad() 110 | { 111 | # $1 - metric name to fetch 112 | res=$(run_ch_query 'events' 'event' $1) 113 | [ -n "$res" ] || res=0 114 | echo "$res" 115 | } 116 | 117 | case "$ITEM" in 118 | DiskUsage) 119 | clickhouse client -h "$CH_HOST" -q 'SELECT total_space,free_space FROM system.disks;' $ADD_FLAGS| awk '{printf($1 - $2)}' 120 | ;; 121 | 122 | Revision) 123 | grep Revision "$CH_PATH/status" | awk '{print $2}' 124 | ;; 125 | 126 | LongestRunningQuery) 127 | run_ch_process_command | sort | tail -1 128 | ;; 129 | 130 | DelayedInserts | \ 131 | HTTPConnection | \ 132 | MemoryTracking | \ 133 | MemoryTrackingInBackgroundProcessingPool | \ 134 | MemoryTrackingInBackgroundMoveProcessingPool | \ 135 | MemoryTrackingInBackgroundSchedulePool | \ 136 | MemoryTrackingForMerges | \ 137 | Query | \ 138 | TCPConnection | \ 139 | MySQLConnection | \ 140 | ReadonlyReplica | \ 141 | Merge | \ 142 | DistributedSend | \ 143 | DistributedFilesToInsert | \ 144 | Read | \ 145 | Write | \ 146 | ZooKeeperWatch ) 147 | run_ch_metric_command "$ITEM" 148 | ;; 149 | 150 | MaxPartCountForPartition| \ 151 | ReplicasMaxAbsoluteDelay| \ 152 | ReplicasSumQueueSize | \ 153 | Uptime ) 154 | run_ch_async_metric_command "$ITEM" 155 | ;; 156 | 157 | InsertedBytes | \ 158 | InsertedRows | \ 159 | InsertQuery | \ 160 | MergedRows | \ 161 | MergedUncompressedBytes | \ 162 | ReadCompressedBytes | \ 163 | ReplicaPartialShutdown | \ 164 | ZooKeeperUserExceptions | \ 165 | ZooKeeperHardwareExceptions | \ 166 | ZooKeeperOtherExceptions | \ 167 | ZooKeeperInit | \ 168 | NetworkErrors | \ 169 | DNSError | \ 170 | DistributedConnectionFailAtAll | \ 171 | DistributedConnectionFailTry | \ 172 | SelectQuery ) 173 | run_ch_event_command "$ITEM" 174 | ;; 175 | 176 | SelectedParts) 177 | run_ch_event_command_zeropad "$ITEM" 178 | ;; 179 | 180 | *) 181 | echo "Unknown argument '$ITEM'. Please check command to run" 182 | exit 1 183 | ;; 184 | esac 185 | -------------------------------------------------------------------------------- /zbx_clickhouse_template.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 3.4 4 | 2020-05-20T05:55:39Z 5 | 6 | 7 | Clickhouse servers 8 | 9 | 10 | 11 | 1982 | 1983 | 1984 | 1985 | {Clickhouse:ch_params[Uptime].nodata(3m)} 1986 | 0 1987 | 1988 | {HOST.HOST} clickhouse-server monitoring have no data, possible clickhouse-server is down, check `systemd status clickhouse-server` and check zbx_clickhouse_monitor.sh and `systemd status zabbix-agent` 1989 | 0 1990 | 1991 | 1992 | 0 1993 | 4 1994 | 1995 | 0 1996 | 1 1997 | 1998 | 1999 | 2000 | 2001 | {Clickhouse:ch_params[Uptime].last()} <= 600 2002 | 0 2003 | 2004 | {HOST.HOST} clickhouse-server recently restarted 2005 | 0 2006 | 2007 | 2008 | 0 2009 | 2 2010 | 2011 | 0 2012 | 1 2013 | 2014 | 2015 | 2016 | 2017 | {Clickhouse:ch_params[DNSError].last()}>0 or {Clickhouse:ch_params[NetworkErrors].last()}>0 2018 | 0 2019 | 2020 | {HOST.HOST} clickhouse DNS errors occurred 2021 | 0 2022 | 2023 | 2024 | 0 2025 | 2 2026 | Please check DNS settings and remote_servers part of /etc/clickhouse-server/ 2027 | 2028 | https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-remote-servers 2029 | 2030 | https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-disable-internal-dns-cache 2031 | 2032 | https://clickhouse.tech/docs/en/query_language/system/#query_language-system-drop-dns-cache 2033 | 0 2034 | 1 2035 | 2036 | 2037 | 2038 | 2039 | {Clickhouse:ch_params[Revision].change()}=1 2040 | 0 2041 | 2042 | {HOST.HOST} clickhouse version changed 2043 | 0 2044 | 2045 | 2046 | 0 2047 | 2 2048 | 2049 | 0 2050 | 0 2051 | 2052 | 2053 | 2054 | 2055 | {Clickhouse:ch_params[DistributedConnectionFailAtAll].last()}>=0 2056 | or 2057 | {Clickhouse:ch_params[DistributedConnectionFailTry].last()}>=0 2058 | or 2059 | {Clickhouse:ch_params[DistributedFilesToInsert].last()}>={$MAX_DELAYED_FILES_TO_DISTRIBUTED_INSERT} 2060 | 0 2061 | 2062 | {HOST.HOST} distributed connection exceptions occurred 2063 | 0 2064 | 2065 | https://clickhouse.tech/docs/en/operations/table_engines/distributed/ 2066 | 0 2067 | 3 2068 | please check communications between Clickhouse servers and <remote_servers> in config.xml 2069 | 2070 | 2071 | https://clickhouse.tech/docs/en/operations/table_engines/distributed/ 2072 | 2073 | https://clickhouse.tech/docs/en/sql-reference/statements/system/#query-language-system-distributed 2074 | 2075 | https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-remote-servers 2076 | 2077 | 2078 | When you insert data to distributed table. 2079 | Data is written to target *MergreTree tables asynchronously. When inserted in the table, the data block is just written to the local file system. The data is sent to the remote servers in the background as soon as possible. The period for sending data is managed by the distributed_directory_monitor_sleep_time_ms and distributed_directory_monitor_max_sleep_time_ms settings. The Distributed engine sends each file with inserted data separately, but you can enable batch sending of files with the distributed_directory_monitor_batch_inserts setting. 2080 | 0 2081 | 0 2082 | 2083 | 2084 | 2085 | 2086 | {Clickhouse:ch_params[DelayedInserts].last()} > 0 2087 | 0 2088 | 2089 | {HOST.HOST} have INSERT queries that are throttled due to high number of active data parts for partition in a MergeTree, please decrease INSERT frequency 2090 | 0 2091 | 2092 | https://clickhouse.tech/docs/en/development/architecture/#merge-tree 2093 | 0 2094 | 5 2095 | INSERT queries that are throttled due to high number of active data parts for partition in a MergeTree table. 2096 | 0 2097 | 0 2098 | 2099 | 2100 | 2101 | 2102 | {Clickhouse:ch_params[LongestRunningQuery].last()} >= {$MAX_QUERY_TIME} 2103 | 0 2104 | 2105 | {HOST.HOST} have queries which running more than {$MAX_QUERY_TIME} sec 2106 | 0 2107 | 2108 | 2109 | 0 2110 | 2 2111 | 2112 | 0 2113 | 1 2114 | 2115 | 2116 | 2117 | 2118 | {Clickhouse:ch_params[ReadonlyReplica].last()} > 0 2119 | 0 2120 | 2121 | {HOST.HOST} have read-only replicated tables, check Zookeeper state 2122 | 0 2123 | 2124 | https://clickhouse.tech/docs/en/operations/table_engines/replication/#recovery-after-failures 2125 | 0 2126 | 5 2127 | Number of Replicated tables that are currently in readonly state due to re-initialization after ZooKeeper session loss or due to startup without ZooKeeper configured. 2128 | 0 2129 | 0 2130 | 2131 | 2132 | 2133 | 2134 | {Clickhouse:ch_params[ReplicasMaxAbsoluteDelay].last(3m)} >= {$MAX_REPLICA_DELAY_DISTRIBUTED_QUERIES} 2135 | 0 2136 | 2137 | {HOST.HOST} have replication lag more {$MAX_REPLICA_DELAY_DISTRIBUTED_QUERIES} sec 2138 | 0 2139 | 2140 | https://clickhouse.tech/docs/en/operations/settings/settings/#settings-max_replica_delay_for_distributed_queries 2141 | 0 2142 | 4 2143 | When replica have too much lag, it can be skipped from Distributed SELECT Queries without errors and you will have wrong query results 2144 | 2145 | Check disks and networks on monitored ClickHouse servers 2146 | 0 2147 | 1 2148 | 2149 | 2150 | 2151 | 2152 | {Clickhouse:ch_params[HTTPConnection].last()} >= {$MAX_HTTP_CONNECTIONS} 2153 | 0 2154 | 2155 | {HOST.HOST} HTTP connections >= {$MAX_HTTP_CONNECTIONS} 2156 | 0 2157 | 2158 | https://clickhouse.tech/docs/en/operations/server_settings/settings/#max-concurrent-queries 2159 | 0 2160 | 2 2161 | The clickhouse is adapted to run not a very large number of parallel requests, not every HTTP connection means a running sql request, but a large number of open tcp connections can cause a spike in sudden sql requests, resulting in performance degradation. 2162 | 0 2163 | 0 2164 | 2165 | 2166 | 2167 | 2168 | {Clickhouse:ch_params[MaxPartCountForPartition].last()} >= {$MAX_PARTS_PER_PARTITION} * 0.9 2169 | 0 2170 | 2171 | {HOST.HOST} MergeTree parts 90% of {$MAX_PARTS_PER_PARTITION}, please decrease INSERT queries frequency 2172 | 0 2173 | 2174 | 2175 | 0 2176 | 4 2177 | Clickhouse MergeTree table engine split each INSERT query to partitions (PARTITION BY expression) and add one or more PARTS per INSERT inside each partition, after that background merge process run, and when you have too much unmerged parts inside partition, SELECT queries performance can significate degrade, so clickhouse try delay insert, or abort it 2178 | 0 2179 | 0 2180 | 2181 | 2182 | 2183 | 2184 | {Clickhouse:ch_params[MySQLConnection].last()} >= {$MAX_MYSQL_CONNECTIONS} 2185 | 0 2186 | 2187 | {HOST.HOST} MySQL connections >= {$MAX_MYSQL_CONNECTIONS} 2188 | 0 2189 | 2190 | https://clickhouse.tech/docs/en/operations/server_settings/settings/#max-concurrent-queries 2191 | 0 2192 | 2 2193 | The clickhouse is adapted to run not a very large number of parallel requests, not every MySQL connection means a running sql request, but a large number of open tcp connections can cause a spike in sudden sql requests, resulting in performance degradation. 2194 | 0 2195 | 0 2196 | 2197 | 2198 | 2199 | 2200 | {Clickhouse:ch_params[InsertQuery].last()}>0 2201 | and 2202 | ( {Clickhouse:ch_params[InsertedRows].last()} / {Clickhouse:ch_params[InsertQuery].last()} ) <= {$MIN_INSERTED_ROWS_PER_QUERY} 2203 | 0 2204 | 2205 | {HOST.HOST} please increase inserted rows per INSERT query 2206 | 0 2207 | 2208 | https://clickhouse.tech/docs/en/introduction/performance/#performance-when-inserting-data 2209 | 0 2210 | 4 2211 | Clickhouse team recommends inserting data in packets of at least 1000 rows or no more than a single request per second. 2212 | 2213 | Please use Buffer table 2214 | https://clickhouse.tech/docs/en/operations/table_engines/buffer/ 2215 | or 2216 | https://github.com/nikepan/clickhouse-bulk 2217 | or 2218 | https://github.com/VKCOM/kittenhouse 2219 | 2220 | 0 2221 | 0 2222 | 2223 | 2224 | 2225 | 2226 | {Clickhouse:ch_params[Query].last(3m)} >= 0.9 * {$MAX_CONCURRENT_QUERIES} 2227 | 0 2228 | 2229 | {HOST.HOST} running queries 90% of {$MAX_CONCURRENT_QUERIES} 2230 | 0 2231 | 2232 | https://clickhouse.tech/docs/en/operations/server_settings/settings/#max-concurrent-queries 2233 | 0 2234 | 4 2235 | Each concurrent SELECT query use memory in JOINs use CPU for running aggregation function and can read lot of data from disk when scan parts in partitions and utilize disk IO. 2236 | 2237 | Each concurrent INSERT query, allocate around 1MB per each column in an inserted table and can utilize disk IO. 2238 | 2239 | Look at following documentation parts 2240 | https://clickhouse.tech/docs/en/operations/settings/query_complexity/ 2241 | https://clickhouse.tech/docs/en/operations/quotas/ 2242 | 0 2243 | 1 2244 | 2245 | 2246 | 2247 | 2248 | {Clickhouse:ch_params[TCPConnection].last()} >= {$MAX_TCP_CONNECTIONS} 2249 | 0 2250 | 2251 | {HOST.HOST} TCP connections >= {$MAX_TCP_CONNECTIONS} 2252 | 0 2253 | 2254 | https://clickhouse.tech/docs/en/operations/server_settings/settings/#max-connections 2255 | 0 2256 | 2 2257 | The clickhouse is adapted to run not a very large number of parallel requests, not every tcp connection means a running sql request, but a large number of open tcp connections can cause a spike in sudden sql requests, resulting in performance degradation. 2258 | 0 2259 | 0 2260 | 2261 | 2262 | 2263 | 2264 | 2265 | 2266 | Concurrent running queries 2267 | 600 2268 | 200 2269 | 0.0000 2270 | 100.0000 2271 | 0 2272 | 1 2273 | 0 2274 | 1 2275 | 0 2276 | 0.0000 2277 | 0.0000 2278 | 1 2279 | 0 2280 | 0 2281 | 0 2282 | 2283 | 2284 | 0 2285 | 0 2286 | DDDD00 2287 | 0 2288 | 4 2289 | 0 2290 | 2291 | Clickhouse 2292 | ch_params[Write] 2293 | 2294 | 2295 | 2296 | 1 2297 | 0 2298 | 00BB00 2299 | 0 2300 | 4 2301 | 0 2302 | 2303 | Clickhouse 2304 | ch_params[Read] 2305 | 2306 | 2307 | 2308 | 2 2309 | 0 2310 | BB0000 2311 | 0 2312 | 4 2313 | 0 2314 | 2315 | Clickhouse 2316 | ch_params[Query] 2317 | 2318 | 2319 | 2320 | 3 2321 | 0 2322 | A54F10 2323 | 1 2324 | 2 2325 | 0 2326 | 2327 | Clickhouse 2328 | ch_params[LongestRunningQuery] 2329 | 2330 | 2331 | 2332 | 2333 | 2334 | Connections 2335 | 600 2336 | 200 2337 | 0.0000 2338 | 100.0000 2339 | 1 2340 | 1 2341 | 0 2342 | 1 2343 | 0 2344 | 0.0000 2345 | 0.0000 2346 | 1 2347 | 0 2348 | 0 2349 | 0 2350 | 2351 | 2352 | 0 2353 | 0 2354 | 1A7C11 2355 | 0 2356 | 2 2357 | 0 2358 | 2359 | Clickhouse 2360 | ch_params[TCPConnection] 2361 | 2362 | 2363 | 2364 | 1 2365 | 0 2366 | F63100 2367 | 0 2368 | 2 2369 | 0 2370 | 2371 | Clickhouse 2372 | ch_params[HTTPConnection] 2373 | 2374 | 2375 | 2376 | 2 2377 | 0 2378 | CCCC00 2379 | 0 2380 | 2 2381 | 0 2382 | 2383 | Clickhouse 2384 | ch_params[MySQLConnection] 2385 | 2386 | 2387 | 2388 | 3 2389 | 0 2390 | A54F10 2391 | 0 2392 | 2 2393 | 0 2394 | 2395 | Clickhouse 2396 | ch_params[DistributedSend] 2397 | 2398 | 2399 | 2400 | 2401 | 2402 | Database size 2403 | 600 2404 | 200 2405 | 0.0000 2406 | 100.0000 2407 | 0 2408 | 1 2409 | 0 2410 | 1 2411 | 0 2412 | 0.0000 2413 | 0.0000 2414 | 1 2415 | 0 2416 | 0 2417 | 0 2418 | 2419 | 2420 | 0 2421 | 0 2422 | AA0000 2423 | 0 2424 | 2 2425 | 0 2426 | 2427 | Clickhouse 2428 | ch_params[DiskUsage] 2429 | 2430 | 2431 | 2432 | 2433 | 2434 | Distributed 2435 | 900 2436 | 200 2437 | 0.0000 2438 | 100.0000 2439 | 1 2440 | 1 2441 | 0 2442 | 1 2443 | 0 2444 | 0.0000 2445 | 0.0000 2446 | 1 2447 | 0 2448 | 0 2449 | 0 2450 | 2451 | 2452 | 0 2453 | 0 2454 | CC0000 2455 | 0 2456 | 2 2457 | 0 2458 | 2459 | Clickhouse 2460 | ch_params[DistributedConnectionFailAtAll] 2461 | 2462 | 2463 | 2464 | 1 2465 | 0 2466 | CCCC00 2467 | 0 2468 | 2 2469 | 0 2470 | 2471 | Clickhouse 2472 | ch_params[DistributedConnectionFailTry] 2473 | 2474 | 2475 | 2476 | 2 2477 | 0 2478 | 00BB00 2479 | 1 2480 | 2 2481 | 0 2482 | 2483 | Clickhouse 2484 | ch_params[DistributedFilesToInsert] 2485 | 2486 | 2487 | 2488 | 2489 | 2490 | Finished Queries 2491 | 600 2492 | 200 2493 | 0.0000 2494 | 100.0000 2495 | 0 2496 | 1 2497 | 1 2498 | 1 2499 | 0 2500 | 0.0000 2501 | 0.0000 2502 | 1 2503 | 0 2504 | 0 2505 | 0 2506 | 2507 | 2508 | 0 2509 | 0 2510 | 4CAF50 2511 | 0 2512 | 2 2513 | 0 2514 | 2515 | Clickhouse 2516 | ch_params[SelectQuery] 2517 | 2518 | 2519 | 2520 | 1 2521 | 0 2522 | DDDD00 2523 | 0 2524 | 2 2525 | 0 2526 | 2527 | Clickhouse 2528 | ch_params[InsertQuery] 2529 | 2530 | 2531 | 2532 | 2533 | 2534 | Insert / Merge rows/sec 2535 | 600 2536 | 200 2537 | 0.0000 2538 | 100.0000 2539 | 0 2540 | 1 2541 | 0 2542 | 1 2543 | 0 2544 | 0.0000 2545 | 0.0000 2546 | 1 2547 | 0 2548 | 0 2549 | 0 2550 | 2551 | 2552 | 0 2553 | 0 2554 | DDDD00 2555 | 0 2556 | 2 2557 | 0 2558 | 2559 | Clickhouse 2560 | ch_params[InsertedRows] 2561 | 2562 | 2563 | 2564 | 1 2565 | 0 2566 | CC0000 2567 | 0 2568 | 2 2569 | 0 2570 | 2571 | Clickhouse 2572 | ch_params[MergedRows] 2573 | 2574 | 2575 | 2576 | 2577 | 2578 | Memory Usage 2579 | 600 2580 | 200 2581 | 0.0000 2582 | 100.0000 2583 | 0 2584 | 1 2585 | 0 2586 | 1 2587 | 0 2588 | 0.0000 2589 | 0.0000 2590 | 1 2591 | 0 2592 | 0 2593 | 0 2594 | 2595 | 2596 | 0 2597 | 0 2598 | F63100 2599 | 0 2600 | 2 2601 | 0 2602 | 2603 | Clickhouse 2604 | ch_params[MemoryTracking] 2605 | 2606 | 2607 | 2608 | 1 2609 | 0 2610 | FFFF33 2611 | 0 2612 | 2 2613 | 0 2614 | 2615 | Clickhouse 2616 | ch_params[MemoryTrackingForMerges] 2617 | 2618 | 2619 | 2620 | 2 2621 | 0 2622 | AAAA00 2623 | 0 2624 | 2 2625 | 0 2626 | 2627 | Clickhouse 2628 | ch_params[MemoryTrackingInBackgroundProcessingPool] 2629 | 2630 | 2631 | 2632 | 3 2633 | 0 2634 | 000099 2635 | 0 2636 | 2 2637 | 0 2638 | 2639 | Clickhouse 2640 | ch_params[MemoryTrackingInBackgroundMoveProcessingPool] 2641 | 2642 | 2643 | 2644 | 4 2645 | 0 2646 | 00DDDD 2647 | 0 2648 | 2 2649 | 0 2650 | 2651 | Clickhouse 2652 | ch_params[MemoryTrackingInBackgroundSchedulePool] 2653 | 2654 | 2655 | 2656 | 2657 | 2658 | Replication 2659 | 600 2660 | 200 2661 | 0.0000 2662 | 100.0000 2663 | 1 2664 | 1 2665 | 0 2666 | 1 2667 | 0 2668 | 0.0000 2669 | 0.0000 2670 | 1 2671 | 0 2672 | 0 2673 | 0 2674 | 2675 | 2676 | 0 2677 | 0 2678 | EE0000 2679 | 0 2680 | 2 2681 | 0 2682 | 2683 | Clickhouse 2684 | ch_params[ReadonlyReplica] 2685 | 2686 | 2687 | 2688 | 1 2689 | 0 2690 | DDDD00 2691 | 0 2692 | 2 2693 | 0 2694 | 2695 | Clickhouse 2696 | ch_params[ReplicaPartialShutdown] 2697 | 2698 | 2699 | 2700 | 2 2701 | 0 2702 | 2774A4 2703 | 1 2704 | 2 2705 | 0 2706 | 2707 | Clickhouse 2708 | ch_params[ReplicasMaxAbsoluteDelay] 2709 | 2710 | 2711 | 2712 | 3 2713 | 0 2714 | A54F10 2715 | 0 2716 | 2 2717 | 0 2718 | 2719 | Clickhouse 2720 | ch_params[ReplicasSumQueueSize] 2721 | 2722 | 2723 | 2724 | 2725 | 2726 | Write / Merge bytes/sec 2727 | 600 2728 | 200 2729 | 0.0000 2730 | 100.0000 2731 | 0 2732 | 1 2733 | 0 2734 | 1 2735 | 0 2736 | 0.0000 2737 | 0.0000 2738 | 1 2739 | 0 2740 | 0 2741 | 0 2742 | 2743 | 2744 | 0 2745 | 0 2746 | CCCC00 2747 | 0 2748 | 2 2749 | 0 2750 | 2751 | Clickhouse 2752 | ch_params[InsertedBytes] 2753 | 2754 | 2755 | 2756 | 1 2757 | 0 2758 | BB0000 2759 | 0 2760 | 2 2761 | 0 2762 | 2763 | Clickhouse 2764 | ch_params[MergedUncompressedBytes] 2765 | 2766 | 2767 | 2768 | 2769 | 2770 | Zookeeper 2771 | 600 2772 | 200 2773 | 0.0000 2774 | 100.0000 2775 | 0 2776 | 1 2777 | 0 2778 | 1 2779 | 0 2780 | 0.0000 2781 | 0.0000 2782 | 1 2783 | 0 2784 | 0 2785 | 0 2786 | 2787 | 2788 | 0 2789 | 0 2790 | FF3333 2791 | 0 2792 | 2 2793 | 0 2794 | 2795 | Clickhouse 2796 | ch_params[ZooKeeperHardwareExceptions] 2797 | 2798 | 2799 | 2800 | 1 2801 | 0 2802 | 2774A4 2803 | 1 2804 | 2 2805 | 0 2806 | 2807 | Clickhouse 2808 | ch_params[ZooKeeperWatch] 2809 | 2810 | 2811 | 2812 | 2 2813 | 0 2814 | CC0000 2815 | 0 2816 | 2 2817 | 0 2818 | 2819 | Clickhouse 2820 | ch_params[ZooKeeperOtherExceptions] 2821 | 2822 | 2823 | 2824 | 3 2825 | 0 2826 | CC0000 2827 | 0 2828 | 2 2829 | 0 2830 | 2831 | Clickhouse 2832 | ch_params[ZooKeeperUserExceptions] 2833 | 2834 | 2835 | 2836 | 2837 | 2838 | 2839 | --------------------------------------------------------------------------------