├── LICENSE ├── README.md ├── prepare.elasticsearch.sh ├── zabbix_to_elasticsearch.sql ├── zabbix_to_elasticsearch_step1.sh └── zabbix_to_elasticsearch_step2.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Ilya Ableev 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Zabbix-Events-to-ElasticSearch 2 | Prototype for exporting Zabbix events to ElasticSearch storage 3 | -------------------------------------------------------------------------------- /prepare.elasticsearch.sh: -------------------------------------------------------------------------------- 1 | URL="elasticsearch.local:9200" 2 | 3 | curl -XPUT "${URL}/zabbix_events?pretty" -H 'Content-Type: application/json' -d' 4 | { 5 | "mappings": { 6 | "event": { 7 | "properties": { 8 | "zabbix_server": {"type": "text"}, 9 | "hostname": { "type": "text"}, 10 | "trigger": {"type": "text"}, 11 | "triggerid": {"type": "long"}, 12 | "ts": { "type": "date", "format": "epoch_second"}, 13 | "severity": { "type": "text"}, 14 | "nseverity": {"type": "integer"}, 15 | "eventid": {"type": "long"}, 16 | "status": {"type": "text"} 17 | } 18 | } 19 | } 20 | }' 21 | 22 | curl -XPUT "${URL}/zabbix_history?pretty" -H 'Content-Type: application/json' -d' 23 | { 24 | "mappings": { 25 | "event": { 26 | "properties": { 27 | "zabbix_server": {"type": "text"}, 28 | "hostname": { "type": "text"}, 29 | "trigger": {"type": "text"}, 30 | "triggerid": {"type": "long"}, 31 | "eventid": {"type": "long"}, 32 | "eventrecoveryid": {"type": "long"}, 33 | "duration": {"type": "integer"}, 34 | "started": { "type": "date", "format": "epoch_second"}, 35 | "finished": {"type": "date", "format": "epoch_second"}, 36 | "severity": { "type": "text"}, 37 | "nseverity": {"type": "integer"} 38 | } 39 | } 40 | } 41 | }' -------------------------------------------------------------------------------- /zabbix_to_elasticsearch.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE `zabbix_to_elasticsearch` ( 2 | `id` int(11) NOT NULL AUTO_INCREMENT, 3 | `eventid` bigint(12) NOT NULL DEFAULT '0', 4 | `eventrecoveryid` bigint(12) DEFAULT NULL, 5 | `triggerid` bigint(12) NOT NULL DEFAULT '0', 6 | `hostname` varchar(64) COLLATE utf8_unicode_ci NOT NULL DEFAULT '', 7 | `triggerdescription` varchar(255) COLLATE utf8_unicode_ci NOT NULL DEFAULT '', 8 | `started` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', 9 | `finished` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', 10 | `severity` enum('Information','Warning','Average','High','Disaster') COLLATE utf8_unicode_ci DEFAULT NULL, 11 | `nseverity` tinyint(1) NOT NULL DEFAULT '0', 12 | `is_e_done` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'flag shows is eventid processed or not', 13 | `is_er_done` tinyint(1) NOT NULL DEFAULT '0' COMMENT 'flag shows is eventrecoveryid processed or not', 14 | PRIMARY KEY (`id`), 15 | UNIQUE KEY `ts_triggerid_host` (`started`,`triggerid`,`hostname`) 16 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; 17 | -------------------------------------------------------------------------------- /zabbix_to_elasticsearch_step1.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | TO=$1 4 | SUBJECT=$2 5 | BODY=$3 6 | 7 | DB="zabbix_support" 8 | TABLE="zabbix_to_elasticsearch" 9 | 10 | LOG="/var/log/zabbix_to_elasticsearch_step1.sh.log" 11 | 12 | date >> ${LOG} 13 | 14 | HOST=$(echo "${BODY}" | grep -Po '_HOST:\K.*') 15 | TRIGGER=$(echo "${BODY}" | grep -Po '_TRIGGER_NAME:\K.*' | sed -e "s/\"/'/g;s/@e:\[.*//g") 16 | TS_STARTED=$(echo "${BODY}" | grep -Po '_DATE_TIME:\K.*') 17 | TS_FINISHED=$(echo "${BODY}" | grep -Po '_EVENT_RECOVERY_DATE_TIME:\K.*') 18 | STATUS=$(echo "${BODY}" | grep -Po '_TRIGGER_STATUS:\K.*') 19 | SEVERITY=$(echo "${BODY}" | grep -Po '_TRIGGER_SEVERITY:\K.*') 20 | NSEVERITY=$(echo "${BODY}" | grep -Po '_TRIGGER_NSEVERITY:\K.*') 21 | TRIGGERID=$(echo "${BODY}" | grep -Po '_TRIGGER_ID:\K.*') 22 | EVENTID=$(echo "${BODY}" | grep -Po '_EVENT_ID:\K.*') 23 | EVENTRECOVERYID=$(echo "${BODY}" | grep -Po '_EVENT_RECOVERY_ID:\K.*') 24 | 25 | if [ -n "${EVENTRECOVERYID}" ] 26 | then 27 | SQL="UPDATE ${DB}.${TABLE} SET triggerdescription=\"${TRIGGER}\", finished = \"${TS_FINISHED}\", 28 | eventrecoveryid = ${EVENTRECOVERYID}, severity = \"${SEVERITY}\", nseverity = \"${NSEVERITY}\" 29 | WHERE eventid = ${EVENTID}" 30 | else 31 | SQL="INSERT INTO ${DB}.${TABLE} (\`hostname\`,\`triggerdescription\`,\`started\`,\`severity\`, 32 | \`nseverity\`,\`triggerid\`,\`eventid\`) VALUES (\"${HOST}\", \"${TRIGGER}\", \"${TS_STARTED}\", 33 | \"${SEVERITY}\", \"${NSEVERITY}\", ${TRIGGERID}, ${EVENTID});" 34 | fi 35 | 36 | # uncomment this if you want to see queries in the log file 37 | # echo "${SQL}" >>${LOG} 38 | 39 | echo "${SQL}" | mysql 2>>${LOG} 40 | 41 | -------------------------------------------------------------------------------- /zabbix_to_elasticsearch_step2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding: utf-8 3 | 4 | import requests 5 | import json 6 | import sys 7 | import MySQLdb 8 | 9 | zabbix_server = "zabbix1.local" 10 | support_database = "zabbix_support" 11 | support_table = "zabbix_to_elasticsearch" 12 | 13 | 14 | def main(): 15 | 16 | args = sys.argv 17 | 18 | db_connect = MySQLdb.connect(host="localhost", database=support_database) 19 | db_connect.autocommit(True) 20 | db_cursor = db_connect.cursor(MySQLdb.cursors.DictCursor) 21 | 22 | if "events" in args: 23 | 24 | q_get_events = "SELECT *, UNIX_TIMESTAMP(started) AS started_ts FROM {0} " \ 25 | "WHERE is_e_done = 0".format(support_table) 26 | 27 | db_cursor.execute(q_get_events) 28 | events = db_cursor.fetchall() 29 | 30 | for event in events: 31 | data_event = { 32 | "hostname": event["hostname"], 33 | "trigger": event["triggerdescription"], 34 | "severity": event["severity"], 35 | "nseverity": event["nseverity"], 36 | "ts": event["started_ts"], 37 | "triggerid": event["triggerid"], 38 | "eventid": event["eventid"], 39 | "status": "PROBLEM", 40 | "zabbix_server": zabbix_server 41 | } 42 | #print(data_event) 43 | result_event = requests.put("http://monitoring.mlan:9200/zabbix_events/event/" + str(event["eventid"]) + 44 | "?pretty", 45 | data=json.dumps(data_event), headers={'Content-type': 'application/json'}) 46 | print(result_event.content) 47 | if result_event.status_code in (200, 201): 48 | q_mark_as_done = "UPDATE {0} SET is_e_done = 1 WHERE eventid = {1}"\ 49 | .format(support_table, event["eventid"]) 50 | db_cursor.execute(q_mark_as_done) 51 | else: 52 | print(result_event.status_code) 53 | sys.exit(1) 54 | print "---" 55 | 56 | if "history" in args: 57 | 58 | q_get_history = "SELECT *, UNIX_TIMESTAMP(started) AS started_ts, UNIX_TIMESTAMP(finished) AS finished_ts, " \ 59 | "finished-started AS duration " \ 60 | "FROM {0} " \ 61 | "WHERE eventrecoveryid IS NOT NULL " \ 62 | "AND is_er_done = 0".format(support_table) 63 | history = db_cursor.execute(q_get_history) 64 | 65 | for event in history: 66 | data_history = { 67 | "hostname": event["hostname"], 68 | "trigger": event["triggerdescription"], 69 | "severity": event["severity"], 70 | "nseverity": event["nseverity"], 71 | "started": event["started_ts"], 72 | "finished": event["finished_ts"], 73 | "duration": event["duration"], 74 | "triggerid": event["triggerid"], 75 | "eventid": event["eventid"], 76 | "eventrecoveryid": event["eventrecoveryid"], 77 | "zabbix_server": zabbix_server 78 | } 79 | #print(data_history) 80 | result1 = requests.put("http://monitoring.mlan:9200/zabbix_history/event/" + str(event["eventid"]) + "?pretty", 81 | data=json.dumps(data_history), headers={'Content-type': 'application/json'}) 82 | #print(result1.content) 83 | 84 | data_event = { 85 | "hostname": event["hostname"], 86 | "trigger": event["triggerdescription"], 87 | "severity": event["severity"], 88 | "nseverity": event["nseverity"], 89 | "ts": event["finished_ts"], 90 | "triggerid": event["triggerid"], 91 | "eventid": event["eventrecoveryid"], 92 | "status": "OK", 93 | "zabbix_server": zabbix_server 94 | } 95 | #print(data_event) 96 | result2 = requests.put("http://monitoring.mlan:9200/zabbix_events/event/" + str(event["eventrecoveryid"]) + 97 | "?pretty", 98 | data=json.dumps(data_event), headers={'Content-type': 'application/json'}) 99 | print(result2.content) 100 | if result1.status_code in (200, 201) and result2.status_code in (200, 201): 101 | q_mark_as_done = "UPDATE {0} SET is_er_done = 1 WHERE eventid = {1}"\ 102 | .format(support_table, event["eventid"]) 103 | res = db_cursor.execute(q_mark_as_done) 104 | else: 105 | print(result1.status_code, result2.status_code) 106 | sys.exit(1) 107 | print "---" 108 | 109 | q_delete_useless = "DELETE FROM {0} WHERE is_er_done = 1".format(support_table) 110 | res = db_cursor.execute(q_delete_useless) 111 | 112 | 113 | if __name__ == "__main__": 114 | main() 115 | --------------------------------------------------------------------------------