├── tests ├── __init__.py └── test_alerting.py ├── data ├── data_file ├── grafana_alerts.cfg ├── html_version_item.html └── html_version_main.html ├── grafana_alerts ├── package_data.dat ├── __init__.py ├── html_version_item.html ├── launcher.py ├── html_version_main.html ├── alerting.py └── reporting.py ├── setup.cfg ├── MANIFEST.in ├── LICENSE ├── .gitignore ├── README.rst ├── DESCRIPTION.rst └── setup.py /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/data_file: -------------------------------------------------------------------------------- 1 | some data -------------------------------------------------------------------------------- /grafana_alerts/package_data.dat: -------------------------------------------------------------------------------- 1 | some data -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | # This flag says that the code is written to work on both Python 2 and Python 3 | # 3. If at all possible, it is good practice to do this. If you cannot, you 4 | # will need to generate wheels for each Python version that you support. 5 | universal=0 6 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include DESCRIPTION.rst 2 | 3 | # Include the test suite (FIXME: does not work yet) 4 | # recursive-include tests * 5 | 6 | # If using Python 2.6 or less, then have to include package data, even though 7 | # it's already declared in setup.py 8 | include grafana_alerts/*.dat 9 | include grafana_alerts/*.html 10 | 11 | recursive-include data * 12 | -------------------------------------------------------------------------------- /grafana_alerts/__init__.py: -------------------------------------------------------------------------------- 1 | from grafana_alerts.launcher import Launcher 2 | 3 | __author__ = 'Pablo Alcaraz' 4 | __copyright__ = "Copyright 2015, Pablo Alcaraz" 5 | # __credits__ = [""] 6 | __license__ = "Apache Software License V2.0" 7 | 8 | 9 | def main(): 10 | """Entry point for the application script""" 11 | the_launcher = Launcher() 12 | return the_launcher.launch() 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2015 Pablo Alcaraz. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); you 4 | may not use this file except in compliance with the License. You may 5 | obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 | implied. See the License for the specific language governing 13 | permissions and limitations under the License. 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Backup files 2 | *.~ 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | bin/ 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | MANIFEST 26 | 27 | # Installer logs 28 | pip-log.txt 29 | pip-delete-this-directory.txt 30 | 31 | # Unit test / coverage reports 32 | .tox/ 33 | .coverage 34 | .cache 35 | nosetests.xml 36 | coverage.xml 37 | 38 | # Translations 39 | *.mo 40 | 41 | # Sphinx documentation 42 | docs/_build/ 43 | 44 | # intellij files 45 | .idea 46 | grafana_alerts.iml -------------------------------------------------------------------------------- /data/grafana_alerts.cfg: -------------------------------------------------------------------------------- 1 | # 2 | # Configuration file. 3 | # 4 | 5 | # The URL where grafana server is listening. It must finish with the character '/' (default value: http://localhost:3130) 6 | grafana_url = http://localhost:3130 7 | 8 | # Grafana token with viewer access (default value: empty string) 9 | grafana_token = 10 | 11 | # email to use as alert sender (default value: grafana-alert@localhost) 12 | email_from = alert@example.com 13 | 14 | # smtp server to use (default value: localhost) 15 | smtp_server = localhost 16 | 17 | # smtp server host to use (default value: 25) 18 | # if port is not 25, starts a tls session. 19 | smtp_port = 25 20 | 21 | # smtp server username to use if it is needed. Optional. Leave it commented if not used. (default value: no username) 22 | #smtp_username = my_smtp_username 23 | 24 | # smtp server password to use if it is needed. Optional. Leave it commented if not used. (default value: no password) 25 | #smtp_password = my_smtp_password -------------------------------------------------------------------------------- /data/html_version_item.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 32 | 33 |
4 |

'{target}' status is '{diff_event}' for '{title}'

5 |
9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 21 | 24 | 29 | 30 |
beforeeventcurrent
17 | {old_target} status is {old_alertName} for {old_title} 18 |
19 | value of {title}: {old_value} / {old_condition} 20 | 		22 | {diff_event} 23 | 25 | {current_target} status is {current_alertName} for {current_title} 26 |
27 | value of {current_title}: {current_value} / {current_condition} 28 |
31 |
34 | 35 | -------------------------------------------------------------------------------- /grafana_alerts/html_version_item.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 32 | 33 |
4 |

'{target}' status is '{diff_event}' for '{title}'

5 |
9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 21 | 24 | 29 | 30 |
beforeeventcurrent
17 | {old_target} status is {old_alertName} for {old_title} 18 |
19 | value of {title}: {old_value} / {old_condition} 20 | 		22 | {diff_event} 23 | 25 | {current_target} status is {current_alertName} for {current_title} 26 |
27 | value of {current_title}: {current_value} / {current_condition} 28 |
31 |
34 | 35 | -------------------------------------------------------------------------------- /grafana_alerts/launcher.py: -------------------------------------------------------------------------------- 1 | """Grafana Alert launcher. 2 | 3 | """ 4 | from grafana_alerts.alerting import AlertCheckerCoordinator 5 | 6 | __author__ = 'Pablo Alcaraz' 7 | __copyright__ = "Copyright 2015, Pablo Alcaraz" 8 | # __credits__ = [""] 9 | __license__ = "Apache Software License V2.0" 10 | 11 | 12 | class Launcher: 13 | def launch(self): 14 | configuration = Configuration() 15 | alert_checker = AlertCheckerCoordinator(configuration) 16 | alert_checker.check() 17 | 18 | 19 | class Configuration: 20 | """Configuration.""" 21 | 22 | def __init__(self): 23 | # TODO make sure the url finishes with '/' or requests could fail. 24 | self.grafana_url = 'http://localhost:3130/' 25 | self.grafana_token = "" 26 | self.email_from = "grafana-alert@localhost" 27 | self.smtp_server = "localhost" 28 | self.smtp_port = 25 29 | self.smtp_username = None 30 | self.smtp_password = None 31 | self.read_config() 32 | 33 | def read_config(self): 34 | try: 35 | with open("/etc/grafana_alerts/grafana_alerts.cfg", "r") as config_file: 36 | config = config_file.readlines() 37 | for line in config: 38 | l = line.strip() 39 | if len(l) == 0 or l.startswith('#'): 40 | continue 41 | k, v = [x.strip() for x in l.split('=', 1)] 42 | setattr(self, k, v) 43 | except BaseException as e: 44 | raise RuntimeError("Error reading configuration /etc/grafana_alerts/grafana_alerts.cfg", e) 45 | -------------------------------------------------------------------------------- /data/html_version_main.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Alerts report 6 | 7 | 8 | 45 | 46 | 47 | 48 | 50 | 51 |
Alert report  49 |
52 |
53 | 54 | 55 | 56 | 57 |
58 |
59 | 60 | 61 | 63 | 64 |
{html_version_items} 62 |
65 |
66 | 67 | 68 | 69 | 70 |
71 |
72 | 73 | 74 | 83 |
75 |

{companyName}

76 | 77 |

78 | Sent at {date} {time} (reporting server time) / generated by grafana-alerts
80 | message_signature {message_signature} 81 |

82 |
84 |
85 | 86 | -------------------------------------------------------------------------------- /grafana_alerts/html_version_main.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Alerts report 6 | 7 | 8 | 45 | 46 | 47 | 48 | 50 | 51 |
Alert report  49 |
52 |
53 | 54 | 55 | 56 | 57 |
58 |
59 | 60 | 61 | 63 | 64 |
{html_version_items} 62 |
65 |
66 | 67 | 68 | 69 | 70 |
71 |
72 | 73 | 74 | 83 |
75 |

{companyName}

76 | 77 |

78 | Sent at {date} {time} (reporting server time) / generated by grafana-alerts
80 | message_signature {message_signature} 81 |

82 |
84 |
85 | 86 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Grafana Alert Module 2 | ==================== 3 | 4 | grafana-alerts Collects stats from a grafana server based on information available 5 | from the Grafana Dashboards. Then compare those values to an alert table and 6 | throws alert emails if the case is needed. 7 | 8 | Sites 9 | ----- 10 | 11 | Installation package: https://pypi.python.org/pypi/grafana_alerts 12 | 13 | Project Home: https://github.com/pabloa/grafana-alerts 14 | 15 | Issues and bugs: https://github.com/pabloa/grafana-alerts/issues 16 | 17 | 18 | Installation 19 | ------------ 20 | :: 21 | 22 | sudo pip install grafana-alerts 23 | 24 | if you get an error, perhaps it is because the version available is a development 25 | version. In this case try with:: 26 | 27 | sudo pip install --pre grafana-alerts 28 | 29 | 30 | 31 | Configuration 32 | ------------- 33 | 34 | Create a file /etc/grafana_alerts/grafana_alerts.cfg 35 | with:: 36 | 37 | # 38 | # Grafana alerts configuration file. 39 | # 40 | 41 | # The URL where grafana server is listening. It must finish with the character '/' (default value: http://localhost:3130) 42 | grafana_url = http://yourgrafanaserver.com/grafana/ 43 | 44 | # Grafana token with viewer access (default value: empty string) 45 | grafana_token = qwertysDssdsfsfsdfSFsfsfEWrwrwERwrewrwrWeRwRwerWRwERwerWRwerweRwrEWrWErwerWeRwRwrewerr== 46 | 47 | # email to use as alert sender (default value: grafana-alert@localhost) 48 | email_from = alert@example.com 49 | 50 | # smtp server to use (default value: localhost) 51 | smtp_server = localhost 52 | 53 | # smtp server host to use (default value: 25) 54 | # if port is not 25, starts a tls session. 55 | smtp_port = 25 56 | 57 | # smtp server username to use if it is needed. Optional. Leave it commented if not used. (default value: no username) 58 | #smtp_username = my_smtp_username 59 | 60 | # smtp server password to use if it is needed. Optional. Leave it commented if not used. (default value: no password) 61 | #smtp_password = my_smtp_password 62 | 63 | 64 | Add a cron task to execute grafana_alerts for example each 3 minutes::: 65 | 66 | */3 * * * * grafana_alerts 67 | 68 | 69 | Monitoring Dashboards 70 | --------------------- 71 | 72 | Dashboards to be monitored for alerts must be marked with the tag "monitored" 73 | 74 | In each monitored Dashboard, add a text panel with title 'alerts' and a description of your alerts. For example::: 75 | 76 | 50<=x<=100; normal; server@example.com 77 | 78 | 35=0.6.1' 101 | ], 102 | 103 | # List additional groups of dependencies here (e.g. development 104 | # dependencies). You can install these using the following syntax, 105 | # for example: 106 | # $ pip install -e .[dev,test] 107 | extras_require={ 108 | 'dev': ['check-manifest'], 109 | 'test': ['coverage'], 110 | }, 111 | 112 | # If there are data files included in your packages that need to be 113 | # installed, specify them here. If using Python 2.6 or less, then these 114 | # have to be included in MANIFEST.in as well. 115 | # TODO Verify this. 116 | package_data={ 117 | 'grafana_alerts': ['package_data.dat', '*.html'], 118 | }, 119 | 120 | # Although 'package_data' is the preferred approach, in some case you may 121 | # need to place data files outside of your packages. See: 122 | # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa 123 | # In this case, 'data_file' will be installed into '/my_data' 124 | data_files=[ 125 | ('/etc/grafana_alerts', ['data/grafana_alerts.cfg']), 126 | # ('grafana_alerts/data', ['data/html_version_item.html', 'data/html_version_main.html']) 127 | ], 128 | # data_files=[('/etc/grafana_alerts', ['data/*'])], 129 | 130 | # To provide executable scripts, use entry points in preference to the 131 | # "scripts" keyword. Entry points provide cross-platform support and allow 132 | # pip to create the appropriate form of executable for the target platform. 133 | entry_points={ 134 | 'console_scripts': [ 135 | 'grafanaAlerts=grafana_alerts:main', 136 | ], 137 | }, 138 | ) 139 | -------------------------------------------------------------------------------- /grafana_alerts/alerting.py: -------------------------------------------------------------------------------- 1 | """Alerts""" 2 | import urllib2 3 | import json 4 | 5 | import jmespath 6 | 7 | from grafana_alerts.reporting import AlertEvaluationResult, MailAlertReporter 8 | 9 | __author__ = 'Pablo Alcaraz' 10 | __copyright__ = "Copyright 2015, Pablo Alcaraz" 11 | # __credits__ = [""] 12 | __license__ = "Apache Software License V2.0" 13 | 14 | _GRAFANA_URL_PATH_OBTAIN_DASHBOARDS = 'api/search?limit=10&query=&tag=monitored' 15 | _GRAFANA_URL_PATH_DASHBOARD = 'api/dashboards/db/{slug}' 16 | _GRAFANA_URL_PATH_OBTAIN_METRICS = 'api/datasources/proxy/1/render' 17 | 18 | 19 | class NotMonitoreableDashboard(RuntimeError): 20 | def __init__(self, message): 21 | self.message = message 22 | 23 | class AlertCheckerCoordinator: 24 | """Entry point to the alert checking module.""" 25 | 26 | def __init__(self, configuration): 27 | self.configuration = configuration 28 | self.alert_reporter = MailAlertReporter(email_from=self.configuration.email_from, 29 | smtp_server=self.configuration.smtp_server, 30 | smtp_port=self.configuration.smtp_port, 31 | email_username=self.configuration.smtp_username, 32 | email_password=self.configuration.smtp_password) 33 | 34 | def check(self): 35 | """Check if there is something to report""" 36 | # Get all the dashboards to use for checking 37 | scanner = DashboardScanner(self.configuration.grafana_url, self.configuration.grafana_token) 38 | dashboard_data_list = scanner.obtain_dashboards() 39 | print dashboard_data_list 40 | for d in dashboard_data_list: 41 | try: 42 | print "Dashboard: " + d['title'] 43 | # {u'slug': u'typrod-storage', u'tags': [], u'isStarred': False, u'id': 4, u'title': u'TyProd Storage'} 44 | dashboard = Dashboard(self.configuration.grafana_url, self.configuration.grafana_token, d['title'], d['slug'], d['tags']) 45 | alert_checkers = dashboard.obtain_alert_checkers() 46 | print alert_checkers 47 | 48 | # For each set of alert checkers, evaluate them 49 | for alert_checker in alert_checkers: 50 | alert_checker.check() 51 | reported_alerts = alert_checker.calculate_reported_alerts() 52 | # for each set of reported alerts, report whatever is best 53 | self.alert_reporter.report(reported_alerts) 54 | except NotMonitoreableDashboard as e: 55 | print "Dashboard {title} cannot be monitored. Reason: {reason}".format(title=d['title'], reason=e.message) 56 | continue 57 | 58 | 59 | class AlertChecker: 60 | """Command to check metrics.""" 61 | 62 | def __init__(self, grafana_url, grafana_token, title, grafana_targets): 63 | self.grafana_url = grafana_url 64 | self.grafana_token = grafana_token 65 | self.title = title 66 | self.grafana_targets = grafana_targets 67 | self.checkedExecuted = False 68 | self.responses = [] 69 | self.alert_conditions = None 70 | 71 | def set_alert_conditions(self, alert_conditions): 72 | """Alerts conditions are composed by an array of elements. 73 | each element is an array like: 74 | 75 | [["interval1","status1","alert destination1","short description","long description"], 76 | ["interval2","status2","alert destination2","short description","long description"], 77 | ["intervaln","statusN","alert destinationN","short description","long description"]] 78 | 79 | interval: string representing an interval like: 80 | "x<=0": -infinite < x <= 0 81 | "0<=x<50": [0;50) 82 | "50<=x": 50 <= x < infinite 83 | 84 | status: "normal", "warning", "critical" 85 | 86 | alert destination: 1 or more emails separated by "," 87 | 88 | example: 89 | [["50<=x<=100", "normal", "p@q.com"], 90 | ["50 0: 128 | x = float(sum(data)) / len(data) 129 | else: 130 | x = float('nan') 131 | alert_evaluation_result.set_current_value(x) 132 | 133 | # evaluate all the alert conditions and create a current alert status. 134 | for alert_condition in self.alert_conditions: 135 | condition = alert_condition[0] 136 | activated = eval(condition) 137 | alert_evaluation_result.add_alert_condition_result(name=alert_condition[1], condition=condition, 138 | activated=activated, 139 | alert_destination=alert_condition[2], 140 | title=self.title) 141 | 142 | alert_evaluation_result_list.append(alert_evaluation_result) 143 | 144 | return alert_evaluation_result_list 145 | 146 | 147 | class DashboardScanner: 148 | """Provides access to grafana dashboards""" 149 | def __init__(self, grafana_url, grafana_token): 150 | self.grafana_url = grafana_url 151 | self.grafana_token = grafana_token 152 | 153 | def obtain_dashboards(self): 154 | request = urllib2.Request(self.grafana_url + _GRAFANA_URL_PATH_OBTAIN_DASHBOARDS, 155 | headers={"Accept": "application/json", 156 | "Authorization": "Bearer " + self.grafana_token}) 157 | contents = urllib2.urlopen(request).read() 158 | print contents 159 | data = json.loads(contents) 160 | dashboards = jmespath.search('dashboards', data) 161 | return dashboards 162 | 163 | 164 | class Dashboard: 165 | def __init__(self, grafana_url, grafana_token, title, slug, tags): 166 | self.grafana_url = grafana_url 167 | self.grafana_token = grafana_token 168 | self.title = title 169 | self.slug = slug 170 | self.tags = tags 171 | 172 | def obtain_alert_checkers(self): 173 | """check metrics and return a list of triggered alerts.""" 174 | dashboard_info = self._obtain_dashboard_rows() 175 | alert_checkers = self._create_alert_checkers(dashboard_info) 176 | return alert_checkers 177 | 178 | def _obtain_dashboard_rows(self): 179 | """Get a list of dashboard rows.""" 180 | request = urllib2.Request(self.grafana_url + _GRAFANA_URL_PATH_DASHBOARD.format(slug=self.slug), 181 | headers={"Accept": "application/json", 182 | "Authorization": "Bearer " + self.grafana_token}) 183 | contents = urllib2.urlopen(request).read() 184 | # Fix \n inside json values. 185 | contents = contents.replace('\r\n', '\\r\\n').replace('\n', '\\n') 186 | print "Contents is: " 187 | print contents 188 | try: 189 | data = json.loads(contents) 190 | dashboard = jmespath.search('model.rows[*].panels[*]', data) 191 | return dashboard 192 | except ValueError: 193 | raise NotMonitoreableDashboard( 194 | "The definition of dashboard {title} does not look like valid json.".format(title=self.title)) 195 | 196 | def _create_alert_checkers(self, dashboard_info): 197 | """check metrics and return a list of alerts to evaluate. 198 | :return AlertChecker list of all the metrics 199 | """ 200 | alert_checkers = [] 201 | for dashboard_row in dashboard_info: 202 | print dashboard_row 203 | # creates alert checkers for each panel in the row. 204 | # TODO add alert checker creation to a builder dashboard_row2alert_checker_list. 205 | alert_conditions = [] # map of alert conditions( text -> alert parameters) 206 | for panel in dashboard_row: 207 | print panel 208 | print panel['type'] 209 | if panel['type'] == "graph": 210 | # print row['leftYAxisLabel'] 211 | # print row['y_formats'] 212 | alert_checker = AlertChecker(self.grafana_url, self.grafana_token, panel['title'], panel['targets']) 213 | alert_checkers.append(alert_checker) 214 | elif panel['type'] == "singlestat": 215 | alert_checker = AlertChecker(self.grafana_url, self.grafana_token, panel['title'], panel['targets']) 216 | # print row['thresholds'] 217 | alert_checkers.append(alert_checker) 218 | elif panel['type'] == "text": 219 | # print panel['title'] 220 | if panel['title'] == 'alerts': 221 | # read alert parameters to apply to all the alert checkers of this dashboard. 222 | for line in panel['content'].splitlines(): 223 | # TODO replace alert_definition_list for an object 224 | alert_definition_list = [s.strip() for s in line.split(';')] 225 | if len(alert_definition_list) > 1: 226 | alert_conditions.append(alert_definition_list) 227 | else: 228 | print "Unknown type {type}. Ignoring.".format(type=panel['type']) 229 | 230 | if len(alert_conditions) > 0: 231 | # There are alert conditions, add them to all the alert_checkers. 232 | for alert_checker in alert_checkers: 233 | alert_checker.set_alert_conditions(alert_conditions) 234 | return alert_checkers 235 | -------------------------------------------------------------------------------- /grafana_alerts/reporting.py: -------------------------------------------------------------------------------- 1 | from email.mime.multipart import MIMEMultipart 2 | from email.mime.text import MIMEText 3 | import hashlib 4 | import os 5 | import pickle 6 | import smtplib 7 | import datetime 8 | 9 | import pkg_resources 10 | 11 | __author__ = 'Pablo Alcaraz' 12 | __copyright__ = "Copyright 2015, Pablo Alcaraz" 13 | # __credits__ = [""] 14 | __license__ = "Apache Software License V2.0" 15 | 16 | _GRAFANA_URL_PATH_OBTAIN_DASHBOARDS = '/api/search?limit=10&query=&tag=' 17 | _GRAFANA_URL_PATH_DASHBOARD = '/api/dashboards/db/{slug}' 18 | 19 | 20 | class AlertEvaluationResult: 21 | """Alert evaluation information. All the information needed to decide if an alert should be sent and the information 22 | to send should be here.""" 23 | 24 | def __init__(self, title, target): 25 | self.title = title 26 | self.target = target 27 | self.alert_conditions = {} 28 | self.current_alert_condition_status = None 29 | self.value = None 30 | 31 | def set_current_value(self, value): 32 | self.value = value 33 | 34 | def add_alert_condition_result(self, name, condition, activated, alert_destination, title): 35 | # TODO Alert condition result should have a type and others values based on type. Ot they could be a class... 36 | alert_condition_status = { 37 | 'name': name, 38 | 'condition': condition, 39 | 'activated': activated, 40 | 'alert_destination': alert_destination, 41 | 'title': title 42 | } 43 | self.alert_conditions[name] = alert_condition_status 44 | if alert_condition_status['activated']: 45 | # point to the activated alert condition status. 46 | self.current_alert_condition_status = alert_condition_status 47 | 48 | 49 | class BaseAlertReporter: 50 | def report(self, reported_alert): 51 | raise NotImplemented("Implement me") 52 | 53 | 54 | class MailAlertReporter(BaseAlertReporter): 55 | def __init__(self, email_from, smtp_server, smtp_port, email_username=None, email_password=None): 56 | self.email_from = email_from 57 | self.smtp_server = smtp_server 58 | self.smtp_port = smtp_port 59 | self.email_username = email_username 60 | self.email_password = email_password 61 | # TODO remove 62 | self.sent_emails_counter = 0 63 | 64 | def report(self, reported_alerts): 65 | """filter the reported alerts by the configured criteria and if there is some alert to send, 66 | then it reports it""" 67 | filtered_reported_alert = self._filter_current_reported_alerts(reported_alerts=reported_alerts) 68 | diff_report = self._generated_diff_report(filtered_reported_alert) 69 | # TODO keys should have better names. 70 | alerts_to_send_map = self._group_by(diff_report, 'alert_destination') 71 | self._send_alerts_if_any(alerts_to_send_map) 72 | 73 | def _filter_current_reported_alerts(self, reported_alerts): 74 | """Filter current reported alerts""" 75 | # TODO Add filtering capabilites 76 | return reported_alerts 77 | 78 | def _generated_diff_report(self, reported_alerts): 79 | """Compares current alert list with the last one and creates a diff object""" 80 | GRAFANA_MONITOR_STATE = '/tmp/grafana_monitor.state' 81 | # read last state 82 | old_alerts_state = {} 83 | if os.path.isfile(GRAFANA_MONITOR_STATE): 84 | with open(GRAFANA_MONITOR_STATE, 'r') as f: 85 | old_alerts_state = pickle.load(f) 86 | 87 | # collect current state 88 | current_alerts_state = {} 89 | for aer in reported_alerts: 90 | if aer.current_alert_condition_status is None: 91 | # TODO This happens because we are not getting stats from grafana, perhaps server was reainstalled without monitoring tools? 92 | # TODO generate a warning or something according to config definition. 93 | aer_current_alert_condition_status_name = "None" 94 | else: 95 | aer_current_alert_condition_status_name = aer.current_alert_condition_status['name'] 96 | key = "{target}, {title}, {alert_name}".format(target=aer.target, title=aer.title, alert_name=aer_current_alert_condition_status_name) 97 | # value = "{condition},{activated}".format(condition=aer.current_alert_condition_status['condition'], activated=aer.current_alert_condition_status['activated']) 98 | value = aer 99 | current_alerts_state[key] = value 100 | 101 | # persist new state 102 | with open(GRAFANA_MONITOR_STATE, 'w') as f: 103 | pickle.dump(current_alerts_state, f) 104 | 105 | # compare old with new state and generates report with 4 causes of diff: changed, lost, new, unchanged. 106 | diff_report = [] 107 | for current_key, current_value in current_alerts_state.iteritems(): 108 | current_element = current_value 109 | old_element = None 110 | diff = 'new' 111 | if old_alerts_state.has_key(current_key): 112 | old_value = old_alerts_state[current_key] 113 | old_element = old_value 114 | if old_value.current_alert_condition_status is None and current_value.current_alert_condition_status is None: 115 | # value in the alert is 'unchanged' 116 | diff = 'unchanged' 117 | elif old_value.current_alert_condition_status['condition'] != current_value.current_alert_condition_status['condition'] or old_value.current_alert_condition_status['activated'] != current_value.current_alert_condition_status['activated']: 118 | # value in the alert is 'changed' 119 | diff = 'changed' 120 | else: 121 | # value in the alert is 'unchanged' 122 | diff = 'unchanged' 123 | 124 | # TODO Call this DTO AlertEvent 125 | diff_report.append({ 126 | 'diff_event': diff, 127 | 'old': old_element, 128 | 'current': current_element 129 | }) 130 | 131 | current_element = None 132 | diff = 'lost' 133 | for old_key, old_value in old_alerts_state.iteritems(): 134 | old_element = old_value 135 | if not current_alerts_state.has_key(old_key): 136 | diff_report.append({ 137 | 'diff_event': diff, 138 | 'old': old_element, 139 | 'current': current_element 140 | }) 141 | 142 | return diff_report 143 | 144 | 145 | def _group_by(self, diff_report, group_by): 146 | """Given reported alerts evaluation results. Consolidates them by the given key. 147 | For example: 1 mail with a list of alerts groups by destination, etc. 148 | :return a map, the key is the field 'group_by', the value is a list of alert evaluation results. 149 | """ 150 | consolidated_map = {} 151 | 152 | for alert_event in diff_report: 153 | # get the available version of alert_evaluation_result 154 | alert_evaluation_result = alert_event['current'] 155 | if alert_evaluation_result is None: 156 | alert_evaluation_result = alert_event['old'] 157 | 158 | # calculate the group key 159 | if alert_evaluation_result.current_alert_condition_status is not None: 160 | key = alert_evaluation_result.current_alert_condition_status[group_by] 161 | if key is None: 162 | key = eval("alert_evaluation_result." + group_by) 163 | 164 | 165 | # and add the alert_event to the consolidated map 166 | if not consolidated_map.has_key(key): 167 | consolidated_map[key] = [] 168 | consolidated_map[key].append(alert_event) 169 | return consolidated_map 170 | 171 | def _send_alerts_if_any(self, alerts_to_send_map): 172 | """Evaluate if there are news to send. If that is the case, send the alerts. 173 | This version only supports lists grouped by email.""" 174 | for email_to_string, alert_event_list in alerts_to_send_map.iteritems(): 175 | if not self._is_something_to_report(alert_event_list): 176 | continue 177 | html_version_main = self._html_version_main() 178 | html_version_items = self._html_version_items(alert_event_list) 179 | html_version = html_version_main.format(html_version_items=html_version_items, 180 | # TODO Externalize variables 181 | date=datetime.datetime.now().strftime("%B %d, %Y"), 182 | time=datetime.datetime.now().strftime("%I:%M %p"), 183 | message_signature=hashlib.sha256(html_version_main + html_version_items + datetime.datetime.now().strftime("%B %d, %Y - %I:%M %p")).hexdigest(), 184 | companyName="" 185 | ) 186 | # text_version = bs.get_text() 187 | 188 | # create email 189 | email = MIMEMultipart('alternative') 190 | # TODO calculate this value. 191 | email['Subject'] = "monitoring alert" 192 | email['From'] = self.email_from 193 | # email_to = email_to_string.split(',') 194 | email['To'] = email_to_string 195 | # part_text = MIMEText(text_version, 'plain', 'utf-8') 196 | # email.attach(part_text) 197 | part_html = MIMEText(html_version, 'html', 'utf-8') 198 | email.attach(part_html) 199 | 200 | self._send_email(email, email_to_string) 201 | 202 | def _send_email(self, email, email_to_string): 203 | # send mail 204 | mail_server = smtplib.SMTP(host=self.smtp_server, port=self.smtp_port) 205 | if int(self.smtp_port) != 25: 206 | mail_server.starttls() 207 | if self.email_username is not None: 208 | mail_server.login(self.email_username, self.email_password) 209 | try: 210 | mail_server.sendmail(self.email_from, email_to_string, email.as_string()) 211 | self.sent_emails_counter += 1 212 | finally: 213 | mail_server.close() 214 | 215 | def _html_version_items(self, alert_event_list): 216 | """transform each alert_event in html.""" 217 | alert_event_style = { 218 | 'new': 'background-color: lightcyan', 219 | 'lost': 'background-color: lightgray', 220 | 'changed': 'background-color: lightgreen', 221 | 'unchanged':'background-color: white'} 222 | 223 | alert_condition_style = { 224 | 'normal':'color: darkgreen', 225 | 'warning':'color: darkorange', 226 | 'critical': 'color: darkred', 227 | 'none': 'color: sienna' 228 | } 229 | html = '' 230 | for alert_event in alert_event_list: 231 | # send alert_event only if something interesting happened. 232 | if alert_event['diff_event'] != 'unchanged' \ 233 | or alert_event['current'] is None \ 234 | or alert_event['current'].current_alert_condition_status is None \ 235 | or alert_event['current'].current_alert_condition_status['name'] != 'normal' \ 236 | or alert_event['old'] is None \ 237 | or alert_event['old'].current_alert_condition_status is None \ 238 | or alert_event['old'].current_alert_condition_status['name'] != 'normal': 239 | # TODO provide a better solution for undefined values 240 | variables = { 241 | 'old_target':'', 'old_alertName':'', 'old_title': '', 'old_value': '', 'old_condition':'', 242 | 'current_target':'', 'current_alertName':'', 'current_title': '', 'current_value': '', 'current_condition':'', 243 | 'alertName':'warning' 244 | } 245 | for version, alert_evaluation_result in {'old': alert_event['old'], 'current': alert_event['current']}.iteritems(): 246 | if alert_evaluation_result is not None: 247 | if alert_evaluation_result.current_alert_condition_status is not None: 248 | for k, v in alert_evaluation_result.current_alert_condition_status.iteritems(): 249 | variables[version + '_' + k] = v 250 | # add known value in case 'old' or 'current' values are null, it keeps the last value 251 | variables[k] = v 252 | else: 253 | # current_alert_condition_status are none, add some default values for templates 254 | variables['title'] = '-' # Bug? Panel title, it should be defined always 255 | variables[version + '_target'] = alert_evaluation_result.target 256 | variables[version + '_value'] = alert_evaluation_result.value 257 | # variables[version + '_alertName'] = alert_evaluation_result.current_alert_condition_status['name'] 258 | variables[version + '_alertName'] = alert_evaluation_result.current_alert_condition_status[ 259 | 'name'] if alert_evaluation_result.current_alert_condition_status is not None else 'none' 260 | # variables['_date'] = datetime.datetime.now().strftime("%B %d, %Y"), 261 | # variables['_time'] = datetime.datetime.now().strftime("%I:%M %p"), 262 | # add known value in case 'old' or 'current' values are null, it keeps the last value 263 | variables['target'] = alert_evaluation_result.target 264 | variables['value'] = alert_evaluation_result.value 265 | # variables['alertName'] = alert_evaluation_result.current_alert_condition_status['name'] 266 | variables['alertName'] = alert_evaluation_result.current_alert_condition_status[ 267 | 'name'] if alert_evaluation_result.current_alert_condition_status is not None else 'none' 268 | variables['diff_event'] = alert_event['diff_event'] 269 | 270 | # add styles for diff_event and alert event 271 | variables['alert_event_style'] = alert_event_style[variables['diff_event']] 272 | variables['alert_condition_style'] = alert_condition_style[variables['alertName']] 273 | 274 | html_item = self._html_version_item().format(**variables) 275 | html += html_item 276 | return html 277 | 278 | def _html_version_item(self): 279 | """html version of the list of alert_evaluation_result_list.""" 280 | # TODO review where the template is stored and how path is solved 281 | # return self._get_content_of_resource_file("../data/html_version_item.html") 282 | return self._get_content_of_resource_file("html_version_item.html") 283 | 284 | def _html_version_main(self): 285 | """html version of the email. It must returns everything except "html_version_items""" 286 | # TODO review where the template is stored and how path is solved 287 | # return self._get_content_of_resource_file("../data/html_version_main.html") 288 | return self._get_content_of_resource_file("html_version_main.html") 289 | 290 | def get_sent_emails_counter(self): 291 | return self.sent_emails_counter 292 | 293 | def _is_something_to_report(self, alert_event_list): 294 | """Return True if the alert should be sent.""" 295 | # TODO externalize this condition. 296 | for alert_event in alert_event_list: 297 | if alert_event['current'] is None: 298 | return True 299 | # elif alert_event['current'].current_alert_condition_status is None: 300 | # return True 301 | else: 302 | if alert_event['current'].current_alert_condition_status['name'] != 'normal': 303 | return True 304 | if alert_event['old'] is None: 305 | return True 306 | else: 307 | if alert_event['old'].current_alert_condition_status['name'] != 'normal': 308 | return True 309 | return False 310 | 311 | def _get_content_of_resource_file(self, resource_file): 312 | """read the file from the package resource and returns its content. File must exists.""" 313 | return pkg_resources.resource_string('grafana_alerts', resource_file) 314 | 315 | 316 | class ConsoleAlertReporter: 317 | pass 318 | --------------------------------------------------------------------------------