├── LICENSE ├── README.md └── icinga2-cleanup-influxdb.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 mj84 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # icinga2-cleanup-influxdb 2 | This script can be used to cleanup unused Icinga2 data in InfluxDB. 3 | 4 | The process is as follows: 5 | 6 | - collect list of hostnames present in Icinga2 via REST API 7 | - collect all measurements in Icinga2’s InfluxDB 8 | - collect all hostnames within a measurement 9 | - check if each hostname is still present in Icinga2, if not, drop ALL series for that host 10 | 11 | Requirements 12 | --- 13 | - python3 (tested with python 3.6.8) 14 | - python-requests (tested with 2.22.0) 15 | - influx binary in $PATH 16 | 17 | Usage 18 | --- 19 | 20 | - modify the following variables in the script as needed: 21 | - INFLUX_DATABASE 22 | - ICINGA2_API_URL 23 | - ICINGA2_API_USERNAME 24 | - ICINGA2_API_PASSWORD 25 | - run the script as root user on your InfluxDB host 26 | 27 | Notes 28 | --- 29 | 30 | **Run this at your own risk and create backups if your InfluxDB contains critical data!** 31 | 32 | Depending on the amount of series to be dropped, this script can be quite IO-intensive. 33 | 34 | Influx tries to compact its files as soon as a DROP SERIES command is issued. 35 | If another DROP SERIES command is issued before the compaction is finished, the compaction will be aborted which causes corresponding messages in the InfluxDB log. 36 | This is no issue, as the compaction will run again after this script is finished. 37 | -------------------------------------------------------------------------------- /icinga2-cleanup-influxdb.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import json 3 | import subprocess 4 | import requests 5 | # Disable SSL certificate validation 6 | from requests.packages.urllib3.exceptions import InsecureRequestWarning 7 | requests.packages.urllib3.disable_warnings(InsecureRequestWarning) 8 | 9 | INFLUX_DATABASE = 'icinga2' 10 | ICINGA2_API_URL = 'https://1.2.3.4:5665' 11 | ICINGA2_API_USERNAME = 'icinga2-api-user' 12 | ICINGA2_API_PASSWORD = 'password' 13 | 14 | # Read hostnames from Icinga2 15 | print('''+------------------------------------------------------------------+ 16 | Getting hostnames from Icinga2''') 17 | icinga2_hostnames_headers = {'X-HTTP-Method-Override': 'GET'} 18 | icinga2_hostnames_data = { 'attrs': ['name'] } 19 | icinga2_hostnames_request = requests.post('%s/v1/objects/hosts' % ICINGA2_API_URL, 20 | auth=(ICINGA2_API_USERNAME, ICINGA2_API_PASSWORD), 21 | headers=icinga2_hostnames_headers, 22 | json=icinga2_hostnames_data, 23 | verify=False) 24 | icinga2_hostnames_json = icinga2_hostnames_request.json()['results'] 25 | icinga2_hostnames = [] 26 | for hostname in icinga2_hostnames_json: 27 | icinga2_hostnames.append(hostname['name']) 28 | print('''Icinga2 returned %s hosts. 29 | +------------------------------------------------------------------+''' % len(icinga2_hostnames)) 30 | 31 | # Read available measurements in selected DB 32 | print('''+------------------------------------------------------------------+ 33 | Getting measurement list from InfluxDB''') 34 | db_list_measurements_command = ['influx', '-database', INFLUX_DATABASE, '-execute', 'SHOW MEASUREMENTS', '-format', 'json' ] 35 | db_list_measurements_result = subprocess.run(db_list_measurements_command, stdout=subprocess.PIPE) 36 | db_measurements_json = json.loads(db_list_measurements_result.stdout.decode('utf-8'))['results'][0]['series'][0]['values'] 37 | db_measurements = [] 38 | for measurement in db_measurements_json: 39 | db_measurements.append(measurement[0]) 40 | print('''InfluxDB returned %s measurements. 41 | +------------------------------------------------------------------+''' % len(db_measurements)) 42 | 43 | for measurement in db_measurements: 44 | print('''+------------------------------------------------------------------+ 45 | Cleaning up measurement %s''' % measurement) 46 | # Get hostnames in measurement 47 | measurement_list_hostnames_command = ['influx', '-database', INFLUX_DATABASE, '-execute', 'SHOW TAG VALUES FROM "%s" WITH KEY = "hostname"' % measurement, '-format', 'json' ] 48 | measurement_list_hostnames_result = subprocess.run(measurement_list_hostnames_command, stdout=subprocess.PIPE) 49 | measurement_hostnames_json = json.loads(measurement_list_hostnames_result.stdout.decode('utf-8'))['results'][0] 50 | if 'series' not in measurement_hostnames_json: 51 | # No hostnames in this measurement 52 | print('No hostnames in measurement %s.' % measurement) 53 | continue 54 | measurement_hostnames_json_series = measurement_hostnames_json['series'][0]['values'] 55 | measurement_hostnames = [] 56 | measurement_orphaned_hosts = [] 57 | for hostname in measurement_hostnames_json_series: 58 | measurement_hostname = hostname[1] 59 | measurement_hostnames.append(measurement_hostname) 60 | if measurement_hostname not in icinga2_hostnames: 61 | print('Hostname %s does not exist in Icinga2!' % measurement_hostname) 62 | measurement_orphaned_hosts.append(measurement_hostname) 63 | print('Dropping all series for host %s' % measurement_hostname) 64 | drop_series_command = ['influx', '-database', INFLUX_DATABASE, '-execute', 'DROP SERIES WHERE "hostname" = \'%s\'' % measurement_hostname ] 65 | drop_series_result = subprocess.run(drop_series_command, stdout=subprocess.PIPE) 66 | # Uncomment the line below to print output of DROP SERIES command, which is usually empty 67 | #print('DROP SERIES returned: %s' % drop_series_result.stdout.decode('utf-8')) 68 | print('''Measurement %s returned %s/%s orphaned/total hosts. 69 | +------------------------------------------------------------------+''' % (measurement, len(measurement_orphaned_hosts), len(measurement_hostnames))) 70 | --------------------------------------------------------------------------------