.
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | kvm-snapshot-backup
2 | ===================
3 |
4 | Creates KVM external disk incremental snapshot backups.
5 |
6 | Requirements
7 | ------------
8 | - python 3.x
9 | - libvirt-python
10 | - virsh
11 |
12 | Usage
13 | -----
14 |
15 | 1. Incremental backup
16 | # ./kvm_snapshot_backup.py backup -d vm1 -b /backups/vm
17 | It creates domain snapshot and copy backing file of current disk to 'backup_dir' (-b).
18 | After this it validates all backups disk files are consistent.
19 | You can backup your domain disk files and merge complete base to current snapshot weekly, monthly or whatever.
20 |
21 | 2. Merge base to current snapshot
22 | # ./kvm_snapshot_backup.py merge -d vm1
23 | Merges complete base to snapshot and removes old disk files (backing file tree). Finally you have one file for every disk device in domain.
It uses virsh blockpull mechanism.
24 |
25 | 2. Rotate backup disk files
26 | # ./kvm_snapshot_backup.py rotate -d vm1 -b /backups/vm [-r 1]
27 | Rotates domain backup disk files in 'backup_dir' (-b) leaving the last X (-r) backups, defaults to 1.
28 |
--------------------------------------------------------------------------------
/kvm_snapshot_backup.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | # -*- coding: utf-8 -*-
3 |
4 | import argparse
5 | import fcntl
6 | import glob
7 | import shutil
8 | from datetime import datetime
9 | import libvirt
10 | import logging
11 | import os
12 | import re
13 | import shlex
14 | import subprocess
15 | import sys
16 | from xml.etree import ElementTree
17 | from collections import namedtuple
18 |
19 |
20 | class Domain(object):
21 | def __init__(self, libvirt_domain: libvirt.virDomain):
22 | self.libvirt_domain = libvirt_domain
23 | self.name = libvirt_domain.name()
24 | self.libvirt_snapshot = None
25 |
26 | def get_disks(self):
27 | """ Gets all domain disk as namedtuple('DiskInfo', ['device', 'file', 'format']) """
28 | # root node
29 | root = ElementTree.fromstring(self.libvirt_domain.XMLDesc())
30 |
31 | # search entries
32 | disks = root.findall("./devices/disk[@device='disk']")
33 |
34 | # for every disk get drivers, sources and targets
35 | drivers = [disk.find("driver").attrib for disk in disks]
36 | sources = [disk.find("source").attrib for disk in disks]
37 | targets = [disk.find("target").attrib for disk in disks]
38 |
39 | # iterate drivers, sources and targets
40 | if len(drivers) != len(sources) != len(targets):
41 | raise RuntimeError("Drivers, sources and targets lengths are different %s:%s:%s" % (
42 | len(drivers), len(sources), len(targets)))
43 |
44 | disk_info = namedtuple('DiskInfo', ['device', 'file', 'format'])
45 |
46 | # all disks info
47 | disks_info = []
48 |
49 | for i in range(len(sources)):
50 | disks_info.append(disk_info(targets[i]["dev"], sources[i]["file"], drivers[i]["type"]))
51 |
52 | return disks_info
53 |
54 | def create_snapshot_xml(self):
55 | """ Creates snapshot XML """
56 | snapshot_name = datetime.now().strftime("%Y%m%d_%H%M%S")
57 |
58 | disk_specs = []
59 | domain_disks = self.get_disks()
60 |
61 | for disk in domain_disks:
62 | disk_basedir = os.path.dirname(disk.file)
63 | disk_specs += ["--diskspec %s,file=\"%s/%s_%s-%s.%s\"" % (
64 | disk.device, disk_basedir, self.name, disk.device, snapshot_name, disk.format)]
65 |
66 | if not disk_specs:
67 | raise RuntimeError("Wrong disk devices specified? Available devices: %s" % domain_disks)
68 |
69 | snapshot_create_cmd = (
70 | "virsh snapshot-create-as --domain {domain_name} {snapshot_name} {disk_specs}"
71 | " --disk-only --atomic --quiesce --print-xml").format(
72 | domain_name=self.name, snapshot_name=snapshot_name, disk_specs=" ".join(disk_specs))
73 | logging.debug("Executing: '%s'" % snapshot_create_cmd)
74 |
75 | snapshot_create_cmds = shlex.split(snapshot_create_cmd)
76 |
77 | create_xml = subprocess.Popen(snapshot_create_cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
78 |
79 | snapshot_xml = create_xml.stdout.read()
80 |
81 | status = create_xml.wait()
82 |
83 | if status != 0:
84 | logging.error("Error for '%s': %s" % (snapshot_create_cmd, create_xml.stderr.read()))
85 | logging.critical("{exe} returned {status} state".format(exe=snapshot_create_cmds[0], status=status))
86 | raise Exception("snapshot-create-as didn't work properly")
87 |
88 | return snapshot_xml
89 |
90 | def create_snapshot(self):
91 | """ Creates domain snapshot """
92 | logging.debug("Creating snapshot XML")
93 | snapshot_xml = self.create_snapshot_xml()
94 |
95 | logging.info("Creating snapshot based on snapshot XML")
96 | self.libvirt_snapshot = self.libvirt_domain.snapshotCreateXML(
97 | snapshot_xml.decode('utf-8'),
98 | flags=sum(
99 | [libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_DISK_ONLY,
100 | libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_ATOMIC,
101 | # libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_QUIESCE,
102 | libvirt.VIR_DOMAIN_SNAPSHOT_CREATE_NO_METADATA]))
103 | return self.libvirt_snapshot
104 |
105 | def backup_incremental(self, backup_dir: str):
106 | """ Makes incremental backup with snapshots (with check that we have all backups files).
107 | It creates snapshot, backup snapshot backing file and checks that every parent backing files are in backups.
108 | """
109 | # create snapshot if not exists
110 | if self.libvirt_snapshot is None:
111 | self.create_snapshot()
112 |
113 | backup_domain_dir = self.get_backup_domain_dir(backup_dir)
114 |
115 | # create backup domain directory if not exists
116 | if not os.path.exists(backup_domain_dir) and not os.path.isdir(backup_domain_dir):
117 | logging.info("Creating directory '%s'" % backup_domain_dir)
118 | os.mkdir(backup_domain_dir)
119 |
120 | # for every disk
121 | for disk in self.get_disks():
122 | # get current backing file from disk_info
123 | backing_file = DiskImageHelper.get_backing_file(disk.file)
124 | # while it is a backing file for every snapshot disk
125 | while backing_file is not None:
126 | # backup file
127 | backup_file = os.path.join(backup_domain_dir, os.path.basename(backing_file))
128 | # do backup if not exists
129 | backing_file_copy_result = False
130 | if os.path.isfile(backup_file) is False:
131 | # copy
132 | logging.info("Copying '%s' to '%s'" % (backing_file, backup_file))
133 | shutil.copy2(backing_file, backup_file)
134 | backing_file_copy_result = True
135 |
136 | # set parent backing file
137 | backing_file = DiskImageHelper.get_backing_file(backing_file)
138 | # set valid backing file for backup_file after copy
139 | if backing_file_copy_result and backing_file:
140 | DiskImageHelper.set_backing_file(os.path.basename(backing_file), backup_file)
141 |
142 | # backup domain XML
143 | backup_xml_file = "%s/%s.xml" % (backup_domain_dir, self.name)
144 | logging.info("Creating domain XML backup '%s" % backup_xml_file)
145 | backup_xml_fo = open(backup_xml_file, 'w')
146 | backup_xml_fo.write(self.libvirt_domain.XMLDesc())
147 | backup_xml_fo.close()
148 |
149 | def get_backup_domain_dir(self, backup_dir):
150 | """ Gets full backup domain dir """
151 | # prepare backup domain directory path
152 | backup_domain_dir = os.path.join(backup_dir, self.name)
153 | return backup_domain_dir
154 |
155 | def merge_snapshot(self):
156 | """ Merges base to snapshot and removes old disk files """
157 | disks = self.get_disks()
158 | disk_files_tree = []
159 | for disk in disks:
160 | disk_files_tree += (DiskImageHelper.get_backing_files_tree(disk.file))
161 | merge_snapshot_cmd = "virsh blockpull --domain {domain_name} {disk_path} --wait".format(
162 | domain_name=self.name, disk_path=disk.file)
163 |
164 | logging.debug("Executing: '%s'" % merge_snapshot_cmd)
165 | logging.info("Merging base to new snapshot for '%s' device" % disk.device)
166 |
167 | # launch command
168 | merge_snapshot_cmds = shlex.split(merge_snapshot_cmd)
169 | merge_snapshot = subprocess.Popen(merge_snapshot_cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
170 | shell=False)
171 |
172 | # wait to terminate
173 | status = merge_snapshot.wait()
174 |
175 | if status != 0:
176 | logging.error("Error for '%s': %s" % (merge_snapshot_cmd, merge_snapshot.stderr.read()))
177 | logging.critical("{exe} returned {status} state".format(exe=merge_snapshot_cmds[0], status=status))
178 | raise Exception("blockpull didn't work properly")
179 |
180 | current_disk_files = [disk.file for disk in self.get_disks()]
181 |
182 | # remove old disk device files without current ones
183 | for file in [disk_file_tree for disk_file_tree in disk_files_tree if disk_file_tree not in current_disk_files]:
184 | logging.info("Removing old disk file: '%s'" % file)
185 | os.remove(file)
186 |
187 | def backup_rotate_daily(self, backup_dir: str, rotate: int):
188 | """ Rotates domain backup disk files in 'backup_dir' leaving the last 'rotate' backups """
189 | if rotate < 1:
190 | raise Exception("Rotate should be more than 0")
191 | backup_domain_dir = self.get_backup_domain_dir(backup_dir)
192 | # for every file in directory group backups
193 | for disk in self.get_disks():
194 | grouped_files = []
195 | backup_files = glob.glob(
196 | os.path.join(backup_domain_dir, "%s_%s-*.%s" % (self.name, disk.device, disk.format)))
197 | backup_files.sort(key=os.path.getmtime, reverse=True)
198 | backing_file = None
199 | for backup_file in backup_files:
200 | if backing_file is None:
201 | grouped_files.append([])
202 | grouped_files[-1].append(backup_file)
203 | backing_file = DiskImageHelper.get_backing_file(backup_file)
204 | logging.debug("Grouped backup files %s" % grouped_files)
205 | grouped_files_to_remove = grouped_files[rotate:]
206 | logging.debug("Groups to remove %s" % grouped_files_to_remove)
207 | for group in grouped_files_to_remove:
208 | for file in group:
209 | logging.info("Removing old backup disk file: '%s'" % file)
210 | os.remove(file)
211 |
212 |
213 | class DiskImageHelper(object):
214 | @staticmethod
215 | def get_backing_file(file: str):
216 | """ Gets backing file for disk image """
217 | get_backing_file_cmd = "qemu-img info %s" % file
218 | logging.debug("Executing: '%s'" % get_backing_file_cmd)
219 | out = subprocess.check_output(shlex.split(get_backing_file_cmd))
220 | lines = out.decode('utf-8').split('\n')
221 | for line in lines:
222 | if re.search("backing file:", line):
223 | return line.strip().split()[2]
224 | return None
225 |
226 | @staticmethod
227 | def get_backing_files_tree(file: str):
228 | """ Gets all backing files (snapshot tree) for disk image """
229 | backing_files = []
230 | backing_file = DiskImageHelper.get_backing_file(file)
231 | while backing_file is not None:
232 | backing_files.append(backing_file)
233 | backing_file = DiskImageHelper.get_backing_file(backing_file)
234 | return backing_files
235 |
236 | @staticmethod
237 | def set_backing_file(backing_file: str, file: str):
238 | """ Sets backing file for disk image """
239 | set_backing_file_cmd = "qemu-img rebase -u -b %s %s" % (backing_file, file)
240 | logging.debug("Executing: '%s'" % set_backing_file_cmd)
241 | subprocess.check_output(shlex.split(set_backing_file_cmd))
242 |
243 |
244 | script_name = os.path.basename(sys.argv[0][:-3])
245 |
246 | if __name__ == "__main__":
247 | parser = argparse.ArgumentParser()
248 | subparsers = parser.add_subparsers()
249 | backup_parser = subparsers.add_parser('backup')
250 | backup_parser.add_argument('-a', '--action', action="store_const", const="backup", default="backup")
251 | backup_parser.add_argument('-d', '--domain', required=True)
252 | backup_parser.add_argument('-b', '--backup-dir', dest='backup_dir', required=True)
253 | merge_parser = subparsers.add_parser('merge')
254 | merge_parser.add_argument('-a', '--action', action="store_const", const="merge", default="merge")
255 | merge_parser.add_argument('-d', '--domain', required=True)
256 | rotate_parser = subparsers.add_parser('rotate')
257 | rotate_parser.add_argument('-a', '--action', action="store_const", const="backup", default="rotate")
258 | rotate_parser.add_argument('-r', '--rotate', default=1, type=int)
259 | rotate_parser.add_argument('-d', '--domain', required=True)
260 | rotate_parser.add_argument('-b', '--backup-dir', dest='backup_dir', required=True)
261 | parser.add_argument('-v', dest='verbose', default='INFO', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
262 | type=str)
263 |
264 | command_args = parser.parse_args()
265 |
266 | if not any(vars(command_args).values()) or vars(command_args) == {"verbose": "INFO"}:
267 | parser.print_usage()
268 | exit()
269 |
270 | try:
271 | pid_file = '/tmp/kvm_snapshot_backup.lock'
272 | fp = open(pid_file, 'w')
273 |
274 | # lock script instance, only one instance should run
275 | fcntl.lockf(fp, fcntl.LOCK_EX | fcntl.LOCK_NB)
276 |
277 | logging.basicConfig(format='%(levelname)s: %(message)s', level=getattr(logging, command_args.verbose.upper()))
278 |
279 | logging.debug("START")
280 |
281 | logging.debug("Opening libvirt connection to qemu")
282 | app_libvirt_conn = libvirt.open("qemu:///system")
283 |
284 | app_domain = Domain(app_libvirt_conn.lookupByName(command_args.domain))
285 |
286 | if command_args.action == "backup":
287 | app_domain.backup_incremental(command_args.backup_dir)
288 | elif command_args.action == "merge":
289 | app_domain.merge_snapshot()
290 | elif command_args.action == "rotate":
291 | app_domain.backup_rotate_daily(command_args.backup_dir, command_args.rotate)
292 |
293 | app_libvirt_conn.close()
294 |
295 | except IOError as ioe:
296 | logging.error(str(ioe))
297 | except RuntimeError as re:
298 | logging.error(str(re))
299 |
300 | logging.debug("STOP")
301 |
--------------------------------------------------------------------------------