├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── enhancement---feature-request.md │ └── installation-problem.md ├── dependabot.yml └── workflows │ ├── build.yml │ ├── ci-fstrim.yml │ ├── ci-fstrimsmall.yml │ ├── ci-nosparsedetect.yml │ ├── ci-ubuntu-latest.yml │ └── docker.yml ├── .gitignore ├── .pylintrc ├── CNAME ├── Changelog ├── LICENSE ├── MANIFEST.in ├── README.md ├── _config.yml ├── docker ├── Dockerfile └── README.md ├── libvirtnbdbackup ├── __init__.py ├── argopt.py ├── backup │ ├── __init__.py │ ├── check.py │ ├── disk.py │ ├── job.py │ ├── metadata.py │ ├── partialfile.py │ ├── server.py │ └── target.py ├── block.py ├── chunk.py ├── common.py ├── exceptions.py ├── extenthandler │ ├── __init__.py │ └── extenthandler.py ├── logcount.py ├── lz4.py ├── map │ ├── __init__.py │ ├── changes.py │ ├── ranges.py │ └── requirements.py ├── nbdcli │ ├── __init__.py │ ├── client.py │ ├── context.py │ └── exceptions.py ├── objects.py ├── output │ ├── __init__.py │ ├── exceptions.py │ ├── stream.py │ └── target.py ├── qemu │ ├── __init__.py │ ├── command.py │ ├── exceptions.py │ └── util.py ├── restore │ ├── __init__.py │ ├── data.py │ ├── disk.py │ ├── files.py │ ├── header.py │ ├── image.py │ ├── sequence.py │ ├── server.py │ └── vmconfig.py ├── sighandle.py ├── sparsestream │ ├── __init__.py │ ├── exceptions.py │ ├── streamer.py │ └── types.py ├── ssh │ ├── __init__.py │ ├── client.py │ └── exceptions.py └── virt │ ├── __init__.py │ ├── checkpoint.py │ ├── client.py │ ├── disktype.py │ ├── exceptions.py │ ├── fs.py │ └── xml.py ├── man ├── virtnbdbackup.1 ├── virtnbdmap.1 └── virtnbdrestore.1 ├── requirements.txt ├── screenshot.jpg ├── scripts ├── create-cert.sh └── mangen.sh ├── setup.cfg ├── setup.py ├── stdeb.cfg ├── t ├── Makefile ├── README.txt ├── agent-exec.sh ├── fstest.bats ├── fstest │ ├── config.bash │ └── fstest.xml ├── fstrim.bats ├── fstrim │ ├── README.txt │ ├── config.bash │ └── fstrim.xml ├── fstrimsmall.bats ├── fstrimsmall │ ├── README.txt │ ├── config.bash │ └── fstrimsmall.xml ├── genimage.sh ├── nosparsedetect.bats ├── nosparsedetect │ ├── README.txt │ ├── config.bash │ └── nosparsedetect.xml ├── tests.bats ├── vm1 │ ├── .gitignore │ ├── UEFI_VARS.gz │ ├── config.bash │ ├── vm1-sda.qcow2.gz │ └── vm1.xml ├── vm2 │ ├── config.bash │ └── vm2.xml ├── vm3 │ ├── config.bash │ ├── vm3-sda.qcow2 │ ├── vm3-sdb.qcow2 │ └── vm3.xml ├── vm4 │ ├── README.txt │ ├── config.bash │ ├── vm4-sda.qcow2 │ ├── vm4-sdb.qcow2 │ └── vm4.xml └── vm5 │ ├── README.txt │ ├── config.bash │ ├── vm5-sda.qcow2 │ └── vm5.xml ├── venv └── create.sh ├── virtnbd-nbdkit-plugin ├── virtnbdbackup ├── virtnbdmap └── virtnbdrestore /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: abbbi 2 | buy_me_a_coffee: abbbi 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a bug report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Check README/FAQ/Discussions first** 11 | 12 | Before filing an issue, please check if your problem is listed in the 13 | README FAQ Section or was already part of an past discussion. 14 | 15 | **Version used** 16 | Provide output of `virtnbdbackup -V` 17 | 18 | **Describe the bug** 19 | A clear and concise description of what the bug is. 20 | 21 | **Expected behavior** 22 | A clear and concise description of what you expected to happen. 23 | 24 | 25 | **Hypervisor and/or virtual machine information:** 26 | - OS: [e.g. Alma, Debian] 27 | - HV type [e.g. opennebula, plain libvirt, rhev] 28 | - Virtual machine configuration [virsh dumpxml ..] 29 | 30 | **Logfiles:** 31 | Please attach generated logfiles relevant to the reported issue. 32 | 33 | **Workaround:** 34 | Share possible workarounds, if any. 35 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/enhancement---feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Enhancement / feature request 3 | about: Propose a new feature or request. 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/installation-problem.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Installation problem 3 | about: Issues during setup or installation of the utility 4 | title: '' 5 | labels: installation 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | - package-ecosystem: "docker" 8 | directory: "/" 9 | schedule: 10 | interval: "daily" 11 | -------------------------------------------------------------------------------- /.github/workflows/ci-fstrim.yml: -------------------------------------------------------------------------------- 1 | name: fstrim CI on ubuntu-22.04 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | 8 | runs-on: ubuntu-22.04 9 | 10 | steps: 11 | - uses: actions/checkout@v4 12 | - uses: abbbi/github-actions-tune@v1 13 | - name: Set up libvirt 14 | run: | 15 | sudo apt-get update 16 | sudo apt-get install -y \ 17 | apparmor-profiles \ 18 | bridge-utils \ 19 | dnsmasq-base \ 20 | ebtables \ 21 | libarchive-tools \ 22 | libguestfs-tools \ 23 | libvirt-clients \ 24 | libvirt-daemon \ 25 | libvirt-daemon-system \ 26 | qemu-kvm \ 27 | qemu-utils \ 28 | python3-libnbd \ 29 | python3-tqdm \ 30 | python3-lz4 \ 31 | python3-libvirt \ 32 | python3-lxml \ 33 | python3-paramiko\ 34 | python3-scp \ 35 | python3-colorlog \ 36 | nbdkit \ 37 | nbdkit-plugin-python \ 38 | unzip \ 39 | libnbd-bin \ 40 | ; 41 | # start daemon 42 | echo 'security_driver = "none"' | sudo tee -a /etc/libvirt/qemu.conf 43 | sudo aa-teardown 44 | sudo rm -f /etc/apparmor.d/libvirt/libvirt* 45 | sudo systemctl start libvirtd 46 | sudo systemctl restart libvirtd 47 | sudo modprobe nbd max_partitions=10 48 | - name: Execute tests (fstrim) 49 | run: cd t && sudo -E make fstrim.tests 50 | -------------------------------------------------------------------------------- /.github/workflows/ci-fstrimsmall.yml: -------------------------------------------------------------------------------- 1 | name: fstrimsmall CI on ubuntu-22.04 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | 8 | runs-on: ubuntu-22.04 9 | 10 | steps: 11 | - uses: actions/checkout@v4 12 | - uses: abbbi/github-actions-tune@v1 13 | - name: Set up libvirt 14 | run: | 15 | sudo apt-get update 16 | sudo apt-get install -y \ 17 | apparmor-profiles \ 18 | bridge-utils \ 19 | dnsmasq-base \ 20 | ebtables \ 21 | libarchive-tools \ 22 | libguestfs-tools \ 23 | libvirt-clients \ 24 | libvirt-daemon \ 25 | libvirt-daemon-system \ 26 | qemu-kvm \ 27 | qemu-utils \ 28 | python3-libnbd \ 29 | python3-tqdm \ 30 | python3-lz4 \ 31 | python3-libvirt \ 32 | python3-lxml \ 33 | python3-paramiko\ 34 | python3-scp \ 35 | python3-colorlog \ 36 | nbdkit \ 37 | nbdkit-plugin-python \ 38 | unzip \ 39 | libnbd-bin \ 40 | ; 41 | # start daemon 42 | echo 'security_driver = "none"' | sudo tee -a /etc/libvirt/qemu.conf 43 | sudo aa-teardown 44 | sudo rm -f /etc/apparmor.d/libvirt/libvirt* 45 | sudo systemctl start libvirtd 46 | sudo systemctl restart libvirtd 47 | sudo modprobe nbd max_partitions=10 48 | - name: Execute tests (fstrimsmall) 49 | run: cd t && sudo -E make fstrimsmall.tests 50 | -------------------------------------------------------------------------------- /.github/workflows/ci-nosparsedetect.yml: -------------------------------------------------------------------------------- 1 | name: nosparsedetect CI on ubuntu-22.04 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | 8 | runs-on: ubuntu-22.04 9 | 10 | steps: 11 | - uses: actions/checkout@v4 12 | - uses: abbbi/github-actions-tune@v1 13 | - name: Set up libvirt 14 | run: | 15 | sudo apt-get update 16 | sudo apt-get install -y \ 17 | apparmor-profiles \ 18 | bridge-utils \ 19 | dnsmasq-base \ 20 | ebtables \ 21 | libarchive-tools \ 22 | libguestfs-tools \ 23 | libvirt-clients \ 24 | libvirt-daemon \ 25 | libvirt-daemon-system \ 26 | qemu-kvm \ 27 | qemu-utils \ 28 | python3-libnbd \ 29 | python3-tqdm \ 30 | python3-lz4 \ 31 | python3-libvirt \ 32 | python3-lxml \ 33 | python3-paramiko\ 34 | python3-scp \ 35 | python3-colorlog \ 36 | nbdkit \ 37 | nbdkit-plugin-python \ 38 | unzip \ 39 | libnbd-bin \ 40 | ; 41 | # start daemon 42 | echo 'security_driver = "none"' | sudo tee -a /etc/libvirt/qemu.conf 43 | sudo aa-teardown 44 | sudo rm -f /etc/apparmor.d/libvirt/libvirt* 45 | sudo systemctl start libvirtd 46 | sudo systemctl restart libvirtd 47 | sudo modprobe nbd max_partitions=10 48 | - name: Execute tests (nosparsedetect) 49 | run: cd t && sudo -E make nosparsedetect.tests 50 | -------------------------------------------------------------------------------- /.github/workflows/ci-ubuntu-latest.yml: -------------------------------------------------------------------------------- 1 | name: virtnbdbackup CI on ubuntu-22.04 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | 8 | runs-on: ubuntu-22.04 9 | 10 | steps: 11 | - uses: actions/checkout@v4 12 | - uses: abbbi/github-actions-tune@v1 13 | - name: codespell 14 | run: | 15 | sudo apt-get update 16 | sudo apt-get install codespell -y 17 | codespell libvirtnbdbackup 18 | codespell virtnbdbackup 19 | codespell virtnbdrestore 20 | codespell virtnbdmap 21 | codespell virtnbd-nbdkit-plugin 22 | codespell README.md 23 | codespell Changelog 24 | - name: Python code format test 25 | run: | 26 | sudo pip3 install black==25.1.0 27 | black --check . 28 | black --check virtnbdbackup 29 | black --check virtnbdrestore 30 | black --check virtnbdmap 31 | black --check virtnbd-nbdkit-plugin 32 | - name: Python lint test 33 | run: | 34 | sudo apt-get install python3-colorlog python3-paramiko -y 35 | sudo pip3 install pylint==2.14.5 36 | pylint libvirtnbdbackup 37 | pylint virtnbd-nbdkit-plugin 38 | - name: Python typing tests 39 | run: | 40 | sudo pip3 install mypy 41 | sudo pip3 install types-paramiko 42 | mypy --ignore-missing libvirtnbdbackup 43 | mypy --ignore-missing virtnbdbackup 44 | mypy --ignore-missing virtnbdrestore 45 | mypy --ignore-missing virtnbdmap 46 | - name: Set up ssh access to localhost 47 | run: | 48 | sudo hostname -f 49 | sudo hostname 50 | sudo ssh-keygen -f /root/.ssh/id_rsa -N "" 51 | sudo cat /root/.ssh/id_rsa.pub | sudo tee -a /root/.ssh/authorized_keys 52 | sudo ssh-keyscan -H localhost | sudo tee -a /root/.ssh/known_hosts 53 | sudo ssh -l root localhost true 54 | sudo ssh-agent | sudo tee -a /root/agent 55 | sudo -- bash -c 'source /root/agent; ssh-add' 56 | - name: Set up libvirt 57 | run: | 58 | sudo apt-get update 59 | sudo apt-get install -y \ 60 | apparmor-profiles \ 61 | bridge-utils \ 62 | dnsmasq-base \ 63 | ebtables \ 64 | libarchive-tools \ 65 | libguestfs-tools \ 66 | libvirt-clients \ 67 | libvirt-daemon \ 68 | libvirt-daemon-system \ 69 | qemu-kvm \ 70 | qemu-utils \ 71 | python3-libnbd \ 72 | python3-tqdm \ 73 | python3-lz4 \ 74 | python3-libvirt \ 75 | python3-lxml \ 76 | python3-paramiko\ 77 | python3-scp \ 78 | python3-colorlog \ 79 | nbdkit \ 80 | nbdkit-plugin-python \ 81 | unzip \ 82 | libnbd-bin \ 83 | ; 84 | # start daemon 85 | echo 'security_driver = "none"' | sudo tee -a /etc/libvirt/qemu.conf 86 | sudo aa-teardown 87 | sudo rm -f /etc/apparmor.d/libvirt/libvirt* 88 | sudo systemctl start libvirtd 89 | sudo systemctl restart libvirtd 90 | sudo modprobe nbd max_partitions=10 91 | - name: Execute tests (vm1) 92 | run: cd t && sudo -E make vm1.tests && cd - 93 | - name: Execute tests (vm3) 94 | run: cd t && sudo -E make vm3.tests && cd - 95 | - name: Execute tests (vm4) 96 | run: cd t && sudo -E make vm4.tests && cd - 97 | - name: Perform installation 98 | run: sudo python3 setup.py install 99 | - name: Execute commands 100 | run: virtnbdbackup -h && virtnbdrestore -h 101 | - name: Create log archive 102 | if: always() 103 | run: | 104 | sudo find /tmp/ -type d -name '.guestfs-*' | sudo xargs rm -rf 105 | sudo tar -czvf /log.tar.gz /tmp/tmp.* 106 | sudo chmod go+r /log.tar.gz 107 | - name: Archive Test Results 108 | if: always() 109 | uses: actions/upload-artifact@v4 110 | with: 111 | name: test-results 112 | path: /log.tar.gz 113 | -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: Docker 2 | 3 | on: 4 | push: 5 | branches: 6 | - "main" 7 | - "master" 8 | workflow_dispatch: 9 | release: 10 | types: [published, edited] 11 | 12 | jobs: 13 | build-and-publish-image: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v4 18 | - name: Docker metadata 19 | id: meta 20 | uses: docker/metadata-action@v5 21 | with: 22 | images: | 23 | ghcr.io/${{ github.repository_owner }}/virtnbdbackup 24 | tags: | 25 | type=ref,event=branch 26 | type=ref,event=tag 27 | type=ref,event=pr 28 | 29 | - name: Set up QEMU 30 | uses: docker/setup-qemu-action@v3 31 | 32 | - name: Set up Docker Buildx 33 | uses: docker/setup-buildx-action@v3 34 | 35 | - name: Login to GitHub Container Registry 36 | if: github.event_name != 'pull_request' 37 | uses: docker/login-action@v3 38 | with: 39 | registry: ghcr.io 40 | username: ${{ github.repository_owner }} 41 | password: ${{ secrets.GITHUB_TOKEN }} 42 | 43 | - name: Build and Publish Docker Image 44 | uses: docker/build-push-action@v6 45 | with: 46 | file: ./docker/Dockerfile 47 | push: ${{ github.event_name != 'pull_request' }} 48 | platforms: linux/amd64 49 | tags: ${{ steps.meta.outputs.tags }} 50 | labels: ${{ steps.meta.outputs.labels }} 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # System Files 2 | .DS_Store 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | env/ 15 | build/ 16 | develop-eggs/ 17 | dist/ 18 | downloads/ 19 | eggs/ 20 | .eggs/ 21 | lib/ 22 | lib64/ 23 | parts/ 24 | sdist/ 25 | var/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | .vagrant/ 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *,cover 50 | .hypothesis/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | local_settings.py 59 | 60 | # Sphinx documentation 61 | docs/_build/ 62 | 63 | # PyBuilder 64 | target/ 65 | 66 | #Ipython Notebook 67 | .ipynb_checkpoints 68 | 69 | # pyenv 70 | .python-version 71 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MESSAGES CONTROL] 2 | disable=C0103,R0903 3 | ignored-modules=libvirt,lxml,nbd,lz4.frame,tqdm,scp,lxml.etree,nbdkit 4 | -------------------------------------------------------------------------------- /CNAME: -------------------------------------------------------------------------------- 1 | libvirtbackup.grinser.de -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include LICENSE 2 | include Changelog 3 | include docker/* 4 | include man/* 5 | include requirements.txt 6 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | remote_theme: abbbi/virtnbdbackup-theme 2 | show_downloads: true 3 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # based on Docker image created by 2 | # Adrián Parilli 3 | # Uses parent directory as context: 4 | # git clone https://github.com/abbbi/virtnbdbackup 5 | # cd virtnbdbackup 6 | # docker build -f docker/Dockerfile . 7 | FROM debian:bookworm-slim 8 | 9 | ARG DEBIAN_FRONTEND="noninteractive" 10 | ARG source="https://github.com/abbbi/virtnbdbackup" 11 | 12 | LABEL container.name="virtnbdbackup-docker" 13 | LABEL container.source.description="Backup utiliy for Libvirt kvm / qemu with Incremental backup support via NBD" 14 | LABEL container.description="virtnbdbackup and virtnbdrestore (plus dependencies) to run on hosts with libvirt >= 6.0.0" 15 | LABEL container.source=$source 16 | LABEL container.version="1.1" 17 | LABEL maintainer="Michael Ablassmeier " 18 | 19 | COPY . /tmp/build/ 20 | 21 | # Deploys dependencies and pulls sources, installing virtnbdbackup and removing unnecessary content: 22 | RUN \ 23 | apt-get update && \ 24 | apt-get install -y --no-install-recommends \ 25 | ca-certificates openssh-client python3-all python3-libnbd python3-libvirt python3-lz4 python3-setuptools python3-tqdm qemu-utils python3-lxml python3-paramiko python3-colorlog && \ 26 | cd /tmp/build/ && python3 setup.py install && cd .. && \ 27 | apt-get purge -y ca-certificates && apt-get -y autoremove --purge && apt-get clean && \ 28 | rm -rf /var/lib/apt/lists/* /tmp/* 29 | 30 | # Default folder: 31 | WORKDIR / 32 | -------------------------------------------------------------------------------- /docker/README.md: -------------------------------------------------------------------------------- 1 | ## Overview 2 | 3 | This dockerfile is intended for scenarios where isn't viable to provide the necessary environment, such as dependencies or tools, due to system limitations; such as an old OS version, inmutable or embedded rootfs, live distros, docker oriented OSes, etc. 4 | 5 | Originally was created to be used on Unraid OS (tested since v6.9.2), and should work equally fine on any other GNU/Linux distro as much as [requirements](#requirements) are accomplished. 6 | 7 | Includes `virtnbdbackup`, `virtnbdrestore` and similar utils, installed along with their required dependecies. Other utilities, such as latest Qemu Utils and OpenSSH Client, are also included to leverage all available features. 8 | 9 | Currently, is being built from latest `debian:bookworm-slim` official image. 10 | 11 | ## Requirements 12 | 13 | - Docker Engine on the host server. See [Docker Documentation](https://docs.docker.com/get-docker/) for further instructions 14 | - Libvirt >=v6.0.0. on the host server (minimal). A version >=7.6.0 is necessary to avoid [patching XML VM definitions](../README.md#libvirt-versions--760-debian-bullseye-ubuntu-20x) 15 | - Qemu Guest Agent installed and running inside the guest OS. For *NIX guests, use the latest available version according the distro (installed by default on Debian 12 when provisioned via ISO). For Windows guests, install latest [VirtIO drivers](https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/archive-virtio/) 16 | 17 | ## Bind mounts: 18 | 19 | *All the trick consists into set the right bind mounts for your host OS case* 20 | 21 | - Virtnbdbackup needs to access libvirt's socket in order to work correctly, and attempts this via `/var/run/libvirt` path. 22 | 23 | In basically all mainstream distros of today (Debian, RedHat, Archlinux and the countless distros based on these) as in this image, `/var/run` is a symlink to `/run` and `/var/lock` a symlink to `run/lock`. 24 | Therefore, for the vast majority of scenarios the correct bind mount is: `-v /run:/run` 25 | 26 | But in some operating systems, `/run` and `/var/run` are still separated folders. Under this scenario you need to bind mount with `-v /var/run:/run` 27 | And most likely, you will need to mount with either `-v /var/lock:/run/lock` or `-v /var/run/lock:/run/lock` in order to run this container correctly. 28 | 29 | If you're in trouble with this, read [Main FAQ](../README.md#faq) first, and identify the error you're getting in order to set the correct bind mounts that work for the specific host that serves Docker. 30 | 31 | - Virtnbdbackup and virtnbdrestore create sockets for backup/restoration jobs tasks at `/var/tmp`. Ensure to *always* add a bind mount with `-v /var/tmp:/var/tmp` 32 | 33 | - When working with VMs that require to boot with UEFI emulation (e.g. Windows 10 and up), addiitonal bind mounts are needed: 34 | 35 | Path to `/etc/libvirt/qemu/nvram` is required to backup/restore nvram files per VM (which seems to be the same on Qemu implementations tested so far) 36 | 37 | Path to your distro correspondent OVMF files. This is `/usr/share/OVMF` on Debian based, and `/usr/share/qemu/ovmf-x64` on Unraid (feel free to report this path on other distributions) 38 | 39 | - Finally, using identical *host:container* bind mounts for virtual disk locations (as well nvram & ovmf binaries, when applies), is necessary to allow backup/restore commands to find out the files at the expected locations, in concordance with VM definitions at the host side. 40 | 41 | ## Usage Examples 42 | 43 | For detailed info about options, also see [backup](../README.md#backup-examples) and [restoration](../README.md#restoration-examples) examples 44 | 45 | ### Full or incremental backup: 46 | 47 | ``` 48 | docker run --rm \ 49 | -v /run:/run \ 50 | -v /var/tmp:/var/tmp \ 51 | -v /etc/libvirt/qemu/nvram:/etc/libvirt/qemu/nvram \ 52 | -v /usr/share/OVMF:/usr/share/OVMF \ 53 | -v /:/backups \ 54 | ghcr.io/abbbi/virtnbdbackup:master \ 55 | virtnbdbackup -d -l auto -o /backups/ 56 | ``` 57 | 58 | Where `` is an example of the actual master backups folder where VM sub-folders are being stored in your system, and `` the VM name (actual path to disk images is not required.) 59 | 60 | ### Full Backup Restoration to an existing VM: 61 | 62 | ``` 63 | docker run --rm \ 64 | -v /run:/run \ 65 | -v /var/tmp:/var/tmp \ 66 | -v /etc/libvirt/qemu/nvram:/etc/libvirt/qemu/nvram \ 67 | -v /usr/share/OVMF:/usr/share/OVMF \ 68 | -v /mnt/backups:/backups \ 69 | -v /:/ \ 70 | ghcr.io/abbbi/virtnbdbackup:master \ 71 | bash -c \ 72 | "mkdir -p //.old && \ 73 | mv ///* //.old/ && \ 74 | virtnbdrestore -i /backups/ -o //" 75 | ``` 76 | 77 | Where `//` is the actual folder where the specific disk image(s) of the VM to restore, are stored on the host system. In this case, bind mounts should be identical. 78 | 79 | On this example, any existing files are being moved to a folder named `.old`, because restore would fail if it finds the same image(s) that is attempting to restore onto the destination. For instance, you might opt to operate with existing images according your needs, e.g. deleting it before to restore from backup. 80 | 81 | ## Interactive Mode: 82 | 83 | This starts a session inside a (volatile) container, provisioning all bind mounts and allowing to do manual backups and restores, as well testing/troubleshooting: 84 | 85 | ``` 86 | docker run -it --rm \ 87 | -v /run:/run \ 88 | -v /var/tmp:/var/tmp \ 89 | -v /etc/libvirt/qemu/nvram:/etc/libvirt/qemu/nvram \ 90 | -v /usr/share/OVMF:/usr/share/OVMF \ 91 | -v /mnt/backups:/backups \ 92 | -v /:/ \ 93 | ghcr.io/abbbi/virtnbdbackup \ 94 | bash 95 | ``` 96 | -------------------------------------------------------------------------------- /libvirtnbdbackup/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | __version__ = "2.29" 19 | -------------------------------------------------------------------------------- /libvirtnbdbackup/argopt.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import os 19 | from getpass import getuser 20 | from argparse import _ArgumentGroup 21 | from libvirtnbdbackup import __version__ 22 | 23 | 24 | def addRemoteArgs(opt: _ArgumentGroup) -> None: 25 | """Common remote backup arguments""" 26 | 27 | user = getuser() or None 28 | 29 | session = "qemu:///system" 30 | if user != "root": 31 | session = "qemu:///session" 32 | 33 | opt.add_argument( 34 | "-U", 35 | "--uri", 36 | default=session, 37 | required=False, 38 | type=str, 39 | help="Libvirt connection URI. (default: %(default)s)", 40 | ) 41 | opt.add_argument( 42 | "--user", 43 | default=None, 44 | required=False, 45 | type=str, 46 | help="User to authenticate against libvirtd. (default: %(default)s)", 47 | ) 48 | opt.add_argument( 49 | "--ssh-user", 50 | default=user, 51 | required=False, 52 | type=str, 53 | help=( 54 | "User to authenticate against remote sshd: " 55 | "used for remote copy of files. (default: %(default)s)" 56 | ), 57 | ) 58 | opt.add_argument( 59 | "--ssh-port", 60 | default=22, 61 | required=False, 62 | type=int, 63 | help=( 64 | "Port to connect to remote sshd: " 65 | "used for remote copy of files. (default: %(default)s)" 66 | ), 67 | ) 68 | opt.add_argument( 69 | "--password", 70 | default=None, 71 | required=False, 72 | type=str, 73 | help="Password to authenticate against libvirtd. (default: %(default)s)", 74 | ) 75 | opt.add_argument( 76 | "-P", 77 | "--nbd-port", 78 | type=int, 79 | default=10809, 80 | required=False, 81 | help=( 82 | "Port used by remote NBD Service, should be unique for each" 83 | " started backup. (default: %(default)s)" 84 | ), 85 | ) 86 | opt.add_argument( 87 | "-I", 88 | "--nbd-ip", 89 | type=str, 90 | default="", 91 | required=False, 92 | help=( 93 | "IP used to bind remote NBD service on" 94 | " (default: hostname returned by libvirtd)" 95 | ), 96 | ) 97 | opt.add_argument( 98 | "--tls", 99 | action="store_true", 100 | required=False, 101 | help="Enable and use TLS for NBD connection. (default: %(default)s)", 102 | ) 103 | opt.add_argument( 104 | "--tls-cert", 105 | type=str, 106 | default="/etc/pki/qemu/", 107 | required=False, 108 | help=( 109 | "Path to TLS certificates used during offline backup" 110 | " and restore. (default: %(default)s)" 111 | ), 112 | ) 113 | 114 | 115 | def addDebugArgs(opt: _ArgumentGroup) -> None: 116 | """Common debug arguments""" 117 | opt.add_argument( 118 | "-v", 119 | "--verbose", 120 | default=False, 121 | help="Enable debug output", 122 | action="store_true", 123 | ) 124 | opt.add_argument( 125 | "-V", 126 | "--version", 127 | default=False, 128 | help="Show version and exit", 129 | action="version", 130 | version=__version__, 131 | ) 132 | 133 | 134 | def addLogArgs(opt, prog): 135 | """Logging related arguments""" 136 | try: 137 | HOME = os.environ["HOME"] 138 | except KeyError: 139 | HOME = "/tmp" 140 | opt.add_argument( 141 | "-L", 142 | "--logfile", 143 | default=f"{HOME}/{prog}.log", 144 | type=str, 145 | help="Path to Logfile (default: %(default)s)", 146 | ) 147 | 148 | 149 | def addLogColorArgs(opt): 150 | """Option to enable or disable colored output""" 151 | opt.add_argument( 152 | "--nocolor", 153 | default=False, 154 | help="Disable colored output (default: %(default)s)", 155 | action="store_true", 156 | ) 157 | -------------------------------------------------------------------------------- /libvirtnbdbackup/backup/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | __title__ = "backup" 19 | __version__ = "0.1" 20 | -------------------------------------------------------------------------------- /libvirtnbdbackup/backup/check.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2024 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import logging 19 | from typing import List, Any 20 | from argparse import Namespace 21 | from libvirt import virDomain 22 | from libvirtnbdbackup import virt 23 | from libvirtnbdbackup import common as lib 24 | from libvirtnbdbackup import exceptions 25 | 26 | log = logging.getLogger() 27 | 28 | 29 | def arguments(args: Namespace) -> None: 30 | """Check passed arguments for validity""" 31 | if args.compress is not False and args.type == "raw": 32 | raise exceptions.BackupException("Compression not supported with raw output.") 33 | 34 | if args.stdout is True and args.type == "raw": 35 | raise exceptions.BackupException("Output type raw not supported to stdout.") 36 | 37 | if args.stdout is True and args.raw is True: 38 | raise exceptions.BackupException( 39 | "Saving raw images to stdout is not supported." 40 | ) 41 | 42 | if args.type == "raw" and args.level in ("inc", "diff"): 43 | raise exceptions.BackupException( 44 | "Stream format raw does not support incremental or differential backup." 45 | ) 46 | 47 | 48 | def targetDir(args: Namespace) -> None: 49 | """Check if target directory backup is started to meets 50 | all requirements based on the backup level executed""" 51 | if ( 52 | args.level not in ("copy", "full", "auto") 53 | and not lib.hasFullBackup(args) 54 | and not args.stdout 55 | ): 56 | raise exceptions.BackupException( 57 | f"Unable to execute [{args.level}] backup: " 58 | f"No full backup found in target directory: [{args.output}]" 59 | ) 60 | 61 | if lib.targetIsEmpty(args) and args.level == "auto": 62 | log.info("Backup mode auto, target folder is empty: executing full backup.") 63 | args.level = "full" 64 | elif not lib.targetIsEmpty(args) and args.level == "auto": 65 | if not lib.hasFullBackup(args): 66 | raise exceptions.BackupException( 67 | "Can't execute switch to auto incremental backup: " 68 | f"specified target folder [{args.output}] does not contain full backup.", 69 | ) 70 | log.info("Backup mode auto: executing incremental backup.") 71 | args.level = "inc" 72 | elif not args.stdout and not args.startonly and not args.killonly: 73 | if not lib.targetIsEmpty(args): 74 | raise exceptions.BackupException( 75 | "Target directory already contains full or copy backup." 76 | ) 77 | 78 | 79 | def vmstate(args, virtClient: virt.client, domObj: virDomain) -> None: 80 | """Check virtual machine state before executing backup 81 | and based on situation, either fallback to regular copy 82 | backup or attempt to bring VM into paused state""" 83 | if domObj.isActive() == 0: 84 | args.offline = True 85 | if args.start_domain is True: 86 | log.info("Starting domain in paused state") 87 | if virtClient.startDomain(domObj) == 0: 88 | args.offline = False 89 | else: 90 | log.info("Failed to start VM in paused mode.") 91 | 92 | if args.level == "full" and args.offline is True: 93 | log.warning("Domain is offline, resetting backup options.") 94 | args.level = "copy" 95 | log.warning("New Backup level: [%s].", args.level) 96 | args.offline = True 97 | 98 | if args.offline is True and args.startonly is True: 99 | raise exceptions.BackupException( 100 | "Domain is offline: must be active for this function." 101 | ) 102 | 103 | 104 | def vmfeature(virtClient: virt.client, domObj: virDomain) -> None: 105 | """Check if required features are enabled in domain config""" 106 | if virtClient.hasIncrementalEnabled(domObj) is False: 107 | raise exceptions.BackupException( 108 | "Virtual machine does not support required backup features, " 109 | "please adjust virtual machine configuration." 110 | ) 111 | 112 | 113 | def diskformat(args: Namespace, disks: List[Any]) -> None: 114 | """Check if disks meet requirements for backup mode, if not, switch 115 | backup job to type copy.""" 116 | if args.level != "copy" and lib.hasQcowDisks(disks) is False: 117 | args.level = "copy" 118 | raise exceptions.BackupException( 119 | "Only raw disks attached, switching to backup mode copy." 120 | ) 121 | 122 | 123 | def blockjobs( 124 | args, virtClient: virt.client, domObj: virDomain, disks: List[Any] 125 | ) -> None: 126 | """Check if there is an already active backup operation on the domain 127 | disks. If so, fail accordingly""" 128 | if ( 129 | not args.killonly 130 | and not args.offline 131 | and virtClient.blockJobActive(domObj, disks) 132 | ): 133 | raise exceptions.BackupException( 134 | "Active block job for running domain:" 135 | f" Check with [virsh domjobinfo {args.domain}] or use option -k to kill the active job." 136 | ) 137 | -------------------------------------------------------------------------------- /libvirtnbdbackup/backup/job.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import logging 19 | from argparse import Namespace 20 | from typing import List 21 | from libvirt import virDomain 22 | from libvirtnbdbackup import virt 23 | from libvirtnbdbackup.virt.client import DomainDisk 24 | from libvirtnbdbackup.virt.exceptions import startBackupFailed 25 | 26 | 27 | def start( 28 | args: Namespace, 29 | virtClient: virt.client, 30 | domObj: virDomain, 31 | disks: List[DomainDisk], 32 | ) -> bool: 33 | """Start backup job via libvirt API""" 34 | try: 35 | logging.info("Starting backup job.") 36 | virtClient.startBackup( 37 | args, 38 | domObj, 39 | disks, 40 | ) 41 | logging.debug("Backup job started.") 42 | return True 43 | except startBackupFailed as e: 44 | logging.error(e) 45 | 46 | return False 47 | -------------------------------------------------------------------------------- /libvirtnbdbackup/backup/metadata.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import os 19 | import logging 20 | from argparse import Namespace 21 | from typing import List, Union 22 | 23 | from libvirtnbdbackup import output 24 | from libvirtnbdbackup.virt.client import DomainDisk 25 | from libvirtnbdbackup import common as lib 26 | from libvirtnbdbackup.qemu import util as qemu 27 | from libvirtnbdbackup.qemu.exceptions import ProcessError 28 | from libvirtnbdbackup.ssh.exceptions import sshError 29 | from libvirtnbdbackup.output.exceptions import OutputException 30 | 31 | 32 | log = logging.getLogger() 33 | 34 | 35 | def backupChecksum(fileStream, targetFile): 36 | """Save the calculated adler32 checksum, it can be verified 37 | by virtnbdbrestore's verify function.'""" 38 | checksum = fileStream.checksum() 39 | logging.info("Checksum for file: [%s]:[%s]", targetFile, checksum) 40 | chksumfile = f"{targetFile}.chksum" 41 | logging.info("Saving checksum to: [%s]", chksumfile) 42 | with output.openfile(chksumfile, "w") as cf: 43 | cf.write(f"{checksum}") 44 | 45 | 46 | def backupConfig(args: Namespace, vmConfig: str) -> Union[str, None]: 47 | """Save domain XML config file""" 48 | configFile = f"{args.output}/vmconfig.{lib.getIdent(args)}.xml" 49 | log.info("Saving VM config to: [%s]", configFile) 50 | try: 51 | with output.openfile(configFile, "wb") as fh: 52 | fh.write(vmConfig.encode()) 53 | return configFile 54 | except OutputException as e: 55 | log.error("Failed to save VM config: [%s]", e) 56 | return None 57 | 58 | 59 | def backupDiskInfo(args: Namespace, disk: DomainDisk): 60 | """Save information about qcow image, used to reconstruct 61 | the qemu image with the same settings during restore""" 62 | try: 63 | info = qemu.util("").info(disk.path, args.sshClient) 64 | except ( 65 | ProcessError, 66 | sshError, 67 | ) as errmsg: 68 | log.warning("Failed to read qcow image info: [%s]", errmsg) 69 | return 70 | 71 | configFile = f"{args.output}/{disk.target}.{lib.getIdent(args)}.qcow.json" 72 | try: 73 | with output.openfile(configFile, "wb") as fh: 74 | fh.write(info.out.encode()) 75 | log.info("Saved qcow image config to: [%s]", configFile) 76 | if args.stdout is True: 77 | args.diskInfo.append(configFile) 78 | except OutputException as e: 79 | log.warning("Failed to save qcow image config: [%s]", e) 80 | 81 | 82 | def backupBootConfig(args: Namespace) -> None: 83 | """Save domain uefi/nvram/kernel and loader if configured.""" 84 | for setting, val in args.info.items(): 85 | if args.level != "copy": 86 | tFile = f"{args.output}/{os.path.basename(val)}.{lib.getIdent(args)}" 87 | else: 88 | tFile = f"{args.output}/{os.path.basename(val)}" 89 | log.info("Save additional boot config [%s] to: [%s]", setting, tFile) 90 | lib.copy(args, val, tFile) 91 | args.info[setting] = tFile 92 | 93 | 94 | def backupAutoStart(args: Namespace) -> None: 95 | """Save information if virtual machine was marked 96 | for autostart during system boot""" 97 | log.info("Autostart setting configured for virtual machine.") 98 | autoStartFile = f"{args.output}/autostart.{lib.getIdent(args)}" 99 | try: 100 | with output.openfile(autoStartFile, "wb") as fh: 101 | fh.write(b"True") 102 | except OutputException as e: 103 | log.warning("Failed to save autostart information: [%s]", e) 104 | 105 | 106 | def saveFiles( 107 | args: Namespace, 108 | vmConfig: str, 109 | disks: List[DomainDisk], 110 | fileStream: Union[output.target.Directory, output.target.Zip], 111 | logFile: str, 112 | ): 113 | """Save additional files such as virtual machine configuration 114 | and UEFI / kernel images""" 115 | configFile = backupConfig(args, vmConfig) 116 | 117 | backupBootConfig(args) 118 | for disk in disks: 119 | if disk.format.startswith("qcow"): 120 | backupDiskInfo(args, disk) 121 | if args.stdout is True: 122 | addFiles(args, configFile, fileStream, logFile) 123 | 124 | 125 | def addFiles(args: Namespace, configFile: Union[str, None], zipStream, logFile: str): 126 | """Add backup log and other files to zip archive""" 127 | if configFile is not None: 128 | log.info("Adding vm config to zipfile") 129 | zipStream.zipStream.write(configFile, configFile) 130 | if args.level in ("full", "inc"): 131 | log.info("Adding checkpoint info to zipfile") 132 | zipStream.zipStream.write(args.cpt.file, args.cpt.file) 133 | for dirname, _, files in os.walk(args.checkpointdir): 134 | zipStream.zipStream.write(dirname) 135 | for filename in files: 136 | zipStream.zipStream.write(os.path.join(dirname, filename)) 137 | 138 | for setting, val in args.info.items(): 139 | log.info("Adding additional [%s] setting file [%s] to zipfile", setting, val) 140 | zipStream.zipStream.write(val, os.path.basename(val)) 141 | 142 | for diskInfo in args.diskInfo: 143 | log.info("Adding QCOW image format file [%s] to zipfile", diskInfo) 144 | zipStream.zipStream.write(diskInfo, os.path.basename(diskInfo)) 145 | 146 | log.info("Adding backup log [%s] to zipfile", logFile) 147 | zipStream.zipStream.write(logFile, logFile) 148 | -------------------------------------------------------------------------------- /libvirtnbdbackup/backup/partialfile.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import os 19 | import glob 20 | import logging 21 | from argparse import Namespace 22 | 23 | from libvirtnbdbackup import exceptions 24 | 25 | 26 | log = logging.getLogger() 27 | 28 | 29 | def _exists(args: Namespace) -> int: 30 | """Check for possible partial backup files""" 31 | partialFiles = glob.glob(f"{args.output}/*.partial") 32 | return len(partialFiles) > 0 33 | 34 | 35 | def exists(args: Namespace) -> bool: 36 | """Check if target directory has an partial backup, 37 | makes backup utility exit errnous in case backup 38 | type is full or inc""" 39 | if args.level in ("inc", "diff") and args.stdout is False and _exists(args) is True: 40 | log.error("Partial backup found in target directory: [%s]", args.output) 41 | log.error("One of the last backups seems to have failed.") 42 | log.error("Consider re-executing full backup.") 43 | return True 44 | 45 | return False 46 | 47 | 48 | def rename(targetFilePartial: str, targetFile: str) -> None: 49 | """After backup, move .partial file to real 50 | target file""" 51 | try: 52 | os.rename(targetFilePartial, targetFile) 53 | except OSError as e: 54 | raise exceptions.DiskBackupFailed(f"Failed to rename file: [{e}]") from e 55 | -------------------------------------------------------------------------------- /libvirtnbdbackup/backup/server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import logging 19 | from argparse import Namespace 20 | from typing import Union 21 | from libvirtnbdbackup import nbdcli 22 | from libvirtnbdbackup import virt 23 | from libvirtnbdbackup.virt.client import DomainDisk 24 | from libvirtnbdbackup.qemu import util as qemu 25 | from libvirtnbdbackup.objects import processInfo 26 | from libvirtnbdbackup.nbdcli.exceptions import NbdClientException 27 | from libvirtnbdbackup.exceptions import DiskBackupFailed 28 | 29 | 30 | def setup(args: Namespace, disk: DomainDisk, remoteHost: str, port: int) -> processInfo: 31 | """Start background qemu-nbd process used during backup 32 | if domain is offline, in case of remote backup, initiate 33 | ssh session and start process on remote system.""" 34 | bitMap: str = "" 35 | if args.level in ("inc", "diff"): 36 | bitMap = args.cpt.name 37 | socket = f"{args.socketfile}.{disk.target}" 38 | if remoteHost != "": 39 | logging.info( 40 | "Offline backup, starting remote NBD server, socket: [%s:%s], port: [%s]", 41 | remoteHost, 42 | socket, 43 | port, 44 | ) 45 | nbdProc = qemu.util(disk.target).startRemoteBackupNbdServer( 46 | args, disk, bitMap, port 47 | ) 48 | logging.info("Remote NBD server started, PID: [%s].", nbdProc.pid) 49 | return nbdProc 50 | 51 | logging.info("Offline backup, starting local NBD server, socket: [%s]", socket) 52 | nbdProc = qemu.util(disk.target).startBackupNbdServer( 53 | disk.format, disk.path, socket, bitMap 54 | ) 55 | logging.info("Local NBD Service started, PID: [%s]", nbdProc.pid) 56 | return nbdProc 57 | 58 | 59 | def connect( # pylint: disable=too-many-arguments 60 | args: Namespace, 61 | disk: DomainDisk, 62 | metaContext: str, 63 | remoteIP: str, 64 | port: int, 65 | virtClient: virt.client, 66 | ): 67 | """Connect to started nbd endpoint""" 68 | socket = args.socketfile 69 | if args.offline is True: 70 | socket = f"{args.socketfile}.{disk.target}" 71 | 72 | cType: Union[nbdcli.TCP, nbdcli.Unix] 73 | if virtClient.remoteHost != "": 74 | cType = nbdcli.TCP(disk.target, metaContext, remoteIP, args.tls, port) 75 | else: 76 | cType = nbdcli.Unix(disk.target, metaContext, socket) 77 | 78 | nbdClient = nbdcli.client(cType, args.no_sparse_detection) 79 | 80 | try: 81 | return nbdClient.connect() 82 | except NbdClientException as e: 83 | raise DiskBackupFailed( 84 | f"NBD endpoint: [{cType}]: connection failed: [{e}]" 85 | ) from e 86 | -------------------------------------------------------------------------------- /libvirtnbdbackup/backup/target.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import logging 19 | from typing import BinaryIO 20 | from argparse import Namespace 21 | from libvirtnbdbackup.virt.client import DomainDisk 22 | from libvirtnbdbackup.common import getIdent 23 | 24 | 25 | def get( 26 | args: Namespace, fileStream, targetFile: str, targetFilePartial: str 27 | ) -> BinaryIO: 28 | """Open target file based on output writer""" 29 | if args.stdout is True: 30 | logging.info("Writing data to zip archive.") 31 | fileStream.open(targetFile) 32 | else: 33 | logging.info("Write data to target file: [%s].", targetFilePartial) 34 | fileStream.open(targetFilePartial) 35 | 36 | return fileStream 37 | 38 | 39 | def Set(args: Namespace, disk: DomainDisk, ext: str = "data"): 40 | """Set Target file name to write data to, used for both data files 41 | and qemu disk info""" 42 | targetFile: str = "" 43 | if args.level in ("full", "copy"): 44 | level = args.level 45 | if disk.format == "raw": 46 | level = "copy" 47 | targetFile = f"{args.output}/{disk.target}.{level}.{ext}" 48 | elif args.level in ("inc", "diff"): 49 | cptName = getIdent(args) 50 | targetFile = f"{args.output}/{disk.target}.{args.level}.{cptName}.{ext}" 51 | 52 | targetFilePartial = f"{targetFile}.partial" 53 | 54 | return targetFile, targetFilePartial 55 | -------------------------------------------------------------------------------- /libvirtnbdbackup/block.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | from typing import Generator, IO, Any, Union 19 | from nbd import Error as nbdError 20 | from libvirtnbdbackup import lz4 21 | from libvirtnbdbackup.exceptions import BackupException 22 | 23 | 24 | def step(offset: int, length: int, maxRequestSize: int) -> Generator: 25 | """Process block and ensure to not exceed the maximum request size 26 | from NBD server. 27 | 28 | If length parameter is dict, compression was enabled during 29 | backup, thus we cannot use the offsets and sizes for the 30 | original data, but must use the compressed offsets and sizes 31 | to read the correct lz4 frames from the stream. 32 | """ 33 | blockOffset = offset 34 | if isinstance(length, dict): 35 | blockOffset = offset 36 | compressOffset = list(length.keys())[0] 37 | for part in length[compressOffset]: 38 | blockOffset += part 39 | yield part, blockOffset 40 | else: 41 | blockOffset = offset 42 | while blockOffset < offset + length: 43 | blocklen = min(offset + length - blockOffset, maxRequestSize) 44 | yield blocklen, blockOffset 45 | blockOffset += blocklen 46 | 47 | 48 | def write( 49 | writer: IO[Any], block, nbdCon, btype: str, compress: Union[bool, int] 50 | ) -> int: 51 | """Write single block that does not exceed nbd maxRequestSize 52 | setting. In case compression is enabled, single blocks are 53 | compressed using lz4. 54 | """ 55 | if btype == "raw": 56 | writer.seek(block.offset) 57 | 58 | try: 59 | data = nbdCon.nbd.pread(block.length, block.offset) 60 | except nbdError as e: 61 | raise BackupException(e) from e 62 | 63 | if compress is not False: 64 | data = lz4.compressFrame(data, compress) 65 | 66 | return writer.write(data) 67 | -------------------------------------------------------------------------------- /libvirtnbdbackup/chunk.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | from typing import List, Any, Tuple, IO, Union 19 | from nbd import Error as nbdError 20 | from libvirtnbdbackup import block 21 | from libvirtnbdbackup import lz4 22 | from libvirtnbdbackup.exceptions import DiskBackupFailed 23 | 24 | # pylint: disable=too-many-arguments 25 | 26 | 27 | def write( 28 | writer: IO[Any], blk, nbdCon, btype: str, compress: Union[bool, int], pbar 29 | ) -> Tuple[int, List[int]]: 30 | """During extent processing, consecutive blocks with 31 | the same type(data or zeroed) are unified into one big chunk. 32 | This helps to reduce requests to the NBD Server. 33 | 34 | But in cases where the block to be saved exceeds the maximum 35 | recommended request size (nbdClient.maxRequestSize), we 36 | need to split one big request into multiple not exceeding 37 | the limit 38 | 39 | If compression is enabled, function returns a list of 40 | offsets for the compressed frames, which is appended 41 | to the end of the stream. 42 | """ 43 | wSize = 0 44 | cSizes = [] 45 | for blocklen, blockOffset in block.step( 46 | blk.offset, blk.length, nbdCon.maxRequestSize 47 | ): 48 | if btype == "raw": 49 | writer.seek(blockOffset) 50 | 51 | try: 52 | data = nbdCon.nbd.pread(blocklen, blockOffset) 53 | except nbdError as e: 54 | raise DiskBackupFailed(e) from e 55 | 56 | if compress is not False: 57 | compressed = lz4.compressFrame(data, compress) 58 | wSize += writer.write(compressed) 59 | cSizes.append(len(compressed)) 60 | else: 61 | wSize += writer.write(data) 62 | 63 | pbar.update(blocklen) 64 | 65 | return wSize, cSizes 66 | 67 | 68 | def read( 69 | reader: IO[Any], 70 | offset: int, 71 | length: int, 72 | nbdCon, 73 | compression: bool, 74 | pbar, 75 | ) -> int: 76 | """Read data from reader and write to nbd connection 77 | 78 | If Compression is enabled function receives length information 79 | as dict, which contains the stream offsets for the compressed 80 | lz4 frames. 81 | 82 | Frames are read from the stream at the compressed size information 83 | (offset in the stream). 84 | 85 | After decompression, data is written back to original offset 86 | in the virtual machine disk image. 87 | 88 | If no compression is enabled, data is read from the regular 89 | data header at its position and written to nbd target 90 | directly. 91 | """ 92 | wSize = 0 93 | for blocklen, blockOffset in block.step(offset, length, nbdCon.maxRequestSize): 94 | if compression is True: 95 | data = lz4.decompressFrame(reader.read(blocklen)) 96 | nbdCon.nbd.pwrite(data, offset) 97 | offset += len(data) 98 | wSize += len(data) 99 | else: 100 | data = reader.read(blocklen) 101 | nbdCon.nbd.pwrite(data, blockOffset) 102 | wSize += len(data) 103 | 104 | pbar.update(blocklen) 105 | 106 | return wSize 107 | -------------------------------------------------------------------------------- /libvirtnbdbackup/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exceptions 3 | """ 4 | 5 | 6 | class CheckpointException(Exception): 7 | """Base checkpoint Exception""" 8 | 9 | 10 | class NoCheckpointsFound(CheckpointException): 11 | """Inc or differential backup attempted but 12 | no existing checkpoints are found.""" 13 | 14 | 15 | class RedefineCheckpointError(CheckpointException): 16 | """During redefining existing checkpoints after 17 | vm relocate, an error occurred""" 18 | 19 | 20 | class ReadCheckpointsError(CheckpointException): 21 | """Can't read checkpoint file""" 22 | 23 | 24 | class RemoveCheckpointError(CheckpointException): 25 | """During removal of existing checkpoints after 26 | an error occurred""" 27 | 28 | 29 | class SaveCheckpointError(CheckpointException): 30 | """Unable to append checkpoint to checkpoint 31 | file""" 32 | 33 | 34 | class ForeignCeckpointError(CheckpointException): 35 | """Checkpoint for vm found which was not created 36 | by virtnbdbackup""" 37 | 38 | 39 | class BackupException(Exception): 40 | """Base backup Exception""" 41 | 42 | 43 | class DiskBackupFailed(BackupException): 44 | """Backup of one disk failed""" 45 | 46 | 47 | class DiskBackupWriterException(BackupException): 48 | """Opening the target file writer 49 | failed""" 50 | 51 | 52 | class RestoreException(Exception): 53 | """Base restore Exception""" 54 | 55 | 56 | class UntilCheckpointReached(RestoreException): 57 | """Base restore Exception""" 58 | 59 | 60 | class RestoreError(RestoreException): 61 | """Base restore error Exception""" 62 | -------------------------------------------------------------------------------- /libvirtnbdbackup/extenthandler/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | __title__ = "extenthandler" 19 | __version__ = "0.1" 20 | 21 | from .extenthandler import ExtentHandler 22 | -------------------------------------------------------------------------------- /libvirtnbdbackup/logcount.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import logging 19 | 20 | 21 | class logCount(logging.Handler): 22 | """Custom log handler keeping track of issued log messages""" 23 | 24 | class LogType: 25 | """Log message type""" 26 | 27 | def __init__(self) -> None: 28 | self.warnings = 0 29 | self.errors = 0 30 | 31 | def __init__(self) -> None: 32 | super().__init__() 33 | self.count = self.LogType() 34 | 35 | def emit(self, record: logging.LogRecord) -> None: 36 | if record.levelno == logging.WARNING: 37 | self.count.warnings += 1 38 | if record.levelno in (logging.ERROR, logging.CRITICAL): 39 | self.count.errors += 1 40 | -------------------------------------------------------------------------------- /libvirtnbdbackup/lz4.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import logging 19 | import lz4.frame 20 | 21 | log = logging.getLogger() 22 | 23 | 24 | def decompressFrame(data: bytes) -> bytes: 25 | """Decompress lz4 frame, print frame information""" 26 | frameInfo = lz4.frame.get_frame_info(data) 27 | log.debug("Compressed Frame: %s", frameInfo) 28 | return lz4.frame.decompress(data) 29 | 30 | 31 | def compressFrame(data: bytes, level: int) -> bytes: 32 | """Compress block with to lz4 frame, checksums 33 | enabled for safety 34 | """ 35 | return lz4.frame.compress( 36 | data, 37 | content_checksum=True, 38 | block_checksum=True, 39 | compression_level=level, 40 | ) 41 | -------------------------------------------------------------------------------- /libvirtnbdbackup/map/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | __title__ = "map" 19 | __version__ = "0.1" 20 | -------------------------------------------------------------------------------- /libvirtnbdbackup/map/changes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import os 19 | import logging 20 | from argparse import Namespace 21 | from typing import List 22 | from libvirtnbdbackup import common as lib 23 | from libvirtnbdbackup import output 24 | 25 | 26 | def replay(dataRanges: List, args: Namespace) -> None: 27 | """Replay the changes from an incremental or differential 28 | backup file to the NBD device""" 29 | logging.info("Replaying changes from incremental backups") 30 | blockListInc = list( 31 | filter( 32 | lambda x: x["inc"] is True, 33 | dataRanges, 34 | ) 35 | ) 36 | dataSize = sum(extent["length"] for extent in blockListInc) 37 | progressBar = lib.progressBar(dataSize, "replaying..", args) 38 | with output.openfile(args.device, "wb") as replayDevice: 39 | for extent in blockListInc: 40 | if args.noprogress: 41 | logging.info( 42 | "Replaying offset %s from %s", extent["offset"], extent["file"] 43 | ) 44 | with output.openfile(os.path.abspath(extent["file"]), "rb") as replaySrc: 45 | replaySrc.seek(extent["offset"]) 46 | replayDevice.seek(extent["originalOffset"]) 47 | replayDevice.write(replaySrc.read(extent["length"])) 48 | replayDevice.seek(0) 49 | replayDevice.flush() 50 | progressBar.update(extent["length"]) 51 | 52 | progressBar.close() 53 | -------------------------------------------------------------------------------- /libvirtnbdbackup/map/ranges.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import os 19 | import logging 20 | import json 21 | from typing import List, Dict, Tuple, IO 22 | from libvirtnbdbackup import common as lib 23 | from libvirtnbdbackup import output 24 | from libvirtnbdbackup.output.exceptions import OutputException 25 | from libvirtnbdbackup.exceptions import RestoreError 26 | from libvirtnbdbackup.sparsestream.exceptions import StreamFormatException 27 | 28 | 29 | def _parse(stream, sTypes, reader) -> Tuple[List, Dict]: 30 | """Read block offsets from backup stream image""" 31 | try: 32 | kind, start, length = stream.readFrame(reader) 33 | meta = stream.loadMetadata(reader.read(length)) 34 | except StreamFormatException as errmsg: 35 | logging.error("Unable to read metadata header: %s", errmsg) 36 | raise RestoreError from errmsg 37 | 38 | if lib.isCompressed(meta): 39 | logging.error("Mapping compressed images currently not supported.") 40 | raise RestoreError 41 | 42 | assert reader.read(len(sTypes.TERM)) == sTypes.TERM 43 | 44 | dataRanges: List = [] 45 | count: int = 0 46 | while True: 47 | kind, start, length = stream.readFrame(reader) 48 | if kind == sTypes.STOP: 49 | dataRanges[-1]["nextBlockOffset"] = None 50 | break 51 | 52 | blockInfo = {} 53 | blockInfo["count"] = count 54 | blockInfo["offset"] = reader.tell() 55 | blockInfo["originalOffset"] = start 56 | blockInfo["nextOriginalOffset"] = start + length 57 | blockInfo["length"] = length 58 | blockInfo["data"] = kind == sTypes.DATA 59 | blockInfo["file"] = os.path.abspath(reader.name) 60 | blockInfo["inc"] = meta["incremental"] 61 | 62 | if kind == sTypes.DATA: 63 | reader.seek(length, os.SEEK_CUR) 64 | assert reader.read(len(sTypes.TERM)) == sTypes.TERM 65 | 66 | nextBlockOffset = reader.tell() + sTypes.FRAME_LEN 67 | blockInfo["nextBlockOffset"] = nextBlockOffset 68 | dataRanges.append(blockInfo) 69 | count += 1 70 | 71 | return dataRanges, meta 72 | 73 | 74 | def get(args, stream, sTypes, dataFiles: List) -> List: 75 | """Get data ranges for each file specified""" 76 | dataRanges = [] 77 | for dFile in dataFiles: 78 | try: 79 | reader = output.openfile(dFile, "rb") 80 | except OutputException as e: 81 | logging.error("[%s]: [%s]", dFile, e) 82 | raise RestoreError from e 83 | 84 | Range, meta = _parse(stream, sTypes, reader) 85 | if Range is False or meta is False: 86 | logging.error("Unable to read meta header from backup file.") 87 | raise RestoreError("Invalid header") 88 | dataRanges.extend(Range) 89 | 90 | if args.verbose is True: 91 | logging.info(json.dumps(dataRanges, indent=4)) 92 | else: 93 | logging.info( 94 | "Parsed [%s] block offsets from file [%s]", len(dataRanges), dFile 95 | ) 96 | reader.close() 97 | 98 | return dataRanges 99 | 100 | 101 | def dump(tfile: IO, dataRanges: List) -> bool: 102 | """Dump block map to temporary file""" 103 | try: 104 | tfile.write(json.dumps(dataRanges, indent=4).encode()) 105 | return True 106 | except OSError as e: 107 | logging.error("Unable to write blockmap file: %s", e) 108 | return False 109 | -------------------------------------------------------------------------------- /libvirtnbdbackup/map/requirements.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import os 19 | import sys 20 | import shutil 21 | import logging 22 | from argparse import Namespace 23 | from libvirtnbdbackup import common as lib 24 | 25 | 26 | def executables() -> None: 27 | """Check if required utils are installed""" 28 | for exe in ("nbdkit", "qemu-nbd"): 29 | if not shutil.which(exe): 30 | logging.error("Please install required [%s] utility.", exe) 31 | 32 | 33 | def device(args: Namespace) -> None: 34 | """Check if /dev/nbdX exists, otherwise it is likely 35 | nbd module isn't loaded on the system""" 36 | if not args.device.startswith("/dev/nbd"): 37 | logging.error("Target device [%s] seems not to be an NBD device?", args.device) 38 | 39 | if not lib.exists(args, args.device): 40 | logging.error( 41 | "Target device [%s] does not exist, please load nbd module: [modprobe nbd]", 42 | args.device, 43 | ) 44 | 45 | 46 | def plugin(args: Namespace) -> str: 47 | """Attempt to locate the nbdkit plugin that is passed to the 48 | nbdkit process""" 49 | pluginFileName = "virtnbd-nbdkit-plugin" 50 | installDir = os.path.dirname(sys.argv[0]) 51 | nbdkitModule = f"{installDir}/{pluginFileName}" 52 | 53 | if not lib.exists(args, nbdkitModule): 54 | logging.error("Failed to locate nbdkit plugin: [%s]", pluginFileName) 55 | 56 | return nbdkitModule 57 | -------------------------------------------------------------------------------- /libvirtnbdbackup/nbdcli/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | __title__ = "nbdcli" 19 | __version__ = "0.1" 20 | 21 | from libvirtnbdbackup.objects import Unix, TCP 22 | from .client import client 23 | from . import context 24 | -------------------------------------------------------------------------------- /libvirtnbdbackup/nbdcli/client.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import os 19 | import logging 20 | from time import sleep 21 | import nbd 22 | from libvirtnbdbackup.nbdcli import exceptions 23 | 24 | log = logging.getLogger("nbd") 25 | 26 | 27 | # pylint: disable=too-many-instance-attributes 28 | class client: 29 | """Helper functions for NBD""" 30 | 31 | def __init__(self, cType, no_sparse_detection: bool): 32 | """ 33 | Connect NBD backend 34 | """ 35 | self.cType = cType 36 | self._exportName = cType.exportName 37 | self._metaContext = "" 38 | if cType.metaContext != "": 39 | self._metaContext = cType.metaContext 40 | else: 41 | self._metaContext = nbd.CONTEXT_BASE_ALLOCATION 42 | self.maxRequestSize = 33554432 43 | self.minRequestSize = 65536 44 | self.no_sparse_detection = no_sparse_detection 45 | self.nbd = nbd.NBD() 46 | 47 | def debug(func, args): 48 | """Write NBD debugging messages to logfile instead of 49 | stderr""" 50 | log.debug("%s: %s", func, args) 51 | 52 | self.nbd.set_debug_callback(debug) 53 | self.connection = None 54 | 55 | def _getBlockInfo(self) -> None: 56 | """Read maximum request/block size as advertised by the nbd 57 | server. This is the value which will then be used by default 58 | """ 59 | maxSize = self.nbd.get_block_size(nbd.SIZE_MAXIMUM) 60 | if maxSize != 0: 61 | self.maxRequestSize = maxSize 62 | 63 | log.debug("Block size supported by NBD server: [%s]", maxSize) 64 | 65 | def _connect(self) -> nbd.NBD: 66 | """Setup connection to NBD server endpoint, return 67 | connection handle 68 | """ 69 | if self.cType.tls and not self.nbd.supports_tls(): 70 | raise exceptions.NbdConnectionError( 71 | "Installed python nbd binding is missing required tls features." 72 | ) 73 | 74 | try: 75 | if self.cType.tls: 76 | self.nbd.set_tls(nbd.TLS_REQUIRE) 77 | if self.no_sparse_detection is False: 78 | self.nbd.add_meta_context(nbd.CONTEXT_BASE_ALLOCATION) 79 | if self._metaContext != "": 80 | log.debug( 81 | "Adding meta context to NBD connection: [%s]", self._metaContext 82 | ) 83 | self.nbd.add_meta_context(self._metaContext) 84 | self.nbd.set_export_name(self._exportName) 85 | self.nbd.connect_uri(self.cType.uri) 86 | except nbd.Error as e: 87 | raise exceptions.NbdConnectionError(f"Unable to connect nbd server: {e}") 88 | 89 | self._getBlockInfo() 90 | 91 | return self.nbd 92 | 93 | def connect(self) -> nbd.NBD: 94 | """Wait until NBD endpoint connection can be established. It can take 95 | some time until qemu-nbd process is running and reachable. Attempt to 96 | connect and fail if no connection can be established. In case of unix 97 | domain socket, wait until socket file is created by qemu-nbd.""" 98 | log.info("Waiting until NBD server at [%s] is up.", self.cType.uri) 99 | retry = 0 100 | maxRetry = 20 101 | sleepTime = 1 102 | while True: 103 | sleep(sleepTime) 104 | if retry >= maxRetry: 105 | raise exceptions.NbdConnectionTimeout( 106 | "Timeout during connection to NBD server backend." 107 | ) 108 | 109 | if self.cType.backupSocket and not os.path.exists(self.cType.backupSocket): 110 | log.info("Waiting for NBD Server unix socket, Retry: %s", retry) 111 | retry = retry + 1 112 | continue 113 | 114 | try: 115 | connection = self._connect() 116 | except exceptions.NbdConnectionError as e: 117 | self.nbd = nbd.NBD() 118 | log.info("Waiting for NBD Server connection, Retry: %s [%s]", retry, e) 119 | retry = retry + 1 120 | continue 121 | 122 | log.info("Connection to NBD backend succeeded.") 123 | self.connection = connection 124 | return self 125 | 126 | def disconnect(self) -> None: 127 | """Close nbd connection handle""" 128 | self.nbd.shutdown() 129 | -------------------------------------------------------------------------------- /libvirtnbdbackup/nbdcli/context.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import logging 19 | from argparse import Namespace 20 | from libvirtnbdbackup.virt.client import DomainDisk 21 | 22 | log = logging.getLogger("nbdctx") 23 | 24 | 25 | def get(args: Namespace, disk: DomainDisk) -> str: 26 | """Get required meta context string passed to nbd server based on 27 | backup type""" 28 | metaContext = "" 29 | if args.level not in ("inc", "diff"): 30 | return metaContext 31 | 32 | if args.offline is True: 33 | metaContext = f"qemu:dirty-bitmap:{args.cpt.name}" 34 | else: 35 | metaContext = f"qemu:dirty-bitmap:backup-{disk.target}" 36 | 37 | logging.debug("Using NBD meta context [%s]", metaContext) 38 | 39 | return metaContext 40 | -------------------------------------------------------------------------------- /libvirtnbdbackup/nbdcli/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exceptions 3 | """ 4 | 5 | 6 | class NbdClientException(Exception): 7 | """Nbd exceptions""" 8 | 9 | 10 | class NbdConnectionError(NbdClientException): 11 | """Connection failed""" 12 | 13 | 14 | class NbdConnectionTimeout(NbdClientException): 15 | """Connection timed out""" 16 | -------------------------------------------------------------------------------- /libvirtnbdbackup/objects.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import ipaddress 19 | from dataclasses import dataclass 20 | 21 | 22 | @dataclass 23 | class processInfo: 24 | """Process info object returned by functions calling 25 | various qemu commands 26 | """ 27 | 28 | pid: int 29 | logFile: str 30 | err: str 31 | out: str 32 | pidFile: str 33 | 34 | 35 | @dataclass 36 | class DomainDisk: 37 | """Domain disk object holding information about the disk 38 | attached to a virtual machine""" 39 | 40 | target: str 41 | format: str 42 | filename: str 43 | path: str 44 | backingstores: list 45 | discardOption: str 46 | 47 | 48 | @dataclass 49 | class nbdConn: 50 | """NBD connection""" 51 | 52 | exportName: str 53 | metaContext: str 54 | 55 | 56 | @dataclass 57 | class Unix(nbdConn): 58 | """NBD connection type unix for connection via socket file""" 59 | 60 | backupSocket: str 61 | tls: bool = False 62 | 63 | def __post_init__(self): 64 | self.uri = f"nbd+unix:///{self.exportName}?socket={self.backupSocket}" 65 | 66 | 67 | @dataclass 68 | class TCP(nbdConn): 69 | """NBD connection type tcp for remote backup""" 70 | 71 | hostname: str 72 | tls: bool 73 | port: int = 10809 74 | backupSocket: str = "" 75 | uri_prefix = "nbd://" 76 | 77 | def __post_init__(self): 78 | if self.tls: 79 | self.uri_prefix = "nbds://" 80 | 81 | try: 82 | ip = ipaddress.ip_address(self.hostname) 83 | if ip.version == 6: 84 | self.hostname = f"[{self.hostname}]" 85 | except ValueError: 86 | pass 87 | 88 | self.uri = f"{self.uri_prefix}{self.hostname}:{self.port}/{self.exportName}" 89 | 90 | 91 | @dataclass 92 | class Extent: 93 | """Extent description containing information if block contains 94 | data, offset and length of data to be read/written""" 95 | 96 | context: str 97 | data: bool 98 | offset: int 99 | length: int 100 | 101 | 102 | @dataclass 103 | class _ExtentObj: 104 | """Single Extent object as returned from the NBD server""" 105 | 106 | context: str 107 | length: int 108 | type: int 109 | -------------------------------------------------------------------------------- /libvirtnbdbackup/output/__init__.py: -------------------------------------------------------------------------------- 1 | """Output helper class""" 2 | 3 | __title__ = "output" 4 | __version__ = "0.1" 5 | 6 | from .target import target 7 | 8 | openfile = target.Directory().open 9 | -------------------------------------------------------------------------------- /libvirtnbdbackup/output/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exceptions 3 | """ 4 | 5 | 6 | class OutputException(Exception): 7 | """Outpuhelper exceptions""" 8 | 9 | 10 | class OutputOpenException(OutputException): 11 | """File open failed""" 12 | 13 | 14 | class OutputCreateDirectory(OutputException): 15 | """Can't create output directory""" 16 | -------------------------------------------------------------------------------- /libvirtnbdbackup/output/stream.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | from argparse import Namespace 19 | from typing import Union 20 | from libvirtnbdbackup import output 21 | 22 | 23 | def get( 24 | args: Namespace, repository: output.target 25 | ) -> Union[output.target.Directory, output.target.Zip]: 26 | """Get filehandle for output files based on output 27 | mode""" 28 | fileStream: Union[output.target.Directory, output.target.Zip] 29 | if args.stdout is False: 30 | fileStream = repository.Directory() 31 | else: 32 | fileStream = repository.Zip() 33 | args.output = "./" 34 | args.worker = 1 35 | 36 | return fileStream 37 | -------------------------------------------------------------------------------- /libvirtnbdbackup/output/target.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import os 19 | import sys 20 | import zlib 21 | import zipfile 22 | import logging 23 | import time 24 | import builtins 25 | from typing import IO, Union, Tuple, Any 26 | from libvirtnbdbackup.output import exceptions 27 | 28 | if sys.version_info >= (3, 8): 29 | from typing import Literal 30 | else: 31 | from typing_extensions import Literal 32 | 33 | log = logging.getLogger("output") 34 | 35 | 36 | class target: 37 | """Directs output stream to either regular directory or 38 | zipfile. If other formats are added class should be 39 | used as generic wrapper for open()/write()/close() functions. 40 | """ 41 | 42 | class Directory: 43 | """Backup to target directory""" 44 | 45 | def __init__(self) -> None: 46 | self.fileHandle: IO[Any] 47 | self.chksum: int = 1 48 | 49 | def create(self, targetDir) -> None: 50 | """Create wrapper""" 51 | log.debug("Create: %s", targetDir) 52 | if os.path.exists(targetDir): 53 | if not os.path.isdir(targetDir): 54 | raise exceptions.OutputCreateDirectory( 55 | "Specified target is a file, not a directory" 56 | ) 57 | if not os.path.exists(targetDir): 58 | try: 59 | os.makedirs(targetDir) 60 | except OSError as e: 61 | raise exceptions.OutputCreateDirectory( 62 | f"Failed to create target directory: [{e}]" 63 | ) 64 | 65 | def open( 66 | self, 67 | targetFile: str, 68 | mode: Union[ 69 | Literal["w"], Literal["wb"], Literal["rb"], Literal["r"] 70 | ] = "wb", 71 | ) -> IO[Any]: 72 | """Open target file""" 73 | try: 74 | # pylint: disable=unspecified-encoding,consider-using-with 75 | self.fileHandle = builtins.open(targetFile, mode) 76 | return self.fileHandle 77 | except OSError as e: 78 | raise exceptions.OutputOpenException( 79 | f"Opening target file [{targetFile}] failed: {e}" 80 | ) from e 81 | 82 | def write(self, data: bytes) -> int: 83 | """Write wrapper""" 84 | self.chksum = zlib.adler32(data, self.chksum) 85 | written = self.fileHandle.write(data) 86 | assert written == len(data) 87 | return written 88 | 89 | def read(self, size=-1) -> int: 90 | """Read wrapper""" 91 | return self.fileHandle.read(size) 92 | 93 | def flush(self) -> None: 94 | """Flush wrapper""" 95 | return self.fileHandle.flush() 96 | 97 | def truncate(self, size: int) -> None: 98 | """Truncate target file""" 99 | try: 100 | self.fileHandle.truncate(size) 101 | self.fileHandle.seek(0) 102 | except OSError as e: 103 | raise exceptions.OutputException( 104 | f"Failed to truncate target file: [{e}]" 105 | ) from e 106 | 107 | def close(self) -> None: 108 | """Close wrapper""" 109 | log.debug("Close file") 110 | self.fileHandle.close() 111 | 112 | def seek(self, tgt: int, whence: int = 0) -> int: 113 | """Seek wrapper""" 114 | return self.fileHandle.seek(tgt, whence) 115 | 116 | def checksum(self) -> int: 117 | """Return computed checksum""" 118 | cur = self.chksum 119 | self.chksum = 1 120 | return cur 121 | 122 | class Zip: 123 | """Backup to zip file""" 124 | 125 | def __init__(self) -> None: 126 | self.zipStream: zipfile.ZipFile 127 | self.zipFileStream: IO[bytes] 128 | 129 | log.info("Writing zip file stream to stdout") 130 | try: 131 | # pylint: disable=consider-using-with 132 | self.zipStream = zipfile.ZipFile( 133 | sys.stdout.buffer, "x", zipfile.ZIP_STORED 134 | ) 135 | except zipfile.error as e: 136 | raise exceptions.OutputOpenException( 137 | f"Failed to open zip file: {e}" 138 | ) from e 139 | 140 | def create(self, targetDir) -> None: 141 | """Create wrapper""" 142 | log.debug("Create: %s", targetDir) 143 | target.Directory().create(targetDir) 144 | 145 | def open(self, fileName: str, mode: Literal["w"] = "w") -> IO[bytes]: 146 | """Open wrapper""" 147 | zipFile = zipfile.ZipInfo( 148 | filename=os.path.basename(fileName), 149 | ) 150 | dateTime: time.struct_time = time.localtime(time.time()) 151 | timeStamp: Tuple[int, int, int, int, int, int] = ( 152 | dateTime.tm_year, 153 | dateTime.tm_mon, 154 | dateTime.tm_mday, 155 | dateTime.tm_hour, 156 | dateTime.tm_min, 157 | dateTime.tm_sec, 158 | ) 159 | zipFile.date_time = timeStamp 160 | zipFile.compress_type = zipfile.ZIP_STORED 161 | 162 | try: 163 | # pylint: disable=consider-using-with 164 | self.zipFileStream = self.zipStream.open( 165 | zipFile, mode, force_zip64=True 166 | ) 167 | return self.zipFileStream 168 | except zipfile.error as e: 169 | raise exceptions.OutputOpenException( 170 | f"Failed to open zip stream: {e}" 171 | ) from e 172 | 173 | return False 174 | 175 | def truncate(self, size: int) -> None: 176 | """Truncate target file""" 177 | raise RuntimeError("Not implemented") 178 | 179 | def write(self, data: bytes) -> int: 180 | """Write wrapper""" 181 | return self.zipFileStream.write(data) 182 | 183 | def close(self) -> None: 184 | """Close wrapper""" 185 | log.debug("Close file") 186 | self.zipFileStream.close() 187 | 188 | def checksum(self) -> None: 189 | """Checksum: not implemented for zip file""" 190 | return 191 | -------------------------------------------------------------------------------- /libvirtnbdbackup/qemu/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | __title__ = "qemu" 19 | __version__ = "0.2" 20 | -------------------------------------------------------------------------------- /libvirtnbdbackup/qemu/command.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import logging 19 | import tempfile 20 | import subprocess 21 | from typing import List, Tuple, Union 22 | 23 | from libvirtnbdbackup.qemu.exceptions import ( 24 | ProcessError, 25 | ) 26 | from libvirtnbdbackup.output import openfile 27 | from libvirtnbdbackup.objects import processInfo 28 | 29 | log = logging.getLogger(__name__) 30 | 31 | 32 | def _readlog(logFile: str, cmd: str) -> str: 33 | try: 34 | with openfile(logFile, "rb") as fh: 35 | return fh.read().decode().strip() 36 | except Exception as errmsg: 37 | log.exception(errmsg) 38 | raise ProcessError( 39 | f"Failed to execute [{cmd}]: Unable to get error message: {errmsg}" 40 | ) from errmsg 41 | 42 | 43 | def _readpipe(p) -> Tuple[str, str]: 44 | out = p.stdout.read().decode().strip() 45 | err = p.stderr.read().decode().strip() 46 | return out, err 47 | 48 | 49 | def run(cmdLine: List[str], pidFile: str = "", toPipe: bool = False) -> processInfo: 50 | """Execute passed command""" 51 | logFileName: str = "" 52 | logFile: Union[int, tempfile._TemporaryFileWrapper] 53 | if toPipe is True: 54 | logFile = subprocess.PIPE 55 | else: 56 | # pylint: disable=consider-using-with 57 | logFile = tempfile.NamedTemporaryFile( 58 | delete=False, prefix=cmdLine[0], suffix=".log" 59 | ) 60 | logFileName = logFile.name 61 | 62 | log.debug("CMD: %s", " ".join(cmdLine)) 63 | try: 64 | with subprocess.Popen( 65 | cmdLine, 66 | close_fds=True, 67 | stderr=logFile, 68 | stdout=logFile, 69 | ) as p: 70 | p.wait() 71 | log.debug("Return code: %s", p.returncode) 72 | err: str = "" 73 | out: str = "" 74 | if p.returncode != 0: 75 | log.error("CMD: %s", " ".join(cmdLine)) 76 | log.debug("Read error messages from logfile") 77 | if toPipe is True: 78 | out, err = _readpipe(p) 79 | else: 80 | err = _readlog(logFileName, cmdLine[0]) 81 | raise ProcessError(f"Unable to start [{cmdLine[0]}] error: [{err}]") 82 | 83 | if toPipe is True: 84 | out, err = _readpipe(p) 85 | 86 | if pidFile != "": 87 | realPid = int(_readlog(pidFile, "")) 88 | else: 89 | realPid = p.pid 90 | 91 | process = processInfo(realPid, logFileName, err, out, pidFile) 92 | log.debug("Started [%s] process: [%s]", cmdLine[0], process) 93 | except FileNotFoundError as e: 94 | raise ProcessError(e) from e 95 | 96 | return process 97 | -------------------------------------------------------------------------------- /libvirtnbdbackup/qemu/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exceptions 3 | """ 4 | 5 | 6 | class QemuHelperError(Exception): 7 | """Errors during qemu helper""" 8 | 9 | 10 | class NbdServerProcessError(QemuHelperError): 11 | """Unable to start nbd server for offline backup""" 12 | 13 | 14 | class ProcessError(QemuHelperError): 15 | """Unable to start process""" 16 | -------------------------------------------------------------------------------- /libvirtnbdbackup/restore/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | __title__ = "restore" 19 | __version__ = "0.1" 20 | -------------------------------------------------------------------------------- /libvirtnbdbackup/restore/data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import logging 19 | import pprint 20 | from argparse import Namespace 21 | from libvirtnbdbackup import chunk 22 | from libvirtnbdbackup import lz4 23 | from libvirtnbdbackup import common as lib 24 | from libvirtnbdbackup.sparsestream import types 25 | from libvirtnbdbackup.sparsestream import streamer 26 | from libvirtnbdbackup.sparsestream.exceptions import StreamFormatException 27 | from libvirtnbdbackup.exceptions import RestoreError 28 | from libvirtnbdbackup.exceptions import UntilCheckpointReached 29 | 30 | 31 | def restore( 32 | args: Namespace, 33 | stream: streamer.SparseStream, 34 | disk: str, 35 | targetFile: str, 36 | connection, 37 | ) -> bool: 38 | """Restore the data stream to the target file""" 39 | diskState = False 40 | diskState = _write(args, stream, disk, targetFile, connection) 41 | # no data has been processed 42 | if diskState is None: 43 | diskState = True 44 | 45 | return diskState 46 | 47 | 48 | def _write( # pylint: disable=too-many-branches,too-many-locals,too-many-statements 49 | args: Namespace, 50 | stream: streamer.SparseStream, 51 | dataFile: str, 52 | targetFile: str, 53 | connection, 54 | ) -> bool: 55 | """Restore data for disk""" 56 | sTypes = types.SparseStreamTypes() 57 | 58 | try: 59 | # pylint: disable=consider-using-with 60 | reader = open(dataFile, "rb") 61 | except OSError as errmsg: 62 | logging.error("Failed to open backup file for reading: [%s].", errmsg) 63 | raise RestoreError from errmsg 64 | 65 | try: 66 | kind, start, length = stream.readFrame(reader) 67 | meta = stream.loadMetadata(reader.read(length)) 68 | except StreamFormatException as errmsg: 69 | logging.fatal(errmsg) 70 | raise RestoreError from errmsg 71 | 72 | trailer = None 73 | if lib.isCompressed(meta) is True: 74 | trailer = stream.readCompressionTrailer(reader) 75 | logging.info("Found compression trailer.") 76 | logging.debug("%s", trailer) 77 | 78 | if meta["dataSize"] == 0: 79 | logging.info("File [%s] contains no dirty blocks, skipping.", dataFile) 80 | if meta["checkpointName"] == args.until: 81 | logging.info("Reached checkpoint [%s], stopping", args.until) 82 | raise UntilCheckpointReached 83 | return True 84 | 85 | logging.info( 86 | "Applying data from backup file [%s] to target file [%s].", dataFile, targetFile 87 | ) 88 | pprint.pprint(meta) 89 | assert reader.read(len(sTypes.TERM)) == sTypes.TERM 90 | 91 | progressBar = lib.progressBar( 92 | meta["dataSize"], f"restoring disk [{meta['diskName']}]", args 93 | ) 94 | dataSize: int = 0 95 | dataBlockCnt: int = 0 96 | while True: 97 | try: 98 | kind, start, length = stream.readFrame(reader) 99 | except StreamFormatException as err: 100 | logging.error("Can't read stream at pos: [%s]: [%s]", reader.tell(), err) 101 | raise RestoreError from err 102 | if kind == sTypes.ZERO: 103 | logging.debug("Zero segment from [%s] length: [%s]", start, length) 104 | elif kind == sTypes.DATA: 105 | logging.debug( 106 | "Processing data segment from [%s] length: [%s]", start, length 107 | ) 108 | 109 | originalSize = length 110 | if trailer: 111 | logging.debug("Block: [%s]", dataBlockCnt) 112 | logging.debug("Original block size: [%s]", length) 113 | length = trailer[dataBlockCnt] 114 | logging.debug("Compressed block size: [%s]", length) 115 | 116 | if originalSize >= connection.maxRequestSize: 117 | logging.debug( 118 | "Chunked read/write, start: [%s], len: [%s]", start, length 119 | ) 120 | try: 121 | written = chunk.read( 122 | reader, 123 | start, 124 | length, 125 | connection, 126 | lib.isCompressed(meta), 127 | progressBar, 128 | ) 129 | except Exception as e: 130 | logging.exception(e) 131 | raise RestoreError from e 132 | logging.debug("Wrote: [%s]", written) 133 | else: 134 | try: 135 | data = reader.read(length) 136 | if lib.isCompressed(meta): 137 | data = lz4.decompressFrame(data) 138 | connection.nbd.pwrite(data, start) 139 | written = len(data) 140 | except Exception as e: 141 | logging.exception(e) 142 | raise RestoreError from e 143 | progressBar.update(written) 144 | 145 | assert reader.read(len(sTypes.TERM)) == sTypes.TERM 146 | dataSize += originalSize 147 | dataBlockCnt += 1 148 | elif kind == sTypes.STOP: 149 | progressBar.close() 150 | if dataSize != meta["dataSize"]: 151 | logging.error( 152 | "Restored data size does not match [%s] != [%s]", 153 | dataSize, 154 | meta["dataSize"], 155 | ) 156 | raise RestoreError("Data size mismatch") 157 | break 158 | 159 | logging.info("End of stream, [%s] of data processed", lib.humanize(dataSize)) 160 | if meta["checkpointName"] == args.until: 161 | logging.info("Reached checkpoint [%s], stopping", args.until) 162 | raise UntilCheckpointReached 163 | 164 | if connection.nbd.can_flush() is True: 165 | logging.debug("Flushing NBD connection handle") 166 | connection.nbd.flush() 167 | 168 | return True 169 | -------------------------------------------------------------------------------- /libvirtnbdbackup/restore/disk.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import logging 19 | from argparse import Namespace 20 | from libvirtnbdbackup import virt 21 | from libvirtnbdbackup import common as lib 22 | from libvirtnbdbackup.objects import DomainDisk 23 | from libvirtnbdbackup.restore import server 24 | from libvirtnbdbackup.restore import files 25 | from libvirtnbdbackup.restore import image 26 | from libvirtnbdbackup.restore import header 27 | from libvirtnbdbackup.restore import data 28 | from libvirtnbdbackup.restore import vmconfig 29 | from libvirtnbdbackup.sparsestream import types 30 | from libvirtnbdbackup.sparsestream import streamer 31 | from libvirtnbdbackup.exceptions import RestoreError, UntilCheckpointReached 32 | from libvirtnbdbackup.nbdcli.exceptions import NbdConnectionTimeout 33 | 34 | 35 | def _backingstore(args: Namespace, disk: DomainDisk) -> None: 36 | """If an virtual machine was running on an snapshot image, 37 | warn user, the virtual machine configuration has to be 38 | adjusted before starting the VM is possible. 39 | 40 | User created external or internal Snapshots are not part of 41 | the backup. 42 | """ 43 | if len(disk.backingstores) > 0 and not args.adjust_config: 44 | logging.warning( 45 | "Target image [%s] seems to be a snapshot image.", disk.filename 46 | ) 47 | logging.warning("Target virtual machine configuration must be altered!") 48 | logging.warning("Configured backing store images must be changed.") 49 | 50 | 51 | def restore( # pylint: disable=too-many-branches,too-many-statements,too-many-locals 52 | args: Namespace, ConfigFile: str, virtClient: virt.client 53 | ) -> bytes: 54 | """Handle disk restore operation and adjust virtual machine 55 | configuration accordingly.""" 56 | stream = streamer.SparseStream(types) 57 | vmConfig = vmconfig.read(ConfigFile) 58 | vmDisks = virtClient.getDomainDisks(args, vmConfig) 59 | if not vmDisks: 60 | raise RestoreError("Unable to parse disks from config") 61 | 62 | restConfig: bytes = vmConfig.encode() 63 | for disk in vmDisks: 64 | if args.disk not in (None, disk.target): 65 | logging.info("Skipping disk [%s] for restore", disk.target) 66 | continue 67 | 68 | restoreDisk = lib.getLatest(args.input, f"{disk.target}*.data") 69 | logging.debug("Restoring disk: [%s]", restoreDisk) 70 | if len(restoreDisk) < 1: 71 | logging.warning( 72 | "No backup file for disk [%s] found, assuming it has been excluded.", 73 | disk.target, 74 | ) 75 | if args.adjust_config is True: 76 | restConfig = vmconfig.removeDisk(restConfig.decode(), disk.target) 77 | continue 78 | 79 | targetFile = files.target(args, disk) 80 | 81 | if args.raw and disk.format == "raw": 82 | logging.info("Restoring raw image to [%s]", targetFile) 83 | lib.copy(args, restoreDisk[0], targetFile) 84 | continue 85 | 86 | if "full" not in restoreDisk[0] and "copy" not in restoreDisk[0]: 87 | logging.error( 88 | "[%s]: Unable to locate base full or copy backup.", restoreDisk[0] 89 | ) 90 | raise RestoreError("Failed to locate backup.") 91 | 92 | cptnum = -1 93 | if args.until is not None: 94 | cptnum = int(args.until.split(".")[-1]) 95 | 96 | meta = header.get(restoreDisk[cptnum], stream) 97 | 98 | try: 99 | image.create(args, meta, targetFile, args.sshClient) 100 | except RestoreError as errmsg: 101 | raise RestoreError("Creating target image failed.") from errmsg 102 | 103 | try: 104 | connection = server.start(args, meta["diskName"], targetFile, virtClient) 105 | except NbdConnectionTimeout as e: 106 | raise RestoreError(e) from e 107 | 108 | for dataFile in restoreDisk: 109 | try: 110 | data.restore(args, stream, dataFile, targetFile, connection) 111 | except UntilCheckpointReached: 112 | break 113 | except RestoreError: 114 | break 115 | 116 | _backingstore(args, disk) 117 | if args.adjust_config is True: 118 | restConfig = vmconfig.adjust(disk, restConfig.decode(), targetFile) 119 | 120 | logging.debug("Closing NBD connection") 121 | connection.disconnect() 122 | 123 | if args.adjust_config is True: 124 | restConfig = vmconfig.removeUuid(restConfig.decode()) 125 | restConfig = vmconfig.setVMName(args, restConfig.decode()) 126 | 127 | return restConfig 128 | -------------------------------------------------------------------------------- /libvirtnbdbackup/restore/files.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import os 19 | import zlib 20 | import json 21 | import logging 22 | from typing import List 23 | from argparse import Namespace 24 | from libvirtnbdbackup import virt 25 | from libvirtnbdbackup import output 26 | from libvirtnbdbackup.restore import vmconfig 27 | from libvirtnbdbackup.restore import header 28 | from libvirtnbdbackup import common as lib 29 | from libvirtnbdbackup.objects import DomainDisk 30 | from libvirtnbdbackup.sparsestream import streamer 31 | from libvirtnbdbackup.exceptions import RestoreError 32 | 33 | 34 | def restore(args: Namespace, vmConfig: str, virtClient: virt.client) -> None: 35 | """Notice user if backed up vm had loader / nvram""" 36 | config = vmconfig.read(vmConfig) 37 | info = virtClient.getDomainInfo(config) 38 | 39 | for setting, val in info.items(): 40 | f = lib.getLatest(args.input, f"*{os.path.basename(val)}*", -1) 41 | if lib.exists(args, val): 42 | logging.info( 43 | "File [%s]: for boot option [%s] already exists, skipping.", 44 | val, 45 | setting, 46 | ) 47 | continue 48 | 49 | logging.info( 50 | "Restoring configured file [%s] for boot option [%s]", val, setting 51 | ) 52 | lib.copy(args, f[0], val) 53 | 54 | 55 | def verify(args: Namespace, dataFiles: List[str]) -> bool: 56 | """Compute adler32 checksum for exiting data files and 57 | compare with checksums computed during backup.""" 58 | for dataFile in dataFiles: 59 | if args.disk is not None and not os.path.basename(dataFile).startswith( 60 | args.disk 61 | ): 62 | continue 63 | logging.debug("Using buffer size: %s", args.buffsize) 64 | logging.info("Computing checksum for: %s", dataFile) 65 | 66 | sourceFile = dataFile 67 | if args.sequence: 68 | sourceFile = os.path.join(args.input, dataFile) 69 | 70 | with output.openfile(sourceFile, "rb") as vfh: 71 | adler = 1 72 | data = vfh.read(args.buffsize) 73 | while data: 74 | adler = zlib.adler32(data, adler) 75 | data = vfh.read(args.buffsize) 76 | 77 | chksumFile = f"{sourceFile}.chksum" 78 | logging.info("Checksum result: %s", adler) 79 | if not os.path.exists(chksumFile): 80 | logging.info("No checksum found, skipping: [%s]", sourceFile) 81 | continue 82 | logging.info("Comparing checksum with stored information") 83 | with output.openfile(chksumFile, "r") as s: 84 | storedSum = int(s.read()) 85 | if storedSum != adler: 86 | logging.error("Stored sums do not match: [%s]!=[%s]", storedSum, adler) 87 | return False 88 | 89 | logging.info("OK") 90 | return True 91 | 92 | 93 | def dump(args: Namespace, stream: streamer.SparseStream, dataFiles: List[str]) -> bool: 94 | """Dump stream contents to json output""" 95 | logging.info("Dumping saveset meta information") 96 | entries = [] 97 | for dataFile in dataFiles: 98 | if args.disk is not None and not os.path.basename(dataFile).startswith( 99 | args.disk 100 | ): 101 | continue 102 | logging.info(dataFile) 103 | 104 | sourceFile = dataFile 105 | if args.sequence: 106 | sourceFile = os.path.join(args.input, dataFile) 107 | 108 | try: 109 | meta = header.get(sourceFile, stream) 110 | except RestoreError as e: 111 | logging.error(e) 112 | continue 113 | 114 | entries.append(meta) 115 | 116 | if lib.isCompressed(meta): 117 | logging.info("Compressed stream found: [%s].", meta["compressionMethod"]) 118 | 119 | print(json.dumps(entries, indent=4)) 120 | 121 | return True 122 | 123 | 124 | def target(args: Namespace, disk: DomainDisk) -> str: 125 | """Based on disk information, return target file 126 | to create during restore.""" 127 | if disk.filename is not None: 128 | targetFile = os.path.join(args.output, disk.filename) 129 | else: 130 | targetFile = os.path.join(args.output, disk.target) 131 | 132 | return targetFile 133 | -------------------------------------------------------------------------------- /libvirtnbdbackup/restore/header.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | from typing import Dict 19 | from libvirtnbdbackup import common as lib 20 | from libvirtnbdbackup.sparsestream import streamer 21 | from libvirtnbdbackup.sparsestream.exceptions import StreamFormatException 22 | from libvirtnbdbackup.exceptions import RestoreError 23 | from libvirtnbdbackup.output.exceptions import OutputException 24 | 25 | 26 | def get(diskFile: str, stream: streamer.SparseStream) -> Dict[str, str]: 27 | """Read header from data file""" 28 | try: 29 | return lib.dumpMetaData(diskFile, stream) 30 | except StreamFormatException as errmsg: 31 | raise RestoreError( 32 | f"Reading metadata from [{diskFile}] failed: [{errmsg}]" 33 | ) from errmsg 34 | except OutputException as errmsg: 35 | raise RestoreError( 36 | f"Reading data file [{diskFile}] failed: [{errmsg}]" 37 | ) from errmsg 38 | -------------------------------------------------------------------------------- /libvirtnbdbackup/restore/image.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import os 19 | import logging 20 | import json 21 | from argparse import Namespace 22 | from typing import List, Dict 23 | from libvirtnbdbackup.qemu import util as qemu 24 | from libvirtnbdbackup import output 25 | from libvirtnbdbackup import common as lib 26 | from libvirtnbdbackup.exceptions import RestoreError 27 | from libvirtnbdbackup.qemu.exceptions import ProcessError 28 | from libvirtnbdbackup.output.exceptions import OutputException 29 | from libvirtnbdbackup.ssh.exceptions import sshError 30 | 31 | 32 | def getConfig(args: Namespace, meta: Dict[str, str]) -> List[str]: 33 | """Check if backup includes exported qcow config and return a list 34 | of options passed to qemu-img create command""" 35 | opt: List[str] = [] 36 | qcowConfig = None 37 | qcowConfigFile = lib.getLatest(args.input, f"{meta['diskName']}*.qcow.json*", -1) 38 | if not qcowConfigFile: 39 | logging.warning("No qcow image config found, will use default options.") 40 | return opt 41 | 42 | lastConfigFile = qcowConfigFile[0] 43 | 44 | try: 45 | with output.openfile(lastConfigFile, "rb") as qFh: 46 | qcowConfig = json.loads(qFh.read().decode()) 47 | logging.info("Using QCOW options from backup file: [%s]", lastConfigFile) 48 | except ( 49 | OutputException, 50 | json.decoder.JSONDecodeError, 51 | ) as errmsg: 52 | logging.warning( 53 | "Unable to load original QCOW image config, using defaults: [%s].", 54 | errmsg, 55 | ) 56 | return opt 57 | 58 | try: 59 | opt.append("-o") 60 | opt.append(f"compat={qcowConfig['format-specific']['data']['compat']}") 61 | except KeyError as errmsg: 62 | logging.warning("Unable apply QCOW specific compat option: [%s]", errmsg) 63 | 64 | try: 65 | opt.append("-o") 66 | opt.append(f"cluster_size={qcowConfig['cluster-size']}") 67 | except KeyError as errmsg: 68 | logging.warning("Unable apply QCOW specific cluster_size option: [%s]", errmsg) 69 | 70 | try: 71 | if qcowConfig["format-specific"]["data"]["lazy-refcounts"]: 72 | opt.append("-o") 73 | opt.append("lazy_refcounts=on") 74 | except KeyError as errmsg: 75 | logging.warning( 76 | "Unable apply QCOW specific lazy_refcounts option: [%s]", errmsg 77 | ) 78 | 79 | return opt 80 | 81 | 82 | def create(args: Namespace, meta: Dict[str, str], targetFile: str, sshClient): 83 | """Create target image file""" 84 | logging.info( 85 | "Create virtual disk [%s] format: [%s] size: [%s] based on: [%s] preallocated: [%s]", 86 | targetFile, 87 | meta["diskFormat"], 88 | meta["virtualSize"], 89 | meta["checkpointName"], 90 | args.preallocate, 91 | ) 92 | 93 | options = getConfig(args, meta) 94 | if lib.exists(args, targetFile): 95 | logging.error( 96 | "Target file already exists: [%s], won't overwrite.", 97 | os.path.abspath(targetFile), 98 | ) 99 | raise RestoreError 100 | 101 | qFh = qemu.util(meta["diskName"]) 102 | try: 103 | qFh.create( 104 | args, 105 | targetFile, 106 | int(meta["virtualSize"]), 107 | meta["diskFormat"], 108 | options, 109 | sshClient, 110 | ) 111 | except (ProcessError, sshError) as e: 112 | logging.error("Failed to create restore target: [%s]", e) 113 | raise RestoreError from e 114 | -------------------------------------------------------------------------------- /libvirtnbdbackup/restore/sequence.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import os 19 | from typing import List 20 | from argparse import Namespace 21 | from libvirtnbdbackup import virt 22 | from libvirtnbdbackup import common as lib 23 | from libvirtnbdbackup.restore import header 24 | from libvirtnbdbackup.restore import server 25 | from libvirtnbdbackup.restore import image 26 | from libvirtnbdbackup.restore import data 27 | from libvirtnbdbackup.sparsestream import types 28 | from libvirtnbdbackup.sparsestream import streamer 29 | from libvirtnbdbackup.exceptions import RestoreError 30 | 31 | 32 | def restore(args: Namespace, dataFiles: List[str], virtClient: virt.client) -> bool: 33 | """Reconstruct image from a given set of data files""" 34 | stream = streamer.SparseStream(types) 35 | 36 | result: bool = False 37 | 38 | sourceFile = os.path.join(args.input, dataFiles[-1]) 39 | meta = header.get(sourceFile, stream) 40 | if not meta: 41 | return result 42 | 43 | diskName = meta["diskName"] 44 | targetFile = os.path.join(args.output, diskName) 45 | if lib.exists(args, targetFile): 46 | raise RestoreError(f"Targetfile {targetFile} already exists.") 47 | 48 | try: 49 | image.create(args, meta, targetFile, args.sshClient) 50 | except RestoreError as errmsg: 51 | raise errmsg 52 | 53 | connection = server.start(args, diskName, targetFile, virtClient) 54 | 55 | for disk in dataFiles: 56 | sourceFile = os.path.join(args.input, disk) 57 | result = data.restore(args, stream, sourceFile, targetFile, connection) 58 | 59 | connection.disconnect() 60 | 61 | return result 62 | -------------------------------------------------------------------------------- /libvirtnbdbackup/restore/server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import logging 19 | from argparse import Namespace 20 | from typing import Union 21 | from libvirtnbdbackup import nbdcli 22 | from libvirtnbdbackup import virt 23 | from libvirtnbdbackup.qemu import util as qemu 24 | from libvirtnbdbackup.ssh.exceptions import sshError 25 | from libvirtnbdbackup.qemu.exceptions import ProcessError 26 | from libvirtnbdbackup.exceptions import RestoreError 27 | 28 | log = logging.getLogger("restore") 29 | 30 | 31 | def setup(args: Namespace, exportName: str, targetFile: str, virtClient: virt.client): 32 | """Setup NBD process required for restore, either remote or local""" 33 | qFh = qemu.util(exportName) 34 | cType: Union[nbdcli.TCP, nbdcli.Unix] 35 | if not virtClient.remoteHost: 36 | logging.info("Starting local NBD server on socket: [%s]", args.socketfile) 37 | proc = qFh.startRestoreNbdServer(targetFile, args.socketfile) 38 | cType = nbdcli.Unix(exportName, "", args.socketfile) 39 | else: 40 | remoteIP = virtClient.remoteHost 41 | if args.nbd_ip != "": 42 | remoteIP = args.nbd_ip 43 | logging.info( 44 | "Starting remote NBD server on socket: [%s:%s]", 45 | remoteIP, 46 | args.nbd_port, 47 | ) 48 | proc = qFh.startRemoteRestoreNbdServer(args, targetFile) 49 | cType = nbdcli.TCP(exportName, "", remoteIP, args.tls, args.nbd_port) 50 | 51 | nbdClient = nbdcli.client(cType, False) 52 | logging.info("Started NBD server, PID: [%s]", proc.pid) 53 | return nbdClient.connect() 54 | 55 | 56 | def start(args: Namespace, diskName: str, targetFile: str, virtClient: virt.client): 57 | """Start NDB Service""" 58 | try: 59 | return setup(args, diskName, targetFile, virtClient) 60 | except ProcessError as errmsg: 61 | logging.error(errmsg) 62 | raise RestoreError("Failed to start local NBD server.") from errmsg 63 | except sshError as errmsg: 64 | logging.error(errmsg) 65 | raise RestoreError("Failed to start remote NBD server.") from errmsg 66 | -------------------------------------------------------------------------------- /libvirtnbdbackup/restore/vmconfig.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import os 19 | import tempfile 20 | import logging 21 | from argparse import Namespace 22 | from libvirtnbdbackup import output 23 | from libvirtnbdbackup import common as lib 24 | from libvirtnbdbackup.objects import DomainDisk 25 | from libvirtnbdbackup.virt import xml 26 | from libvirtnbdbackup.virt import disktype 27 | 28 | 29 | def read(ConfigFile: str) -> str: 30 | """Read saved virtual machine config'""" 31 | try: 32 | return output.openfile(ConfigFile, "rb").read().decode() 33 | except: 34 | logging.error("Can't read config file: [%s]", ConfigFile) 35 | raise 36 | 37 | 38 | def removeDisk(vmConfig: str, excluded) -> bytes: 39 | """Remove disk from config, in case it has been excluded 40 | from the backup.""" 41 | tree = xml.asTree(vmConfig) 42 | logging.info("Removing excluded disk [%s] from vm config.", excluded) 43 | try: 44 | target = tree.xpath(f"devices/disk/target[@dev='{excluded}']")[0] 45 | disk = target.getparent() 46 | disk.getparent().remove(disk) 47 | except IndexError: 48 | logging.warning("Removing excluded disk from config failed: no object found.") 49 | 50 | return xml.ElementTree.tostring(tree, encoding="utf8", method="xml") 51 | 52 | 53 | def removeUuid(vmConfig: str) -> bytes: 54 | """Remove the auto generated UUID from the config file to allow 55 | for restore into new name""" 56 | tree = xml.asTree(vmConfig) 57 | try: 58 | logging.info("Removing uuid setting from vm config.") 59 | uuid = tree.xpath("uuid")[0] 60 | tree.remove(uuid) 61 | except IndexError: 62 | pass 63 | 64 | return xml.ElementTree.tostring(tree, encoding="utf8", method="xml") 65 | 66 | 67 | def setVMName(args: Namespace, vmConfig: str) -> bytes: 68 | """Change / set the VM name to be restored""" 69 | tree = xml.asTree(vmConfig) 70 | name = tree.xpath("name")[0] 71 | if args.name is None and not name.text.startswith("restore"): 72 | domainName = f"restore_{name.text}" 73 | logging.info("Change VM name from [%s] to [%s]", name.text, domainName) 74 | name.text = domainName 75 | else: 76 | logging.info("Set name from [%s] to [%s]", name.text, args.name) 77 | name.text = args.name 78 | 79 | return xml.ElementTree.tostring(tree, encoding="utf8", method="xml") 80 | 81 | 82 | def adjust(restoreDisk: DomainDisk, vmConfig: str, targetFile: str) -> bytes: 83 | """Adjust virtual machine configuration after restoring. Changes 84 | the paths to the virtual machine disks and attempts to remove 85 | components excluded during restore.""" 86 | tree = xml.asTree(vmConfig) 87 | for disk in tree.xpath("devices/disk"): 88 | if disk.get("type") == "volume": 89 | logging.info("Disk has type volume, resetting to type file.") 90 | disk.set("type", "file") 91 | 92 | dev = disk.xpath("target")[0].get("dev") 93 | logging.debug("Handling target device: [%s]", dev) 94 | 95 | device = disk.get("device") 96 | driver = disk.xpath("driver")[0].get("type") 97 | 98 | if disktype.Optical(device, dev): 99 | logging.info("Removing device [%s], type [%s] from vm config", dev, device) 100 | disk.getparent().remove(disk) 101 | continue 102 | 103 | if disktype.Raw(driver, device): 104 | logging.warning( 105 | "Removing raw disk [%s] from vm config, use --raw to copy as is.", 106 | dev, 107 | ) 108 | disk.getparent().remove(disk) 109 | continue 110 | backingStore = disk.xpath("backingStore") 111 | if backingStore: 112 | logging.info("Removing existent backing store settings") 113 | disk.remove(backingStore[0]) 114 | 115 | originalFile = disk.xpath("source")[0].get("file") 116 | if dev == restoreDisk.target: 117 | abspath = os.path.abspath(targetFile) 118 | logging.info( 119 | "Change target file for disk [%s] from [%s] to [%s]", 120 | restoreDisk.target, 121 | originalFile, 122 | abspath, 123 | ) 124 | disk.xpath("source")[0].set("file", abspath) 125 | 126 | return xml.ElementTree.tostring(tree, encoding="utf8", method="xml") 127 | 128 | 129 | def restore( 130 | args: Namespace, 131 | vmConfig: str, 132 | adjustedConfig: bytes, 133 | targetFileName: str, 134 | ) -> None: 135 | """Restore either original or adjusted vm configuration 136 | to new directory""" 137 | targetFile = os.path.join(args.output, os.path.basename(targetFileName)) 138 | if args.adjust_config is True: 139 | if args.sshClient: 140 | with tempfile.NamedTemporaryFile(delete=True) as fh: 141 | fh.write(adjustedConfig) 142 | lib.copy(args, fh.name, targetFile) 143 | else: 144 | with output.openfile(targetFile, "wb") as cnf: 145 | cnf.write(adjustedConfig) 146 | logging.info("Adjusted config placed in: [%s]", targetFile) 147 | if args.define is False: 148 | logging.info("Use 'virsh define %s' to define VM", targetFile) 149 | else: 150 | lib.copy(args, vmConfig, targetFile) 151 | logging.info("Copied original vm config to [%s]", targetFile) 152 | logging.info("Note: virtual machine config must be adjusted manually.") 153 | -------------------------------------------------------------------------------- /libvirtnbdbackup/sighandle.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | """ 3 | Copyright (C) 2023 Michael Ablassmeier 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | import os 19 | import sys 20 | from argparse import Namespace 21 | from typing import Any 22 | from libvirt import virDomain 23 | from libvirtnbdbackup import virt 24 | from libvirtnbdbackup import common as lib 25 | from libvirtnbdbackup.objects import processInfo 26 | from libvirtnbdbackup.qemu import util as qemu 27 | 28 | 29 | class Backup: 30 | """Handle signal during backup operation""" 31 | 32 | @staticmethod 33 | def catch( 34 | args: Namespace, 35 | domObj: virDomain, 36 | virtClient: virt.client, 37 | log: Any, 38 | signum: int, 39 | _, 40 | ) -> None: 41 | """Catch signal, attempt to stop running backup job.""" 42 | log.error("Signal caught: %s", signum) 43 | 44 | if args.offline is True: 45 | log.error("Exiting.") 46 | sys.exit(1) 47 | 48 | if virtClient.remoteHost != "": 49 | log.info("Reconnecting remote system to stop backup job.") 50 | try: 51 | virtClient = virt.client(args) 52 | domObj = virtClient.getDomain(args.domain) 53 | except virt.exceptions.connectionFailed as e: 54 | log.error("Reconnecting remote host failed: [%s]", e) 55 | log.error("Unable to stop backup job on remote system.", e) 56 | sys.exit(1) 57 | 58 | log.info("Cleanup: Stopping backup job.") 59 | virtClient.stopBackup(domObj) 60 | virtClient.close() 61 | sys.exit(1) 62 | 63 | 64 | class Map: 65 | """Handle signal during map operation""" 66 | 67 | @staticmethod 68 | def catch( 69 | args: Namespace, 70 | nbdkitProcess: processInfo, 71 | blockMap, 72 | log: Any, 73 | signum, 74 | _, 75 | ): 76 | """Catch signal, attempt to stop processes.""" 77 | log.info("Received signal: [%s]", signum) 78 | qemu.util("").disconnect(args.device) 79 | if not args.verbose: 80 | log.info("Removing temporary blockmap file: [%s]", blockMap.name) 81 | os.remove(blockMap.name) 82 | log.info("Removing nbdkit logfile: [%s]", nbdkitProcess.logFile) 83 | os.remove(nbdkitProcess.logFile) 84 | lib.killProc(nbdkitProcess.pid) 85 | sys.exit(0) 86 | -------------------------------------------------------------------------------- /libvirtnbdbackup/sparsestream/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | __title__ = "sparsestream" 19 | __version__ = "0.1" 20 | -------------------------------------------------------------------------------- /libvirtnbdbackup/sparsestream/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exceptions 3 | """ 4 | 5 | 6 | class StreamFormatException(Exception): 7 | """Wrong metadata header""" 8 | 9 | 10 | class MetaHeaderFormatException(StreamFormatException): 11 | """Wrong metadata header""" 12 | 13 | 14 | class BlockFormatException(StreamFormatException): 15 | """Wrong metadata header""" 16 | 17 | 18 | class FrameformatException(StreamFormatException): 19 | """Frame Format is wrong""" 20 | -------------------------------------------------------------------------------- /libvirtnbdbackup/sparsestream/streamer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | Copyright (C) 2020 Red Hat, Inc. 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | 19 | import json 20 | import os 21 | import datetime 22 | from typing import List, Any, Tuple, Dict 23 | from argparse import Namespace 24 | from libvirtnbdbackup.objects import DomainDisk 25 | from libvirtnbdbackup.sparsestream import exceptions 26 | 27 | 28 | class SparseStream: 29 | """Sparse Stream writer/reader class""" 30 | 31 | def __init__(self, types, version: int = 2) -> None: 32 | """Stream version: 33 | 34 | 1: base version 35 | 2: stream version with compression support 36 | """ 37 | self.version = version 38 | self.compressionMethod: str = "lz4" 39 | self.types = types.SparseStreamTypes() 40 | 41 | def dumpMetadata( 42 | self, 43 | args: Namespace, 44 | virtualSize: int, 45 | dataSize: int, 46 | disk: DomainDisk, 47 | ) -> bytes: 48 | """First block in backup stream is Meta data information 49 | about virtual size of the disk being backed up, as well 50 | as various information regarding backup. 51 | Dumps Metadata frame to be written at start of stream in 52 | json format. 53 | """ 54 | meta = { 55 | "virtualSize": virtualSize, 56 | "dataSize": dataSize, 57 | "date": datetime.datetime.now().isoformat(), 58 | "diskName": disk.target, 59 | "diskFormat": disk.format, 60 | "checkpointName": args.cpt.name, 61 | "compressed": args.compress, 62 | "compressionMethod": self.compressionMethod, 63 | "parentCheckpoint": args.cpt.parent, 64 | "incremental": (args.level in ("inc", "diff")), 65 | "streamVersion": self.version, 66 | } 67 | return json.dumps(meta, indent=4).encode("utf-8") 68 | 69 | def writeCompressionTrailer(self, writer, trailer: List[Any]) -> None: 70 | """Dump compression trailer to end of stream""" 71 | size = writer.write(json.dumps(trailer).encode()) 72 | writer.write(self.types.TERM) 73 | self.writeFrame(writer, self.types.COMP, 0, size) 74 | 75 | def _readHeader(self, reader) -> Tuple[str, str, str]: 76 | """Attempt to read header""" 77 | header = reader.read(self.types.FRAME_LEN) 78 | try: 79 | kind, start, length = header.split(b" ", 2) 80 | except ValueError as err: 81 | raise exceptions.BlockFormatException( 82 | f"Invalid block format: [{err}]" 83 | ) from err 84 | 85 | return kind, start, length 86 | 87 | @staticmethod 88 | def _parseHeader(kind, start: str, length: str) -> Tuple[str, int, int]: 89 | """Return parsed header information""" 90 | try: 91 | return kind, int(start, 16), int(length, 16) 92 | except ValueError as err: 93 | raise exceptions.FrameformatException( 94 | f"Invalid frame format: [{err}]" 95 | ) from err 96 | 97 | def readCompressionTrailer(self, reader) -> Dict[int, Any]: 98 | """If compressed stream is found, information about compressed 99 | block sizes is appended as last json payload. 100 | 101 | Function seeks to end of file and reads trailer information. 102 | """ 103 | pos = reader.tell() 104 | reader.seek(0, os.SEEK_END) 105 | reader.seek(-(self.types.FRAME_LEN + len(self.types.TERM)), os.SEEK_CUR) 106 | _, _, length = self._readHeader(reader) 107 | reader.seek(-(self.types.FRAME_LEN + int(length, 16)), os.SEEK_CUR) 108 | trailer = self.loadMetadata(reader.read(int(length, 16))) 109 | reader.seek(pos) 110 | return trailer 111 | 112 | @staticmethod 113 | def loadMetadata(s: bytes) -> Any: 114 | """Load and parse metadata information 115 | Parameters: 116 | s: (str) Json string as received during data file read 117 | Returns: 118 | json.loads: (dict) Decoded json string as python object 119 | """ 120 | try: 121 | return json.loads(s.decode("utf-8")) 122 | except json.decoder.JSONDecodeError as err: 123 | raise exceptions.MetaHeaderFormatException( 124 | f"Invalid meta header format: [{err}]" 125 | ) from err 126 | 127 | def writeFrame(self, writer, kind, start: int, length: int) -> None: 128 | """Write backup frame 129 | Parameters: 130 | writer: (fh) Writer object that implements .write() 131 | """ 132 | writer.write(self.types.FRAME % (kind, start, length)) 133 | 134 | def readFrame(self, reader) -> Tuple[str, int, int]: 135 | """Read backup frame 136 | Parameters: 137 | reader: (fh) Reader object which implements .read() 138 | """ 139 | kind, start, length = self._readHeader(reader) 140 | return self._parseHeader(kind, start, length) 141 | -------------------------------------------------------------------------------- /libvirtnbdbackup/sparsestream/types.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | Copyright (C) 2020 Red Hat, Inc. 4 | 5 | This program is free software: you can redistribute it and/or modify 6 | it under the terms of the GNU General Public License as published by 7 | the Free Software Foundation, either version 3 of the License, or 8 | (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program. If not, see . 17 | """ 18 | 19 | from dataclasses import dataclass 20 | 21 | 22 | @dataclass(frozen=True) 23 | class SparseStreamTypes: 24 | # pylint: disable=too-many-instance-attributes 25 | """Sparse stream format 26 | 27 | Extended format based on the examples provided by the 28 | ovirt-imageio project: 29 | 30 | https://github.com/oVirt/ovirt-imageio/tree/master/examples 31 | 32 | META: start of meta information header 33 | DATA: data block marker 34 | ZERO: zero block marker 35 | STOP: stop block marker 36 | TERM: termination identifier 37 | FRAME: assembled frame 38 | FRAME_LEN: length of frame 39 | 40 | Stream format 41 | ============= 42 | 43 | Stream is composed of one of more frames. 44 | 45 | Meta frame 46 | ---------- 47 | Stream metadata, must be the first frame. 48 | 49 | "meta" space start length "\r\n" \r\n 50 | 51 | Metadata keys in the json payload: 52 | 53 | - virtual-size: image virtual size in bytes 54 | - data-size: number of bytes in data frames 55 | - date: ISO 8601 date string 56 | 57 | Data frame 58 | ---------- 59 | The header is followed by length bytes and terminator. 60 | "data" space start length "\r\n" "\r\n" 61 | 62 | Zero frame 63 | ---------- 64 | A zero extent, no payload. 65 | "zero" space start length "\r\n" 66 | 67 | Stop frame 68 | ---------- 69 | Marks the end of the stream, no payload. 70 | "stop" space start length "\r\n" 71 | 72 | Regular stream Example 73 | ------- 74 | meta 0000000000000000 0000000000000083\r\n 75 | { 76 | [.]] 77 | }\r\n 78 | data 0000000000000000 00000000000100000\r\n 79 | <1 MiB bytes>\r\n 80 | zero 0000000000100000 00000000040000000\r\n 81 | data 0000000040100000 00000000000001000\r\n 82 | <4096 bytes>\r\n 83 | stop 0000000000000000 00000000000000000\r\n 84 | 85 | 86 | Compressed stream: 87 | ------- 88 | Ends with compression marker: 89 | stop 0000000000000000 00000000000000000\r\n 90 | \r\n 91 | comp 0000000000000000 00000000000000010\r\n 92 | """ 93 | 94 | META: bytes = b"meta" 95 | DATA: bytes = b"data" 96 | COMP: bytes = b"comp" 97 | ZERO: bytes = b"zero" 98 | STOP: bytes = b"stop" 99 | TERM: bytes = b"\r\n" 100 | FRAME: bytes = b"%s %016x %016x" + TERM 101 | FRAME_LEN: int = len(FRAME % (STOP, 0, 0)) 102 | -------------------------------------------------------------------------------- /libvirtnbdbackup/ssh/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | __title__ = "ssh" 19 | __version__ = "0.1" 20 | 21 | from .client import client, Mode 22 | -------------------------------------------------------------------------------- /libvirtnbdbackup/ssh/client.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import logging 19 | import socket 20 | from typing import Tuple, Callable 21 | from enum import Enum 22 | from paramiko import ( 23 | AutoAddPolicy, 24 | SSHClient, 25 | SFTPClient, 26 | SSHException, 27 | AuthenticationException, 28 | ) 29 | from libvirtnbdbackup.ssh import exceptions 30 | from libvirtnbdbackup.objects import processInfo 31 | 32 | log = logging.getLogger("ssh") 33 | 34 | 35 | class Mode(Enum): 36 | """Up or download mode""" 37 | 38 | UPLOAD = 1 39 | DOWNLOAD = 2 40 | 41 | 42 | class client: 43 | """Wrapper around paramiko/sftp put and get functions, to be able to 44 | remote copy files from hypervisor host""" 45 | 46 | def __init__( 47 | self, host: str, user: str, port: int = 22, mode: Mode = Mode.DOWNLOAD 48 | ): 49 | self.client = None 50 | self.host = host 51 | self.user = user 52 | self.port = port 53 | self.copy: Callable[[str, str], None] = self.copyFrom 54 | if mode == Mode.UPLOAD: 55 | self.copy = self.copyTo 56 | self.connection = self.connect() 57 | 58 | def connect(self) -> SSHClient: 59 | """Connect to remote system""" 60 | log.info( 61 | "Connecting remote system [%s] via ssh, username: [%s]", 62 | self.host, 63 | self.user, 64 | ) 65 | try: 66 | cli = SSHClient() 67 | cli.load_system_host_keys() 68 | cli.set_missing_host_key_policy(AutoAddPolicy()) 69 | cli.connect( 70 | self.host, 71 | username=self.user, 72 | port=self.port, 73 | timeout=5000, 74 | ) 75 | return cli 76 | except AuthenticationException as e: 77 | raise exceptions.sshError(f"SSH key authentication failed: {e}") 78 | except socket.gaierror as e: 79 | raise exceptions.sshError(f"Unable to connect: {e}") 80 | except SSHException as e: 81 | raise exceptions.sshError(e) 82 | except Exception as e: 83 | log.exception(e) 84 | raise exceptions.sshError(f"Unhandled exception occurred: {e}") 85 | 86 | @property 87 | def sftp(self) -> SFTPClient: 88 | """Copy file""" 89 | return self.connection.open_sftp() 90 | 91 | def exists(self, filepath: str) -> bool: 92 | """ 93 | Check if remote file exists 94 | """ 95 | try: 96 | self.sftp.stat(filepath) 97 | return True 98 | except IOError: 99 | return False 100 | 101 | def copyFrom(self, filepath: str, localpath: str) -> None: 102 | """ 103 | Get file from remote system 104 | """ 105 | log.info("Downloading file [%s] to [%s]", filepath, localpath) 106 | try: 107 | self.sftp.get(filepath, localpath) 108 | except SSHException as e: 109 | log.warning("Unable to download file: [%s]", e) 110 | 111 | def copyTo(self, localpath: str, remotepath: str) -> None: 112 | """ 113 | Put file to remote system 114 | """ 115 | log.info("Uploading file [%s] to [%s]", localpath, remotepath) 116 | try: 117 | self.sftp.put(localpath, remotepath) 118 | except SSHException as e: 119 | log.warning("Unable to upload file: [%s]", e) 120 | 121 | def _execute(self, cmd) -> Tuple[int, str, str]: 122 | _, stdout, stderr = self.connection.exec_command(cmd) 123 | ret = stdout.channel.recv_exit_status() 124 | err = stderr.read().strip().decode() 125 | out = stdout.read().strip().decode() 126 | return ret, err, out 127 | 128 | def run(self, cmd: str, pidFile: str = "", logFile: str = "") -> processInfo: 129 | """ 130 | Execute command 131 | """ 132 | pid: int = 0 133 | pidOut: str 134 | log.debug("Executing remote command: [%s]", cmd) 135 | ret, err, out = self._execute(cmd) 136 | logerr = "" 137 | if ret == 127: 138 | raise exceptions.sshError(err) 139 | if ret != 0: 140 | log.error( 141 | "Executing remote command failed, return code: [%s] stderr: [%s], stdout: [%s]", 142 | ret, 143 | err, 144 | out, 145 | ) 146 | if logFile: 147 | log.debug("Attempting to catch errors from logfile: [%s]", logFile) 148 | _, _, logerr = self._execute(f"cat {logFile}") 149 | raise exceptions.sshError( 150 | f"Error during remote command: [{cmd}]: [{err}] [{logerr}]" 151 | ) 152 | 153 | if pidFile: 154 | log.debug("PIDfile: [%s]", pidFile) 155 | _, _, pidOut = self._execute(f"cat {pidFile}") 156 | pid = int(pidOut) 157 | log.debug("PID: [%s]", pid) 158 | 159 | return processInfo(pid, logFile, err, out, pidFile) 160 | 161 | def disconnect(self): 162 | """Disconnect""" 163 | if self.sftp: 164 | self.sftp.close() 165 | if self.connection: 166 | self.connection.close() 167 | -------------------------------------------------------------------------------- /libvirtnbdbackup/ssh/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exceptions 3 | """ 4 | 5 | 6 | class sshError(Exception): 7 | """Exception thrown during ssh session""" 8 | -------------------------------------------------------------------------------- /libvirtnbdbackup/virt/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | __title__ = "virt" 19 | __version__ = "0.1" 20 | 21 | from .client import client 22 | -------------------------------------------------------------------------------- /libvirtnbdbackup/virt/disktype.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import logging 19 | from lxml.etree import _Element 20 | 21 | log = logging.getLogger() 22 | 23 | 24 | def Optical(device: list, dev: str) -> bool: 25 | """Check if device is cdrom or floppy""" 26 | if device in ("cdrom", "floppy"): 27 | log.info("Skipping attached [%s] device: [%s].", device, dev) 28 | return True 29 | 30 | return False 31 | 32 | 33 | def Lun(device: list, dev: str) -> bool: 34 | """Check if device is direct attached LUN""" 35 | if device == "lun": 36 | log.warning( 37 | "Skipping direct attached lun [%s], use option --raw to include", 38 | dev, 39 | ) 40 | return True 41 | 42 | return False 43 | 44 | 45 | def Block(disk: _Element, dev: str) -> bool: 46 | """Check if device is direct attached block type device""" 47 | if disk.xpath("target")[0].get("type") == "block": 48 | log.warning( 49 | "Block device [%s] excluded by default, use option --raw to include.", 50 | dev, 51 | ) 52 | return True 53 | 54 | return False 55 | 56 | 57 | def Raw(diskFormat: str, dev: str) -> bool: 58 | """Check if disk has RAW disk format""" 59 | if diskFormat == "raw": 60 | log.warning( 61 | "Raw disk [%s] excluded by default, use option --raw to include.", 62 | dev, 63 | ) 64 | return True 65 | 66 | return False 67 | -------------------------------------------------------------------------------- /libvirtnbdbackup/virt/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | Exceptions 3 | """ 4 | 5 | 6 | class virtHelperError(Exception): 7 | """Errors during libvirt helper""" 8 | 9 | 10 | class domainNotFound(virtHelperError): 11 | """Can't find domain""" 12 | 13 | 14 | class connectionFailed(virtHelperError): 15 | """Can't connect libvirtd domain""" 16 | 17 | 18 | class startBackupFailed(virtHelperError): 19 | """Can't start backup operation""" 20 | -------------------------------------------------------------------------------- /libvirtnbdbackup/virt/fs.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import logging 19 | import libvirt 20 | 21 | log = logging.getLogger("fs") 22 | 23 | 24 | def freeze(domObj: libvirt.virDomain, mountpoints: None) -> bool: 25 | """Attempt to freeze domain filesystems using qemu guest agent""" 26 | state, _ = domObj.state() 27 | if state == libvirt.VIR_DOMAIN_PAUSED: 28 | log.info("Skip freezing filesystems: domain is in paused state") 29 | return False 30 | 31 | log.debug("Attempting to freeze filesystems.") 32 | try: 33 | if mountpoints is not None: 34 | frozen = domObj.fsFreeze(mountpoints.split(",")) 35 | else: 36 | frozen = domObj.fsFreeze() 37 | log.info("Freezed [%s] filesystems.", frozen) 38 | return True 39 | except libvirt.libvirtError as errmsg: 40 | log.warning(errmsg) 41 | return False 42 | 43 | 44 | def thaw(domObj: libvirt.virDomain) -> bool: 45 | """Thaw freezed filesystems""" 46 | log.debug("Attempting to thaw filesystems.") 47 | try: 48 | thawed = domObj.fsThaw() 49 | log.info("Thawed [%s] filesystems.", thawed) 50 | return True 51 | except libvirt.libvirtError as errmsg: 52 | log.warning(errmsg) 53 | return False 54 | -------------------------------------------------------------------------------- /libvirtnbdbackup/virt/xml.py: -------------------------------------------------------------------------------- 1 | """ 2 | Copyright (C) 2023 Michael Ablassmeier 3 | 4 | This program is free software: you can redistribute it and/or modify 5 | it under the terms of the GNU General Public License as published by 6 | the Free Software Foundation, either version 3 of the License, or 7 | (at your option) any later version. 8 | 9 | This program is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU General Public License for more details. 13 | 14 | You should have received a copy of the GNU General Public License 15 | along with this program. If not, see . 16 | """ 17 | 18 | import logging 19 | from lxml.etree import _Element 20 | from lxml import etree as ElementTree 21 | 22 | log = logging.getLogger() 23 | 24 | 25 | def asTree(vmConfig: str) -> _Element: 26 | """Return Etree element for vm config""" 27 | return ElementTree.fromstring(vmConfig) 28 | 29 | 30 | def indent(top: _Element) -> str: 31 | """Indent xml output for debug log""" 32 | try: 33 | ElementTree.indent(top) 34 | except ElementTree.ParseError as errmsg: 35 | log.debug("Failed to parse xml: [%s]", errmsg) 36 | except AttributeError: 37 | # older ElementTree versions dont have the 38 | # indent method, skip silently and use 39 | # non formatted string 40 | pass 41 | 42 | xml = ElementTree.tostring(top).decode() 43 | log.debug("\n%s", xml) 44 | 45 | return xml 46 | -------------------------------------------------------------------------------- /man/virtnbdbackup.1: -------------------------------------------------------------------------------- 1 | .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. 2 | .TH VIRTNBDBACKUP "1" "May 2025" "virtnbdbackup 2.29" "User Commands" 3 | .SH NAME 4 | virtnbdbackup \- backup utility for libvirt 5 | .SH DESCRIPTION 6 | usage: virtnbdbackup [\-h] \fB\-d\fR DOMAIN [\-l {copy,full,inc,diff,auto}] 7 | .TP 8 | [\-t {stream,raw}] [\-r] \fB\-o\fR OUTPUT [\-C CHECKPOINTDIR] 9 | [\-\-scratchdir SCRATCHDIR] [\-S] [\-i INCLUDE] [\-x EXCLUDE] 10 | [\-f SOCKETFILE] [\-n] [\-z [COMPRESS]] [\-w WORKER] 11 | [\-F FREEZE_MOUNTPOINT] [\-e] [\-\-no\-sparse\-detection] 12 | [\-T THRESHOLD] [\-U URI] [\-\-user USER] 13 | [\-\-ssh\-user SSH_USER] [\-\-ssh\-port SSH_PORT] 14 | [\-\-password PASSWORD] [\-P NBD_PORT] [\-I NBD_IP] [\-\-tls] 15 | [\-\-tls\-cert TLS_CERT] [\-L] [\-\-quiet] [\-\-nocolor] [\-q] 16 | [\-s] [\-k] [\-p] [\-v] [\-V] 17 | .PP 18 | Backup libvirt/qemu virtual machines 19 | .SS "options:" 20 | .TP 21 | \fB\-h\fR, \fB\-\-help\fR 22 | show this help message and exit 23 | .SS "General options:" 24 | .TP 25 | \fB\-d\fR DOMAIN, \fB\-\-domain\fR DOMAIN 26 | Domain to backup 27 | .TP 28 | \fB\-l\fR {copy,full,inc,diff,auto}, \fB\-\-level\fR {copy,full,inc,diff,auto} 29 | Backup level. (default: copy) 30 | .TP 31 | \fB\-t\fR {stream,raw}, \fB\-\-type\fR {stream,raw} 32 | Output type: stream or raw. (default: stream) 33 | .TP 34 | \fB\-r\fR, \fB\-\-raw\fR 35 | Include full provisioned disk images in backup. (default: False) 36 | .TP 37 | \fB\-o\fR OUTPUT, \fB\-\-output\fR OUTPUT 38 | Output target directory 39 | .TP 40 | \fB\-C\fR CHECKPOINTDIR, \fB\-\-checkpointdir\fR CHECKPOINTDIR 41 | Persistent libvirt checkpoint storage directory 42 | .TP 43 | \fB\-\-scratchdir\fR SCRATCHDIR 44 | Target dir for temporary scratch file. (default: \fI\,/var/tmp\/\fP) 45 | .TP 46 | \fB\-S\fR, \fB\-\-start\-domain\fR 47 | Start virtual machine if it is offline. (default: False) 48 | .TP 49 | \fB\-i\fR INCLUDE, \fB\-\-include\fR INCLUDE 50 | Backup only disk with target dev name (\fB\-i\fR vda) 51 | .TP 52 | \fB\-x\fR EXCLUDE, \fB\-\-exclude\fR EXCLUDE 53 | Exclude disk(s) with target dev name (\fB\-x\fR vda,vdb) 54 | .TP 55 | \fB\-f\fR SOCKETFILE, \fB\-\-socketfile\fR SOCKETFILE 56 | Use specified file for NBD Server socket (default: \fI\,/var/tmp/virtnbdbackup.1876934\/\fP) 57 | .TP 58 | \fB\-n\fR, \fB\-\-noprogress\fR 59 | Disable progress bar 60 | .TP 61 | \fB\-z\fR [COMPRESS], \fB\-\-compress\fR [COMPRESS] 62 | Compress with lz4 compression level. (default: False) 63 | .TP 64 | \fB\-w\fR WORKER, \fB\-\-worker\fR WORKER 65 | Amount of concurrent workers used to backup multiple disks. (default: amount of disks) 66 | .TP 67 | \fB\-F\fR FREEZE_MOUNTPOINT, \fB\-\-freeze\-mountpoint\fR FREEZE_MOUNTPOINT 68 | If qemu agent available, freeze only filesystems on specified mountpoints within virtual machine (default: all) 69 | .TP 70 | \fB\-e\fR, \fB\-\-strict\fR 71 | Change exit code if warnings occur during backup operation. (default: False) 72 | .TP 73 | \fB\-\-no\-sparse\-detection\fR 74 | Skip detection of sparse ranges during incremental or differential backup. (default: False) 75 | .TP 76 | \fB\-T\fR THRESHOLD, \fB\-\-threshold\fR THRESHOLD 77 | Execute backup only if threshold is reached. 78 | .SS "Remote Backup options:" 79 | .TP 80 | \fB\-U\fR URI, \fB\-\-uri\fR URI 81 | Libvirt connection URI. (default: qemu:///session) 82 | .TP 83 | \fB\-\-user\fR USER 84 | User to authenticate against libvirtd. (default: None) 85 | .TP 86 | \fB\-\-ssh\-user\fR SSH_USER 87 | User to authenticate against remote sshd: used for remote copy of files. (default: abi) 88 | .TP 89 | \fB\-\-ssh\-port\fR SSH_PORT 90 | Port to connect to remote sshd: used for remote copy of files. (default: 22) 91 | .TP 92 | \fB\-\-password\fR PASSWORD 93 | Password to authenticate against libvirtd. (default: None) 94 | .TP 95 | \fB\-P\fR NBD_PORT, \fB\-\-nbd\-port\fR NBD_PORT 96 | Port used by remote NBD Service, should be unique for each started backup. (default: 10809) 97 | .TP 98 | \fB\-I\fR NBD_IP, \fB\-\-nbd\-ip\fR NBD_IP 99 | IP used to bind remote NBD service on (default: hostname returned by libvirtd) 100 | .TP 101 | \fB\-\-tls\fR 102 | Enable and use TLS for NBD connection. (default: False) 103 | .TP 104 | \fB\-\-tls\-cert\fR TLS_CERT 105 | Path to TLS certificates used during offline backup and restore. (default: /etc/pki/qemu/) 106 | .SS "Logging options:" 107 | .TP 108 | \fB\-L\fR, \fB\-\-syslog\fR 109 | Additionally send log messages to syslog (default: False) 110 | .TP 111 | \fB\-\-quiet\fR 112 | Disable logging to stderr (default: False) 113 | .TP 114 | \fB\-\-nocolor\fR 115 | Disable colored output (default: False) 116 | .SS "Debug options:" 117 | .TP 118 | \fB\-q\fR, \fB\-\-qemu\fR 119 | Use Qemu tools to query extents. 120 | .TP 121 | \fB\-s\fR, \fB\-\-startonly\fR 122 | Only initialize backup job via libvirt, do not backup any data 123 | .TP 124 | \fB\-k\fR, \fB\-\-killonly\fR 125 | Kill any running block job 126 | .TP 127 | \fB\-p\fR, \fB\-\-printonly\fR 128 | Quit after printing estimated checkpoint size. 129 | .TP 130 | \fB\-v\fR, \fB\-\-verbose\fR 131 | Enable debug output 132 | .TP 133 | \fB\-V\fR, \fB\-\-version\fR 134 | Show version and exit 135 | .SH EXAMPLES 136 | .IP 137 | # full backup of domain 'webvm' with all attached disks: 138 | .IP 139 | virtnbdbackup \-d webvm \-l full \-o /backup/ 140 | .IP 141 | # incremental backup: 142 | .IP 143 | virtnbdbackup \-d webvm \-l inc \-o /backup/ 144 | .IP 145 | # differential backup: 146 | .IP 147 | virtnbdbackup \-d webvm \-l diff \-o /backup/ 148 | .IP 149 | # full backup, exclude disk 'vda': 150 | .IP 151 | virtnbdbackup \-d webvm \-l full \-x vda \-o /backup/ 152 | .IP 153 | # full backup, backup only disk 'vdb': 154 | .IP 155 | virtnbdbackup \-d webvm \-l full \-i vdb \-o /backup/ 156 | .IP 157 | # full backup, compression enabled: 158 | .IP 159 | virtnbdbackup \-d webvm \-l full \-z \-o /backup/ 160 | .IP 161 | # full backup, create archive: 162 | .IP 163 | virtnbdbackup \-d webvm \-l full \-o \- > backup.zip 164 | .IP 165 | # full backup of vm operating on remote libvirtd: 166 | .IP 167 | virtnbdbackup \-U qemu+ssh://root@remotehost/system \-\-ssh\-user root \-d webvm \-l full \-o /backup/ 168 | -------------------------------------------------------------------------------- /man/virtnbdmap.1: -------------------------------------------------------------------------------- 1 | .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. 2 | .TH VIRTNBDMAP "1" "May 2025" "virtnbdmap 2.29" "User Commands" 3 | .SH NAME 4 | virtnbdmap \- map virtnbdbackup image files to nbd devices 5 | .SH DESCRIPTION 6 | usage: virtnbdmap [\-h] \fB\-f\fR FILE [\-b BLOCKSIZE] [\-d DEVICE] [\-e EXPORT_NAME] 7 | .IP 8 | [\-t THREADS] [\-l LISTEN_ADDRESS] [\-p LISTEN_PORT] [\-n] 9 | [\-L LOGFILE] [\-\-nocolor] [\-r] [\-H] [\-v] [\-V] 10 | .PP 11 | Map backup image(s) to block device 12 | .SS "options:" 13 | .TP 14 | \fB\-h\fR, \fB\-\-help\fR 15 | show this help message and exit 16 | .SS "General options:" 17 | .TP 18 | \fB\-f\fR FILE, \fB\-\-file\fR FILE 19 | List of Backup files to map 20 | .TP 21 | \fB\-b\fR BLOCKSIZE, \fB\-\-blocksize\fR BLOCKSIZE 22 | Maximum blocksize passed to nbdkit. (default: 4096) 23 | .TP 24 | \fB\-d\fR DEVICE, \fB\-\-device\fR DEVICE 25 | Target device. (default: \fI\,/dev/nbd0\/\fP) 26 | .TP 27 | \fB\-e\fR EXPORT_NAME, \fB\-\-export\-name\fR EXPORT_NAME 28 | Export name passed to nbdkit. (default: sda) 29 | .TP 30 | \fB\-t\fR THREADS, \fB\-\-threads\fR THREADS 31 | Amount of threads passed to nbdkit process. (default: 1) 32 | .TP 33 | \fB\-l\fR LISTEN_ADDRESS, \fB\-\-listen\-address\fR LISTEN_ADDRESS 34 | IP Address for nbdkit process to listen on. (default: 127.0.0.1) 35 | .TP 36 | \fB\-p\fR LISTEN_PORT, \fB\-\-listen\-port\fR LISTEN_PORT 37 | Port for nbdkit process to listen on. (default: 10809) 38 | .TP 39 | \fB\-n\fR, \fB\-\-noprogress\fR 40 | Disable progress bar 41 | .SS "Logging options:" 42 | .TP 43 | \fB\-L\fR LOGFILE, \fB\-\-logfile\fR LOGFILE 44 | Path to Logfile (default: \fI\,/home/abi/virtnbdmap.log\/\fP) 45 | .TP 46 | \fB\-\-nocolor\fR 47 | Disable colored output (default: False) 48 | .SS "Debug options:" 49 | .TP 50 | \fB\-r\fR, \fB\-\-readonly\fR 51 | Map image readonly (default: False) 52 | .TP 53 | \fB\-H\fR, \fB\-\-hexdump\fR 54 | Hexdump data to logfile for debugging (default: False) 55 | .TP 56 | \fB\-v\fR, \fB\-\-verbose\fR 57 | Enable debug output 58 | .TP 59 | \fB\-V\fR, \fB\-\-version\fR 60 | Show version and exit 61 | .SH EXAMPLES 62 | .IP 63 | # Map full backup to device /dev/nbd0: 64 | .IP 65 | virtnbdmap \-f /backup/sda.full.data 66 | .IP 67 | # Map full backup to device /dev/nbd2: 68 | .IP 69 | virtnbdmap \-f /backup/sda.full.data \-d /dev/nbd2 70 | .IP 71 | # Map sequence of full and incremental to device /dev/nbd2: 72 | .IP 73 | virtnbdmap \-f /backup/sda.full.data,/backup/sda.inc.1.data \-d /dev/nbd2 74 | -------------------------------------------------------------------------------- /man/virtnbdrestore.1: -------------------------------------------------------------------------------- 1 | .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.3. 2 | .TH VIRTNBDRESTORE "1" "May 2025" "virtnbdrestore 2.29" "User Commands" 3 | .SH NAME 4 | virtnbdrestore \- restore utility for libvirt 5 | .SH DESCRIPTION 6 | usage: virtnbdrestore [\-h] [\-a {dump,restore,verify}] \fB\-i\fR INPUT \fB\-o\fR OUTPUT 7 | .TP 8 | [\-u UNTIL] [\-s SEQUENCE] [\-d DISK] [\-n] [\-f SOCKETFILE] 9 | [\-r] [\-c] [\-D] [\-C CONFIG_FILE] [\-N NAME] [\-B BUFFSIZE] 10 | [\-A] [\-U URI] [\-\-user USER] [\-\-ssh\-user SSH_USER] 11 | [\-\-ssh\-port SSH_PORT] [\-\-password PASSWORD] 12 | [\-P NBD_PORT] [\-I NBD_IP] [\-\-tls] [\-\-tls\-cert TLS_CERT] 13 | [\-L LOGFILE] [\-\-nocolor] [\-v] [\-V] 14 | .PP 15 | Restore virtual machine disks 16 | .SS "options:" 17 | .TP 18 | \fB\-h\fR, \fB\-\-help\fR 19 | show this help message and exit 20 | .SS "General options:" 21 | .TP 22 | \fB\-a\fR {dump,restore,verify}, \fB\-\-action\fR {dump,restore,verify} 23 | Action to perform: (default: restore) 24 | .TP 25 | \fB\-i\fR INPUT, \fB\-\-input\fR INPUT 26 | Directory including a backup set 27 | .TP 28 | \fB\-o\fR OUTPUT, \fB\-\-output\fR OUTPUT 29 | Restore target directory 30 | .TP 31 | \fB\-u\fR UNTIL, \fB\-\-until\fR UNTIL 32 | Restore only until checkpoint, point in time restore. 33 | .TP 34 | \fB\-s\fR SEQUENCE, \fB\-\-sequence\fR SEQUENCE 35 | Restore image based on specified backup files. 36 | .TP 37 | \fB\-d\fR DISK, \fB\-\-disk\fR DISK 38 | Process only disk matching target dev name. (default: None) 39 | .TP 40 | \fB\-n\fR, \fB\-\-noprogress\fR 41 | Disable progress bar 42 | .TP 43 | \fB\-f\fR SOCKETFILE, \fB\-\-socketfile\fR SOCKETFILE 44 | Use specified file for NBD Server socket (default: \fI\,/var/tmp/virtnbdbackup.1876972\/\fP) 45 | .TP 46 | \fB\-r\fR, \fB\-\-raw\fR 47 | Copy raw images as is during restore. (default: False) 48 | .TP 49 | \fB\-c\fR, \fB\-\-adjust\-config\fR 50 | Adjust vm configuration during restore. (default: False) 51 | .TP 52 | \fB\-D\fR, \fB\-\-define\fR 53 | Register/define VM after restore. (default: False) 54 | .TP 55 | \fB\-C\fR CONFIG_FILE, \fB\-\-config\-file\fR CONFIG_FILE 56 | Name of the vm config file used for restore. (default: vmconfig.xml) 57 | .TP 58 | \fB\-N\fR NAME, \fB\-\-name\fR NAME 59 | Define restored domain with specified name 60 | .TP 61 | \fB\-B\fR BUFFSIZE, \fB\-\-buffsize\fR BUFFSIZE 62 | Buffer size to use during verify (default: 8192) 63 | .TP 64 | \fB\-A\fR, \fB\-\-preallocate\fR 65 | Preallocate restored qcow images. (default: False) 66 | .SS "Remote Restore options:" 67 | .TP 68 | \fB\-U\fR URI, \fB\-\-uri\fR URI 69 | Libvirt connection URI. (default: qemu:///session) 70 | .TP 71 | \fB\-\-user\fR USER 72 | User to authenticate against libvirtd. (default: None) 73 | .TP 74 | \fB\-\-ssh\-user\fR SSH_USER 75 | User to authenticate against remote sshd: used for remote copy of files. (default: abi) 76 | .TP 77 | \fB\-\-ssh\-port\fR SSH_PORT 78 | Port to connect to remote sshd: used for remote copy of files. (default: 22) 79 | .TP 80 | \fB\-\-password\fR PASSWORD 81 | Password to authenticate against libvirtd. (default: None) 82 | .TP 83 | \fB\-P\fR NBD_PORT, \fB\-\-nbd\-port\fR NBD_PORT 84 | Port used by remote NBD Service, should be unique for each started backup. (default: 10809) 85 | .TP 86 | \fB\-I\fR NBD_IP, \fB\-\-nbd\-ip\fR NBD_IP 87 | IP used to bind remote NBD service on (default: hostname returned by libvirtd) 88 | .TP 89 | \fB\-\-tls\fR 90 | Enable and use TLS for NBD connection. (default: False) 91 | .TP 92 | \fB\-\-tls\-cert\fR TLS_CERT 93 | Path to TLS certificates used during offline backup and restore. (default: /etc/pki/qemu/) 94 | .SS "Logging options:" 95 | .TP 96 | \fB\-L\fR LOGFILE, \fB\-\-logfile\fR LOGFILE 97 | Path to Logfile (default: \fI\,/home/abi/virtnbdrestore.log\/\fP) 98 | .TP 99 | \fB\-\-nocolor\fR 100 | Disable colored output (default: False) 101 | .SS "Debug options:" 102 | .TP 103 | \fB\-v\fR, \fB\-\-verbose\fR 104 | Enable debug output 105 | .TP 106 | \fB\-V\fR, \fB\-\-version\fR 107 | Show version and exit 108 | .SH EXAMPLES 109 | .IP 110 | # Dump backup metadata: 111 | .IP 112 | virtnbdrestore \-i /backup/ \-o dump 113 | .IP 114 | # Verify checksums for existing data files in backup: 115 | .IP 116 | virtnbdrestore \-i /backup/ \-o verify 117 | .IP 118 | # Complete restore with all disks: 119 | .IP 120 | virtnbdrestore \-i /backup/ \-o /target 121 | .IP 122 | # Complete restore, adjust config and redefine vm after restore: 123 | .IP 124 | virtnbdrestore \-cD \-i /backup/ \-o /target 125 | .IP 126 | # Complete restore, adjust config and redefine vm with name 'foo': 127 | .IP 128 | virtnbdrestore \-cD \-\-name foo \-i /backup/ \-o /target 129 | .IP 130 | # Restore only disk 'vda': 131 | .IP 132 | virtnbdrestore \-i /backup/ \-o /target \-d vda 133 | .IP 134 | # Point in time restore: 135 | .IP 136 | virtnbdrestore \-i /backup/ \-o /target \-\-until virtnbdbackup.2 137 | .IP 138 | # Restore and process specific file sequence: 139 | .IP 140 | virtnbdrestore \-i /backup/ \-o /target \-\-sequence vdb.full.data,vdb.inc.virtnbdbackup.1.data 141 | .IP 142 | # Restore to remote system: 143 | .IP 144 | virtnbdrestore \-U qemu+ssh://root@remotehost/system \-\-ssh\-user root \-i /backup/ \-o /remote_target 145 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | libvirt-python>=6.0.0 2 | tqdm 3 | lz4>=2.1.2 4 | lxml 5 | paramiko 6 | typing_extensions 7 | colorlog 8 | -------------------------------------------------------------------------------- /screenshot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/abbbi/virtnbdbackup/f4f87d3c48dd1aa80a9e713cf82c293e5769021b/screenshot.jpg -------------------------------------------------------------------------------- /scripts/create-cert.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # setup certificates required for spinning up 3 | # backup jobs via NBDS 4 | set -e 5 | SYSTEM_PKIDIR=/etc/pki/qemu/ 6 | USER_PKIDIR=$HOME/.pki/libnbd/ 7 | 8 | mkdir -p "${SYSTEM_PKIDIR}" "${USER_PKIDIR}" 9 | cat < certificate_authority_template.info 10 | cn = virtnbdbackup 11 | ca 12 | cert_signing_key 13 | EOF 14 | certtool --generate-privkey > ca_key.pem 15 | certtool --generate-self-signed \ 16 | --template certificate_authority_template.info \ 17 | --load-privkey ca_key.pem \ 18 | --outfile ca-cert.pem 19 | 20 | cp ca-cert.pem ${SYSTEM_PKIDIR}/ca-cert.pem 21 | 22 | cat < host1_server_template.info 23 | organization = virtnbdbackup 24 | cn = server.example.com 25 | tls_www_server 26 | encryption_key 27 | signing_key 28 | EOF 29 | 30 | certtool --generate-privkey > host1_server_key.pem 31 | certtool --generate-certificate \ 32 | --template host1_server_template.info \ 33 | --load-privkey host1_server_key.pem \ 34 | --load-ca-certificate ca-cert.pem \ 35 | --load-ca-privkey ca_key.pem \ 36 | --outfile host1_server_certificate.pem 37 | 38 | cp host1_server_key.pem ${SYSTEM_PKIDIR}/server-key.pem 39 | cp host1_server_certificate.pem ${SYSTEM_PKIDIR}/server-cert.pem 40 | 41 | cat < host1_client_template.info 42 | country = Country 43 | state = State 44 | locality = City 45 | organization = Name of your organization 46 | cn = client.example.com 47 | tls_www_client 48 | encryption_key 49 | signing_key 50 | EOF 51 | certtool --generate-privkey > host1_client_key.pem 52 | certtool --generate-certificate \ 53 | --template host1_client_template.info \ 54 | --load-privkey host1_client_key.pem \ 55 | --load-ca-certificate ca-cert.pem \ 56 | --load-ca-privkey ca_key.pem \ 57 | --outfile host1_client_certificate.pem 58 | 59 | cp host1_client_certificate.pem "${USER_PKIDIR}"/client-cert.pem 60 | cp host1_client_key.pem "${USER_PKIDIR}"/client-key.pem 61 | cp ca-cert.pem "${USER_PKIDIR}"/ca-cert.pem 62 | 63 | sed -i 's/#backup_tls_x509_verify.*/backup_tls_x509_verify=0/' /etc/libvirt/qemu.conf 64 | systemctl restart libvirtd 65 | -------------------------------------------------------------------------------- /scripts/mangen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | help2man -n "backup utility for libvirt" ./virtnbdbackup -N > man/virtnbdbackup.1 4 | help2man -n "restore utility for libvirt" ./virtnbdrestore -N > man/virtnbdrestore.1 5 | help2man -n "map virtnbdbackup image files to nbd devices" ./virtnbdmap -N > man/virtnbdmap.1 6 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description_file = README.md 3 | long_description = Backup utility for libvirt, using latest changed block tracking features 4 | 5 | [bdist_rpm] 6 | release = 1 7 | packager = Michael Ablassmeier 8 | doc_files = README.md Changelog LICENSE 9 | 10 | # for rhel9 and above, requires must be passed 11 | # via --requires cmd to skip python3-dataclasses 12 | requires = 13 | python3-libvirt 14 | python3-libnbd 15 | python3-lxml 16 | python3-tqdm 17 | python3-lz4 18 | nbdkit-server 19 | nbdkit-python-plugin 20 | python3-dataclasses 21 | python3-paramiko 22 | python3-typing-extensions 23 | python3-colorlog 24 | qemu-img 25 | openssh-clients 26 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Setup virtnbdbackup""" 3 | from setuptools import setup, find_packages 4 | 5 | import libvirtnbdbackup 6 | 7 | with open("requirements.txt") as f: 8 | install_requires = f.read().splitlines() 9 | 10 | setup( 11 | name="virtnbdbackup", 12 | version=libvirtnbdbackup.__version__, 13 | description="Backup utility for libvirt", 14 | url="https://github.com/abbbi/virtnbdbackup/", 15 | author="Michael Ablassmeier", 16 | author_email="abi@grinser.de", 17 | license="GPL", 18 | keywords="libnbd backup libvirt", 19 | packages=find_packages(exclude=("docs", "tests", "env")), 20 | include_package_data=True, 21 | scripts=["virtnbdbackup", "virtnbdrestore", "virtnbdmap", "virtnbd-nbdkit-plugin"], 22 | install_requires=install_requires, 23 | extras_require={ 24 | "dev": [], 25 | "docs": [], 26 | "testing": [], 27 | }, 28 | classifiers=[], 29 | ) 30 | -------------------------------------------------------------------------------- /stdeb.cfg: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | with-python2 = False 3 | with-python3 = True 4 | Build-Depends = python3-libnbd, python3-libvirt, python3-lxml, python3-paramiko, python3-colorlog 5 | Depends3 = python3-libvirt, python3-lxml, python3-libnbd, python3-tqdm, python3-lz4, nbdkit, nbdkit-plugin-python, libnbd-bin, python3-paramiko, python3-colorlog, qemu-utils, openssh-client 6 | Package = virtnbdbackup 7 | Package3 = virtnbdbackup 8 | Section = admin 9 | Suite = testing 10 | Copyright-File: LICENSE 11 | -------------------------------------------------------------------------------- /t/Makefile: -------------------------------------------------------------------------------- 1 | bats = ./bats-core 2 | 3 | vm1.tests: $(bats) 4 | $(call clean) 5 | export TEST=vm1 ; ./bats-core/bin/bats tests.bats 6 | vm2.tests: $(bats) 7 | $(call clean) 8 | export TEST=vm2 ; ./bats-core/bin/bats tests.bats 9 | vm3.tests: $(bats) 10 | $(call clean) 11 | export TEST=vm3 ; ./bats-core/bin/bats tests.bats 12 | vm4.tests: $(bats) 13 | $(call clean) 14 | export TEST=vm4 ; ./bats-core/bin/bats tests.bats 15 | vm5.tests: $(bats) 16 | $(call clean) 17 | export TEST=vm5 ; ./bats-core/bin/bats tests.bats 18 | fstrim.tests: $(bats) 19 | $(call clean) 20 | export TEST=fstrim ; ./bats-core/bin/bats fstrim.bats 21 | fstrimsmall.tests: $(bats) 22 | $(call clean) 23 | export TEST=fstrimsmall ; ./bats-core/bin/bats fstrimsmall.bats 24 | fstest.tests: $(bats) 25 | $(call clean) 26 | export TEST=fstest ; ./bats-core/bin/bats fstest.bats 27 | nosparsedetect.tests: $(bats) 28 | $(call clean) 29 | export TEST=nosparsedetect ; ./bats-core/bin/bats nosparsedetect.bats 30 | 31 | 32 | all: | $(bats) vm1.tests vm2.tests vm3.tests vm4.tests fstrim.tests nosparsedetect.tests fstrimsmall.tests fstest.tests 33 | 34 | clean: 35 | $(call clean) 36 | 37 | define clean 38 | @rm -rf /tmp/testset* 39 | @rm -f /tmp/backup.zip 40 | @rm -f /tmp/*vm*.qcow2 41 | @rm -f *.cpt 42 | @rm -f *.log 43 | @rm -rf checkpoints 44 | @rm -f vmconfig*.xml 45 | @umount -lf /empty || true 46 | for vm in vm1 restore_vm1 vm2 vm3 vm4 vm5 fstrim nosparsedetect restored fstrimsmall fstest; do \ 47 | if virsh list --all | grep $$vm; then \ 48 | virsh destroy $$vm; \ 49 | virsh undefine $$vm --checkpoints-metadata --nvram --managed-save; \ 50 | fi \ 51 | done 52 | endef 53 | 54 | $(bats): 55 | @git clone https://github.com/bats-core/bats-core 56 | 57 | .PHONY: all 58 | -------------------------------------------------------------------------------- /t/README.txt: -------------------------------------------------------------------------------- 1 | Test cases, use BATS. Each VM directory contains a special crafted virtual 2 | machine with a given set of disks to be backed up. The virtual machines are 3 | started but do not alter their filesystems while running, to ensure the same 4 | data-sizes are reported for each test. 5 | 6 | If files within virtual machine should be changed, vm is destroyed and disks 7 | are mounted via guestfs tools. 8 | 9 | To execute test for certain virtual machine use: 10 | 11 | make vm1.test 12 | 13 | to execute all tests: 14 | 15 | make all 16 | 17 | using bats directly: 18 | 19 | export TEST=vm1 20 | ./bats-core/bin/bats tests.bats 21 | -------------------------------------------------------------------------------- /t/agent-exec.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # helper functions to execute commands within test VM using qemu guest agent 3 | # 4 | # like so: 5 | # 6 | # wait_for_agent VM_NAME 7 | # execute_qemu_command VM_NAME "mkdir" '["/incdata"]' 8 | # execute_qemu_command VM_NAME "/bin/ls" '["/"]'# 9 | 10 | execute_qemu_command() { 11 | vm_name="$1" 12 | command="$2" 13 | shift 2 14 | params="$@" 15 | 16 | if [ -n "$params" ]; then 17 | json_payload="{\"execute\": \"guest-exec\", \"arguments\": {\"path\": \"$command\", \"arg\": $params, \"capture-output\": true}}" 18 | else 19 | json_payload="{\"execute\": \"guest-exec\", \"arguments\": {\"path\": \"$command\", \"capture-output\": true}}" 20 | fi 21 | 22 | # Execute the command using virsh 23 | exec_result=$(virsh qemu-agent-command "$vm_name" "$json_payload" --timeout 5 2>&1) 24 | exit_code=$? 25 | 26 | if [ $exit_code -ne 0 ]; then 27 | echo "Error executing QEMU agent command: $exec_result" >&3 28 | return $exit_code 29 | fi 30 | pid=$(echo "$exec_result" | jq -r '.return.pid //empty') 31 | if [ -z "$pid" ]; then 32 | echo "Failed to retrieve PID from guest-exec response." >&3 33 | return 1 34 | fi 35 | status_payload="{\"execute\": \"guest-exec-status\", \"arguments\": {\"pid\": $pid}}" 36 | while true; do 37 | status_result=$(virsh qemu-agent-command "$vm_name" "$status_payload" --timeout 5 2>&1) 38 | if [ $? -ne 0 ]; then 39 | echo "Error fetching guest-exec-status: $status_result" >&3 40 | return 1 41 | fi 42 | exited=$(echo "$status_result" | jq -r '.return.exited') 43 | if [ "$exited" = "true" ]; then 44 | exit_code=$(echo "$status_result" | jq -r '.return.exitcode') 45 | out_data=$(echo "$status_result" | jq -r '.return."out-data" //empty' | base64 -d) 46 | err_data=$(echo "$status_result" | jq -r '.return."err-data" //empty' | base64 -d) 47 | break 48 | fi 49 | sleep 1 50 | done 51 | 52 | if [ -n "$err_data" ]; then 53 | echo "Error output: $err_data" >&3 54 | fi 55 | 56 | if [ -n "$out_data" ]; then 57 | echo "Output: $out_data" 58 | fi 59 | 60 | return "$exit_code" 61 | } 62 | 63 | wait_for_agent() { 64 | vm_name="$1" 65 | TIMEOUT=120 66 | INTERVAL=5 67 | START_TIME=$(date +%s) 68 | while true; do 69 | OUTPUT=$(virsh guestinfo "$vm_name" 2>/dev/null || true) 70 | if echo "$OUTPUT" | grep -q "arch"; then 71 | echo "Guest agent within VM is reachable." >&3 72 | return 0 73 | fi 74 | CURRENT_TIME=$(date +%s) 75 | ELAPSED_TIME=$((CURRENT_TIME - START_TIME)) 76 | if [ "$ELAPSED_TIME" -ge "$TIMEOUT" ]; then 77 | echo "Timeout reached: 2 minutes." >&3 78 | return 1 79 | fi 80 | sleep "$INTERVAL" 81 | done 82 | } 83 | -------------------------------------------------------------------------------- /t/fstest.bats: -------------------------------------------------------------------------------- 1 | if [ -z "$TEST" ]; then 2 | echo "" 3 | echo "Missing required test env" >&2 4 | echo "export TEST= to run specified tests" >&2 5 | echo "" 6 | exit 7 | fi 8 | 9 | if [ -e /root/agent ]; then 10 | source /root/agent > /dev/null 11 | fi 12 | 13 | if [ -z "$TMPDIR" ]; then 14 | export TMPDIR=$(mktemp -d) 15 | chmod go+rwx "$TMPDIR" 16 | fi 17 | 18 | load "$TEST"/config.bash 19 | 20 | 21 | setup() { 22 | aa-teardown >/dev/null || true 23 | } 24 | 25 | @test "Create VM image in ${TMPDIR}/${VM_IMAGE} based on $IMAGE_URL" { 26 | # setup and create image with filesystem 27 | mkdir -p "${TMPDIR}"/empty 28 | curl -o "$TMPDIR"/alpine.tar.gz "$IMAGE_URL" 29 | gunzip "$TMPDIR"/alpine.tar.gz 30 | rm -f "${TMPDIR}"/"${VM_IMAGE}" 31 | run virt-make-fs --partition --type=ext4 --size=+2G --format=qcow2 "${TMPDIR}"/alpine.tar "${TMPDIR}"/"${VM_IMAGE}" 32 | echo "output = ${output}" 33 | [ "$status" -eq 0 ] 34 | run modprobe nbd 35 | echo "output = ${output}" 36 | [ "$status" -eq 0 ] 37 | echo "output = ${output}" 38 | [ "$status" -eq 0 ] 39 | } 40 | @test "Setup: Define and start test VM ${VM}" { 41 | virsh destroy "${VM}" || true 42 | echo "output = ${output}" 43 | virsh undefine "${VM}" --remove-all-storage --checkpoints-metadata || true 44 | echo "output = ${output}" 45 | cp "${VM}"/"${VM}".xml "${TMPDIR}"/ 46 | sed -i "s|__TMPDIR__|${TMPDIR}|g" "${TMPDIR}"/"${VM}".xml 47 | run virsh define "${TMPDIR}"/"${VM}".xml 48 | echo "output = ${output}" 49 | run virsh start "${VM}" 50 | echo "output = ${output}" 51 | [ "$status" -eq 0 ] 52 | echo "output = ${output}" 53 | } 54 | @test "Backup: create full backup" { 55 | run ../virtnbdbackup -d "$VM" -l full -o "${TMPDIR}"/fstrim 56 | echo "output = ${output}" 57 | [[ "${output}" =~ "Saved qcow image config" ]] 58 | [ "$status" -eq 0 ] 59 | } 60 | @test "Destroy VM" { 61 | run virsh destroy "$VM" 62 | [ "$status" -eq 0 ] 63 | } 64 | @test "Map image via NBD, run fstrim" { 65 | run qemu-nbd -c /dev/nbd5 "${TMPDIR}"/"${VM_IMAGE}" --cache=directsync --discard=unmap 66 | echo "output = ${output}" 67 | [ "$status" -eq 0 ] 68 | run sleep 5 && mount /dev/nbd5p1 "${TMPDIR}"/empty 69 | echo "output = ${output}" 70 | [ "$status" -eq 0 ] 71 | run fstrim -v "${TMPDIR}"/empty/ 72 | echo "trimmed = ${output}" >&3 73 | [ "$status" -eq 0 ] 74 | run umount "${TMPDIR}"/empty 75 | echo "output = ${output}" 76 | [ "$status" -eq 0 ] 77 | run qemu-nbd -d /dev/nbd5p1 78 | echo "output = ${output}" 79 | [ "$status" -eq 0 ] 80 | } 81 | @test "Start VM" { 82 | run virsh start "$VM" 83 | echo "output = ${output}" 84 | [ "$status" -eq 0 ] 85 | } 86 | @test "Backup: incremental backup" { 87 | run ../virtnbdbackup -d "$VM" -l inc -o "${TMPDIR}"/fstrim 88 | echo "output = ${output}" 89 | [ "$status" -eq 0 ] 90 | } 91 | @test "Destroy VM 2" { 92 | run virsh destroy "$VM" 93 | [ "$status" -eq 0 ] 94 | } 95 | @test "Map image via NBD, create data, delete data, run fstrim" { 96 | run qemu-nbd -c /dev/nbd5 "${TMPDIR}"/"${VM_IMAGE}" --cache=directsync --discard=unmap 97 | echo "output = ${output}" 98 | [ "$status" -eq 0 ] 99 | run sleep 5 && mount /dev/nbd5p1 "${TMPDIR}"/empty 100 | echo "output = ${output}" 101 | [ "$status" -eq 0 ] 102 | run cp -a "${TMPDIR}"/empty/etc "${TMPDIR}"/empty/etc_before_fstrim2 && sync 103 | echo "output = ${output}" 104 | [ "$status" -eq 0 ] 105 | run rm -rf "${TMPDIR}"/empty/* && sync 106 | run umount "${TMPDIR}"/empty 107 | run sleep 5 && mount /dev/nbd5p1 "${TMPDIR}"/empty 108 | echo "output = ${output}" 109 | [ "$status" -eq 0 ] 110 | run fstrim -v "${TMPDIR}"/empty 111 | echo "trimmed = ${output}" >&3 112 | [ "$status" -eq 0 ] 113 | run tar -xf "${TMPDIR}"/alpine.tar -C "${TMPDIR}"/empty/ && sync 114 | echo "output = ${output}" 115 | [ "$status" -eq 0 ] 116 | run umount "${TMPDIR}"/empty 117 | echo "output = ${output}" 118 | [ "$status" -eq 0 ] 119 | run qemu-nbd -d /dev/nbd5p1 120 | echo "output = ${output}" 121 | [ "$status" -eq 0 ] 122 | } 123 | @test "Copy image for reference" { 124 | run cp "${TMPDIR}"/"${VM_IMAGE}" "${TMPDIR}"/reference_before_backup.qcow2 125 | [ "$status" -eq 0 ] 126 | } 127 | @test "Start VM again" { 128 | run virsh start "$VM" 129 | echo "output = ${output}" 130 | [ "$status" -eq 0 ] 131 | } 132 | @test "Backup: incremental backup again" { 133 | run ../virtnbdbackup -d "$VM" -l inc -o "${TMPDIR}"/fstrim 134 | echo "output = ${output}" 135 | [ "$status" -eq 0 ] 136 | } 137 | @test "Restore" { 138 | run ../virtnbdrestore -i "${TMPDIR}"/fstrim -o "${TMPDIR}"/restore 139 | echo "output = ${output}" 140 | [ "$status" -eq 0 ] 141 | } 142 | @test "Run filesystem check in restored image" { 143 | run guestfish -a "${TMPDIR}"/restore/fstest.qcow2 <<_EOF_ 144 | run 145 | fsck ext4 /dev/sda1 146 | _EOF_ 147 | [ "$status" -eq 0 ] 148 | [[ "${output}" = "0" ]] 149 | } 150 | @test "Compare data in reference image against restored image" { 151 | run guestfish -a "${TMPDIR}"/reference_before_backup.qcow2 -m /dev/sda1 tar-out / "${TMPDIR}"/reference_data.tar 152 | echo "output = ${output}" 153 | [ "$status" -eq 0 ] 154 | run guestfish -a "${TMPDIR}"/restore/fstest.qcow2 -m /dev/sda1 tar-out / "${TMPDIR}"/restore_data.tar 155 | echo "output = ${output}" 156 | [ "$status" -eq 0 ] 157 | run cmp "${TMPDIR}"/restore_data.tar "${TMPDIR}"/reference_data.tar 158 | echo "output = ${output}" 159 | [ "$status" -eq 0 ] 160 | } 161 | -------------------------------------------------------------------------------- /t/fstest/config.bash: -------------------------------------------------------------------------------- 1 | IMAGE_URL="https://dl-cdn.alpinelinux.org/alpine/v3.21/releases/x86_64/alpine-minirootfs-3.21.3-x86_64.tar.gz" 2 | VM="fstest" 3 | VM_IMAGE="fstest.qcow2" 4 | -------------------------------------------------------------------------------- /t/fstrim/README.txt: -------------------------------------------------------------------------------- 1 | This test uses the arch Linux cloud image to create multiple backups with 2 | different actions in between. 3 | 4 | The image includes an Qemu agent that is used to alter data within the virtual 5 | machine while it is active. Actions between incremental backups involve: 6 | 7 | * alter some data by copying /etc,/usr to different target directory 8 | * extract the created files from the image using virt-tar-out 9 | * fstrim 10 | * create some folders 11 | * delete some data 12 | * fstrim again 13 | * create ~500 MB file and store checksum 14 | * restore disk image and check contents, extract the restored files and 15 | compare them against the reference files 16 | * boot virtual machine 17 | * verify checksum of created data file within booted VM 18 | 19 | Also, the image responds very well to fstrim, attempting to trim about 38 GB of 20 | data right from the start, which makes it perfect to test against the dirty 21 | zero regions during backup. 22 | 23 | Restore attempts to boot virtual machine and also checks restored data for 24 | existence and verifies checksums. 25 | -------------------------------------------------------------------------------- /t/fstrim/config.bash: -------------------------------------------------------------------------------- 1 | VM="fstrim" 2 | VM_IMAGE="fstrim.qcow2" 3 | IMG_URL="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-basic.qcow2" 4 | -------------------------------------------------------------------------------- /t/fstrimsmall/README.txt: -------------------------------------------------------------------------------- 1 | Testcase for trimmed block detection, using qemu-io to write zeroed and data 2 | blocks to the image and check if extent handler detects correct amount of 3 | blocks to be backed up. 4 | -------------------------------------------------------------------------------- /t/fstrimsmall/config.bash: -------------------------------------------------------------------------------- 1 | VM="fstrimsmall" 2 | VM_IMAGE="fstrimsmall.qcow2" 3 | -------------------------------------------------------------------------------- /t/genimage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Can be used to simulate disk images with lots of extents. 3 | 4 | # Image file name 5 | IMAGE="test-image.qcow2" 6 | # Image size in KB (300GB converted to KB as an example, adjust if needed) 7 | SIZE_KB=$((300 * 1024 * 1024)) 8 | # Block size in KB (64k) 9 | BLOCK_SIZE_KB=64 10 | # Skip size in KB (128k) 11 | SKIP_SIZE_KB=128 12 | # Step size in KB (block + skip = 192k) 13 | STEP_KB=$((BLOCK_SIZE_KB + SKIP_SIZE_KB)) 14 | # Starting offset in KB (64k) 15 | START_OFFSET_KB=64 16 | # Write data or zero blocks: default: zero, remove to write data 17 | WRITEOPT="-z" 18 | 19 | # Create the qcow2 image if it doesn't exist 20 | # Convert size to bytes for qemu-img 21 | SIZE_BYTES=$((SIZE_KB * 1024)) 22 | if [ ! -f "$IMAGE" ]; then 23 | if ! qemu-img create -f qcow2 "$IMAGE" "$SIZE_BYTES"; then 24 | echo "Failed to create image" 25 | exit 1 26 | fi 27 | fi 28 | 29 | # Calculate number of iterations 30 | # Account for starting offset in calculation 31 | ITERATIONS=$(((SIZE_KB - START_OFFSET_KB) / STEP_KB)) 32 | 33 | echo "Starting write operations..." 34 | echo "Image size: $SIZE_KB KB" 35 | echo "Block size: $BLOCK_SIZE_KB KB" 36 | echo "Skip size: $SKIP_SIZE_KB KB" 37 | echo "Starting offset: $START_OFFSET_KB KB" 38 | echo "Total iterations: $ITERATIONS" 39 | 40 | # Counter for progress 41 | count=0 42 | 43 | # Loop to write 64k blocks of zeroes with 128k skips, starting at 64k 44 | offset_kb=$START_OFFSET_KB 45 | while [ $offset_kb -lt $SIZE_KB ]; do 46 | # Write 64k of zeroes at current offset 47 | if ! qemu-io -c "write ${WRITEOPT} ${offset_kb}k ${BLOCK_SIZE_KB}k" "$IMAGE" > /dev/null 2>&1; then 48 | echo "Write failed at offset ${offset_kb}k" 49 | exit 1 50 | fi 51 | 52 | # Show progress every 1000 iterations 53 | if [ $((count % 1000)) -eq 0 ]; then 54 | echo "Progress: $count/$ITERATIONS (offset: ${offset_kb}k)" 55 | fi 56 | 57 | # Move to next position 58 | offset_kb=$((offset_kb + STEP_KB)) 59 | count=$((count + 1)) 60 | done 61 | 62 | echo "Write operations completed" 63 | echo "Wrote $count blocks of ${BLOCK_SIZE_KB}k zeroes with ${SKIP_SIZE_KB}k skips" 64 | -------------------------------------------------------------------------------- /t/nosparsedetect.bats: -------------------------------------------------------------------------------- 1 | if [ -z "$TEST" ]; then 2 | echo "" 3 | echo "Missing required test env" >&2 4 | echo "export TEST= to run specified tests" >&2 5 | echo "" 6 | exit 7 | fi 8 | 9 | if [ -e /root/agent ]; then 10 | source /root/agent > /dev/null 11 | fi 12 | 13 | if [ -z "$TMPDIR" ]; then 14 | export TMPDIR=$(mktemp -d) 15 | chmod go+rwx $TMPDIR 16 | fi 17 | 18 | load $TEST/config.bash 19 | load agent-exec.sh 20 | 21 | 22 | setup() { 23 | aa-teardown >/dev/null || true 24 | } 25 | 26 | @test "Setup / download vm image ${IMG_URL} to ${TMPDIR}/${VM_IMAGE}" { 27 | if [ ! -e ${TMPDIR}/${VM_IMAGE} ]; then 28 | curl -Ls ${IMG_URL} -o ${TMPDIR}/${VM_IMAGE} 29 | fi 30 | } 31 | 32 | @test "Setup: Define and start test VM ${VM}" { 33 | virsh destroy ${VM} || true 34 | echo "output = ${output}" 35 | virsh undefine ${VM} --remove-all-storage --checkpoints-metadata || true 36 | echo "output = ${output}" 37 | cp ${VM}/${VM}.xml ${TMPDIR}/ 38 | sed -i "s|__TMPDIR__|${TMPDIR}|g" ${TMPDIR}/${VM}.xml 39 | run virsh define ${TMPDIR}/${VM}.xml 40 | echo "output = ${output}" 41 | run virsh start ${VM} 42 | echo "output = ${output}" 43 | [ "$status" -eq 0 ] 44 | echo "output = ${output}" 45 | } 46 | @test "Wait for VM to be reachable via guest agent" { 47 | run wait_for_agent $VM 48 | } 49 | @test "Backup: create full backup" { 50 | run ../virtnbdbackup -d $VM -l full -o ${TMPDIR}/fstrim 51 | [[ "${output}" =~ "Saved qcow image config" ]] 52 | [ "$status" -eq 0 ] 53 | } 54 | @test "Create data in VM 1" { 55 | run execute_qemu_command $VM "cp" '["-a", "/etc", "/incdata"]' 56 | echo "output = ${output}" 57 | [ "$status" -eq 0 ] 58 | run execute_qemu_command $VM sync 59 | [ "$status" -eq 0 ] 60 | } 61 | @test "Execute fstrim" { 62 | run execute_qemu_command $VM "fstrim" '["-v", "/"]' 63 | echo "trimmed = ${output}" >&3 64 | [ "$status" -eq 0 ] 65 | } 66 | @test "Create data in VM 2" { 67 | run execute_qemu_command $VM "cp" '["-a", "/etc", "/incdata2"]' 68 | echo "output = ${output}" 69 | [ "$status" -eq 0 ] 70 | run execute_qemu_command $VM sync 71 | [ "$status" -eq 0 ] 72 | } 73 | @test "Backup: create inc backup with sparse detection" { 74 | run ../virtnbdbackup -d $VM -l inc -o ${TMPDIR}/fstrim 75 | echo "output = ${output}" 76 | [[ "${output}" =~ "sparse blocks for current bitmap" ]] 77 | [ "$status" -eq 0 ] 78 | } 79 | @test "Create data in VM 3" { 80 | run execute_qemu_command $VM "cp" '["-a", "/etc", "/incdata3"]' 81 | [ "$status" -eq 0 ] 82 | run execute_qemu_command $VM sync 83 | [ "$status" -eq 0 ] 84 | } 85 | @test "Backup: create inc backup without sparse detection" { 86 | run ../virtnbdbackup -d $VM -l inc --no-sparse-detect -o ${TMPDIR}/fstrim 87 | echo "output = ${output}" 88 | [[ "${output}" =~ "Skipping detection of sparse" ]] 89 | [ "$status" -eq 0 ] 90 | } 91 | @test "Remove data in VM" { 92 | run execute_qemu_command $VM "rm" '["-rf", "/incdata3"]' 93 | [ "$status" -eq 0 ] 94 | run execute_qemu_command $VM sync 95 | [ "$status" -eq 0 ] 96 | } 97 | @test "Execute fstrim 2" { 98 | run execute_qemu_command $VM "fstrim" '["-v", "/"]' 99 | echo "trimmed = ${output}" >&3 100 | [ "$status" -eq 0 ] 101 | } 102 | @test "Backup: create inc backup again, with sparse detection" { 103 | run ../virtnbdbackup -d $VM -l inc -o ${TMPDIR}/fstrim 104 | echo "output = ${output}" 105 | [[ "${output}" =~ "sparse blocks for current bitmap" ]] 106 | [ "$status" -eq 0 ] 107 | } 108 | @test "Create data in VM 4 and create checksum" { 109 | run execute_qemu_command $VM "dd" '["if=/dev/urandom", "of=/testdata", "bs=1M", "count=500"]' 110 | [ "$status" -eq 0 ] 111 | run execute_qemu_command $VM sync 112 | [ "$status" -eq 0 ] 113 | run execute_qemu_command $VM "md5sum" '["/testdata"]' 114 | [ "$status" -eq 0 ] 115 | echo "checksum = ${output}" 116 | echo ${output} > ${TMPDIR}/data.sum 117 | } 118 | @test "Backup: create inc backup 4, with sparse detection" { 119 | run ../virtnbdbackup -d $VM -l inc --no-sparse-detect -o ${TMPDIR}/fstrim 120 | echo "output = ${output}" 121 | [ "$status" -eq 0 ] 122 | } 123 | @test "Run checksum verify" { 124 | run ../virtnbdrestore -a verify -i ${TMPDIR}/fstrim -o /tmp 125 | echo "output = ${output}" 126 | [ "$status" -eq 0 ] 127 | } 128 | @test "Restore: restore vm with new name" { 129 | run ../virtnbdrestore -cD --name restored -i ${TMPDIR}/fstrim -o ${TMPDIR}/restore 130 | echo "output = ${output}" 131 | [ "$status" -eq 0 ] 132 | } 133 | @test "Verify image contents" { 134 | run virt-ls -a ${TMPDIR}/restore/nosparsedetect.qcow2 / 135 | [[ "${output}" =~ "incdata" ]] 136 | [ "$status" -eq 0 ] 137 | 138 | run virt-ls -a ${TMPDIR}/restore/nosparsedetect.qcow2 /incdata 139 | [[ "${output}" =~ "sudoers" ]] 140 | [ "$status" -eq 0 ] 141 | 142 | run virt-cat -a ${TMPDIR}/restore/nosparsedetect.qcow2 /incdata/sudoers 143 | [[ "${output}" =~ "Uncomment" ]] 144 | [ "$status" -eq 0 ] 145 | 146 | run virt-ls -a ${TMPDIR}/restore/nosparsedetect.qcow2 /incdata2 147 | [[ "${output}" =~ "sudoers" ]] 148 | [ "$status" -eq 0 ] 149 | 150 | run virt-ls -a ${TMPDIR}/restore/nosparsedetect.qcow2 /incdata3 151 | [ "$status" -ne 0 ] 152 | } 153 | @test "Start restored VM" { 154 | run virsh start restored 155 | [ "$status" -eq 0 ] 156 | } 157 | @test "Verify restored VM boots" { 158 | run wait_for_agent restored 159 | [ "$status" -eq 0 ] 160 | } 161 | @test "Verify checksums of data backed up during incremental backup" { 162 | run execute_qemu_command restored "md5sum" '["/testdata"]' 163 | [ "$status" -eq 0 ] 164 | echo "checksum = ${output}" 165 | echo ${output} > ${TMPDIR}/restored.sum 166 | run cmp ${TMPDIR}/restored.sum ${TMPDIR}/data.sum 167 | echo "output = ${output}" 168 | [ "$status" -eq 0 ] 169 | } 170 | @test "Check filesystem consistency after boot" { 171 | run execute_qemu_command restored btrfs '["device","stats","-c", "/"]' 172 | [ "$status" -eq 0 ] 173 | echo "output = ${output}" 174 | } 175 | -------------------------------------------------------------------------------- /t/nosparsedetect/README.txt: -------------------------------------------------------------------------------- 1 | This test uses the arch Linux cloud image to create multiple backups with 2 | different actions in between. 3 | 4 | This testcase is used to do mixed backups with --no-sparse-detect option 5 | on a virtual machine. 6 | -------------------------------------------------------------------------------- /t/nosparsedetect/config.bash: -------------------------------------------------------------------------------- 1 | VM="nosparsedetect" 2 | VM_IMAGE="nosparsedetect.qcow2" 3 | IMG_URL="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-basic.qcow2" 4 | -------------------------------------------------------------------------------- /t/vm1/.gitignore: -------------------------------------------------------------------------------- 1 | vm1-sda.qcow2.gz 2 | -------------------------------------------------------------------------------- /t/vm1/UEFI_VARS.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/abbbi/virtnbdbackup/f4f87d3c48dd1aa80a9e713cf82c293e5769021b/t/vm1/UEFI_VARS.gz -------------------------------------------------------------------------------- /t/vm1/config.bash: -------------------------------------------------------------------------------- 1 | QEMU_FILE=${TMPDIR}/convert.full.raw 2 | CONVERT_FILE=${TMPDIR}/restored.full.raw 3 | BACKUPSET=${TMPDIR}/testset 4 | RESTORESET=${TMPDIR}/restoreset 5 | VM="vm1" 6 | VM_IMAGE="${VM}/vm1-sda.qcow2" 7 | 8 | VM_UEFI_VARS="${VM}/UEFI_VARS.gz" 9 | 10 | 11 | # following outputs are expected for this vm image 12 | DATA_SIZE="6094848" 13 | VIRTUAL_SIZE="52428800" 14 | EXTENT_OUTPUT1="Got 7 extents to backup." 15 | EXTENT_OUTPUT2="${VIRTUAL_SIZE} bytes" 16 | EXTENT_OUTPUT3="${DATA_SIZE} bytes" 17 | 18 | INCTEST="1" 19 | MAPTEST="1" 20 | -------------------------------------------------------------------------------- /t/vm1/vm1-sda.qcow2.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/abbbi/virtnbdbackup/f4f87d3c48dd1aa80a9e713cf82c293e5769021b/t/vm1/vm1-sda.qcow2.gz -------------------------------------------------------------------------------- /t/vm2/config.bash: -------------------------------------------------------------------------------- 1 | QEMU_FILE=${TMPDIR}/convert.full.raw 2 | CONVERT_FILE=${TMPDIR}/restored.full.raw 3 | BACKUPSET=${TMPDIR}/testset 4 | RESTORESET=${TMPDIR}/restoreset 5 | VM="vm2" 6 | # lets use an openstack image for testing, 7 | # as the defined virtual machine has way 8 | # too less memory, it wont boot so no changes 9 | # are applied to the image 10 | # VM image is qcow2 so no persistent bitmaps 11 | # are supported, create only copy backups 12 | VM_IMAGE_URL="https://chuangtzu.ftp.acc.umu.se/cdimage/openstack/archive/10.6.0/debian-10.6.0-openstack-amd64.qcow2" 13 | VM_IMAGE="${VM}/vm2-sda.qcow2" 14 | 15 | if [ ! -e $VM_IMAGE ]; then 16 | echo "downloading test image" 17 | curl $VM_IMAGE_URL > $VM_IMAGE 18 | fi 19 | 20 | # convert downloaded image toqcow format supporting persistent 21 | # bitmaps, to allow full backup 22 | if qemu-img info $VM_IMAGE | grep "compat: 0.10" >/dev/null; then 23 | qemu-img convert -O qcow2 $VM_IMAGE "${VM_IMAGE}.new" 24 | mv "${VM_IMAGE}.new" "${VM_IMAGE}" 25 | fi 26 | 27 | EXTENT_OUTPUT1="Got 866 extents to backup." 28 | EXTENT_OUTPUT2="2147483648 bytes" 29 | EXTENT_OUTPUT3="1394147328 bytes" 30 | 31 | DATA_SIZE="1394147328" 32 | VIRTUAL_SIZE="2147483648" 33 | -------------------------------------------------------------------------------- /t/vm2/vm2.xml: -------------------------------------------------------------------------------- 1 | 2 | vm2 3 | 4 | 5 | 6 | 7 | 8 | 20480 9 | 20480 10 | 1 11 | 12 | /machine 13 | 14 | 15 | hvm 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | destroy 29 | restart 30 | destroy 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 |
43 | 44 | 45 | 46 |
47 | 48 | 49 | 50 |
51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 |
60 | 61 | 62 | 63 | 64 | 65 |
66 | 67 | 68 | 69 | 70 | 71 |
72 | 73 | 74 | 75 | 76 | 77 |
78 | 79 | 80 | 81 | 82 | 83 |
84 | 85 | 86 | 87 | 88 | 89 |
90 | 91 | 92 | 93 | 94 | 95 |
96 | 97 | 98 | 99 |
100 | 101 | 102 | 103 |
104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 |
122 | 123 | 124 | 125 | 126 |
127 | 128 | 129 | 130 |
131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 |
145 | 146 |