├── .devcontainer ├── Dockerfile └── devcontainer.json ├── .gitignore ├── .travis.yml ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── poetry.lock ├── pyproject.toml ├── pytest.ini ├── runrestic ├── __init__.py ├── metrics │ ├── __init__.py │ └── prometheus.py ├── restic │ ├── __init__.py │ ├── installer.py │ ├── output_parsing.py │ ├── runner.py │ ├── shell.py │ └── tools.py └── runrestic │ ├── __init__.py │ ├── configuration.py │ ├── runrestic.py │ ├── schema.json │ └── tools.py ├── sample ├── cron │ └── runrestic ├── example.toml └── systemd │ ├── runrestic.service │ └── runrestic.timer ├── setup.py └── tests ├── __init__.py ├── metrics └── test_metrics.py ├── restic ├── __init__.py ├── test_installer.py ├── test_output_parsing.py ├── test_restic_tools.py ├── test_runner.py ├── test_shell.py └── test_tools_subprocess.py └── runrestic ├── test_configuration.py ├── test_runrestic.py └── test_runrestic_tools.py /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.234.0/containers/ubuntu/.devcontainer/base.Dockerfile 2 | 3 | # [Choice] Ubuntu version (use ubuntu-22.04 or ubuntu-18.04 on local arm64/Apple Silicon): ubuntu-22.04, ubuntu-20.04, ubuntu-18.04 4 | ARG VARIANT="noble" 5 | FROM mcr.microsoft.com/vscode/devcontainers/base:0-${VARIANT} 6 | 7 | # [Optional] Uncomment this section to install additional OS packages. 8 | RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ 9 | && apt-get -y install --no-install-recommends software-properties-common 10 | # Install more python versions for testing purposes 11 | RUN add-apt-repository ppa:deadsnakes/ppa \ 12 | && apt-get update \ 13 | && export DEBIAN_FRONTEND=noninteractive \ 14 | && apt-get -y install --no-install-recommends python3.10-venv python3.11-venv python3.12-venv python3.13-venv pipx 15 | 16 | # Setup default python tools in a venv via pipx to avoid conflicts 17 | ENV PIPX_HOME=/usr/local/py-utils \ 18 | PIPX_BIN_DIR=/usr/local/py-utils/bin 19 | ENV PATH=${PATH}:${PIPX_BIN_DIR} 20 | RUN pipx install black 21 | RUN pipx install isort 22 | RUN pipx install mypy 23 | RUN pipx install poetry 24 | RUN pipx install pycodestyle 25 | RUN pipx install pydocstyle 26 | RUN pipx install pylint 27 | 28 | # Install Python Poetry 29 | USER vscode 30 | # create the virtualenv in the project directory so that they are no longer ephemeral 31 | RUN poetry config virtualenvs.in-project false 32 | RUN poetry config virtualenvs.path ./.virtualenvs 33 | -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | // For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: 2 | // https://github.com/microsoft/vscode-dev-containers/tree/v0.234.0/containers/ubuntu 3 | { 4 | "name": "Ubuntu", 5 | "build": { 6 | "dockerfile": "Dockerfile", 7 | // Update 'VARIANT' to pick an Ubuntu version: jammy / ubuntu-22.04, focal / ubuntu-20.04, bionic /ubuntu-18.04 8 | // Use ubuntu-22.04 or ubuntu-18.04 on local arm64/Apple Silicon. 9 | "args": { 10 | "VARIANT": "ubuntu-22.04" 11 | } 12 | }, 13 | // Set *default* container specific settings.json values on container create. 14 | "settings": { 15 | "python.defaultInterpreterPath": "/usr/local/bin/python", 16 | "python.languageServer": "Pylance", 17 | "python.linting.enabled": true, 18 | "python.linting.pylintEnabled": true, 19 | // "python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8", 20 | "python.formatting.blackPath": "/usr/local/py-utils/bin/black", 21 | // "python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf", 22 | // "python.linting.banditPath": "/usr/local/py-utils/bin/bandit", 23 | // "python.linting.flake8Path": "/usr/local/py-utils/bin/flake8", 24 | "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", 25 | "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", 26 | "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", 27 | "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint" 28 | }, 29 | // Add the IDs of extensions you want installed when the container is created. 30 | "extensions": [ 31 | "ms-python.python", 32 | "ms-python.vscode-pylance", 33 | "ms-python.black-formatter", 34 | "ms-python.isort", 35 | "donjayamanne.githistory", 36 | "eamodio.gitlens", 37 | "davidanson.vscode-markdownlint", 38 | "njpwerner.autodocstring", 39 | "streetsidesoftware.code-spell-checker" 40 | ], 41 | // Use 'forwardPorts' to make a list of ports inside the container available locally. 42 | // "forwardPorts": [], 43 | // Use 'postCreateCommand' to run commands after the container is created. 44 | // "postCreateCommand": "uname -a", 45 | // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. 46 | "remoteUser": "vscode", 47 | // Extra settings for `podman` 48 | "runArgs": [ 49 | "--userns=keep-id" 50 | ], 51 | "containerUser": "vscode" // the value needs to match the value of "remoteUser" 52 | } 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # IntelliJ 2 | .idea/ 3 | 4 | # ---> VisualStudioCode 5 | .vscode/* 6 | !.vscode/settings.json 7 | !.vscode/tasks.json 8 | !.vscode/launch.json 9 | !.vscode/extensions.json 10 | !.vscode/*.code-snippets 11 | 12 | # Python 13 | __pycache__/ 14 | *.egg-info/ 15 | .coverage* 16 | .mypy_cache/ 17 | .pytest_cache/ 18 | dist/ 19 | build/ 20 | .venv 21 | .virtualenvs 22 | build/ 23 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: bionic 2 | language: python 3 | python: 4 | - "3.10" 5 | - "3.11" 6 | - "3.12" 7 | - "3.13" 8 | # - "pypy3" lets not do pypy right now 9 | 10 | before_install: 11 | - pip install poetry 12 | # Fix issue with Python 3.7 and 3.8 build failure caused by "setuptools" 13 | # AttributeError: type object 'Distribution' has no attribute '_finalize_feature_opts' 14 | - pip install setuptools==60.8.2 15 | 16 | install: 17 | - poetry install 18 | 19 | script: 20 | - poetry run black . --check 21 | - poetry run mypy runrestic --ignore-missing-imports --strict 22 | - poetry run pytest 23 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "cSpell.words": [ 3 | "runrestic" 4 | ], 5 | "autoDocstring.docstringFormat": "numpy", 6 | "python.linting.pylintArgs": [ 7 | "--max-line-length=88" 8 | ], 9 | "python.formatting.provider": "black", 10 | "python.formatting.blackArgs": [ 11 | "-l 88" 12 | ], 13 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![python version](https://img.shields.io/badge/python-3.7+-blue.svg) 2 | [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) 3 | ![Travis (.com)](https://api.travis-ci.com/sinnwerkstatt/runrestic.svg?branch=main) 4 | ![PyPI](https://img.shields.io/pypi/v/runrestic) 5 | [![Stackshare: runrestic](https://img.shields.io/badge/stackshare-runrestic-068DFE.svg)](https://stackshare.io/runrestic) 6 | ![PyPI - Downloads](https://img.shields.io/pypi/dm/runrestic) 7 | 8 | # Runrestic 9 | 10 | runrestic is a simple Python wrapper script for the 11 | [Restic](https://restic.net/) backup software that initiates a backup, 12 | prunes any old backups according to a retention policy, and validates backups 13 | for consistency. The script supports specifying your settings in a declarative 14 | configuration file rather than having to put them all on the command-line, and 15 | handles common errors. 16 | 17 | ## Example config 18 | 19 | ```toml 20 | repositories = [ 21 | "/tmp/restic-repo", 22 | "sftp:user@host:/srv/restic-repo", 23 | "s3:s3.amazonaws.com/bucket_name" 24 | ] 25 | 26 | [environment] 27 | RESTIC_PASSWORD = "CHANGEME" 28 | 29 | [backup] 30 | sources = [ 31 | "/home", 32 | "/var" 33 | ] 34 | 35 | [prune] 36 | keep-last = 3 37 | keep-hourly = 5 38 | ``` 39 | 40 | Alternatively you can also just use JSON. For a more comprehensive example see the [example.toml](https://github.com/sinnwerkstatt/runrestic/blob/main/sample/example.toml) 41 | or check the [schema.json](https://github.com/sinnwerkstatt/runrestic/blob/main/runrestic/runrestic/schema.json) 42 | 43 | ## Getting started 44 | 45 | ### Installing runrestic and restic 46 | 47 | To install **runrestic**, run the following command to download and install it: 48 | 49 | ```bash 50 | sudo pip3 install --upgrade runrestic 51 | ``` 52 | 53 |
54 | You can either manually download and install [Restic](https://restic.net/) or you can just run `runrestic` and it'll try to download it for you. 55 | 56 | ### Initializing and running 57 | 58 | Once you have `restic` and `runrestic` ready, you should put a config file in on of the scanned locations, namely: 59 | 60 | - /etc/runrestic.toml 61 | - /etc/runrestic/_example_.toml 62 | - ~/.config/runrestic/_example_.toml 63 | - /etc/runrestic.json 64 | - /etc/runrestic/_example_.json 65 | - ~/.config/runrestic/_example_.json 66 | 67 | Afterwards, run 68 | 69 | ```bash 70 | runrestic init # to initialize all the repos in `repositories` 71 | 72 | runrestic # without actions will do: runrestic backup prune check 73 | # or 74 | runrestic [action] 75 | ``` 76 | 77 |
78 | Certain `restic` flags like `--dry-run/-n` are built into `runrestic` as well and will be passed to restic where applicable. 79 | 80 | If, however, you need to pass along arbitrary other flags you can now add them to the end of your `runrestic` call like so: 81 | 82 | ```bash 83 | runrestic backup -- --one-file-system 84 | ``` 85 | 86 | #### Logs for restic and hooks 87 | 88 | The output of `restic` and the configured pre/post-hooks is added to the `runrestic` logs at the level defined in 89 | `[execution] proc_log_level` (default: DEBUG), which can be overwritten with the CLI option `-p/--proc-log-level`. 90 | 91 | For process log levels greater than `INFO` the output of file names is suppressed and for log levels greater than WARNING 92 | `restic` is executed with the `--quiet` option. If the process log level is set to `DEBUG`, then restic is executed 93 | with the `--verbose` option. 94 | 95 | It is also possible to add `restic` progress messages to the logs by using the CLI option `--show-progress INTERVAL` 96 | where the `INTERVAL` is the number of seconds between the progress messages. 97 | 98 | ### Restic shell 99 | 100 | To use the options defined in `runrestic` with `restic` (e.g. for a backup restore), you can use the `shell` action: 101 | 102 | ```bash 103 | runrestic shell 104 | ``` 105 | 106 | If you are using multiple repositories or configurations, you can select one now. 107 | 108 | ### Prometheus / Grafana metrics 109 | 110 | [@d-matt](https://github.com/d-matt) created a nice dashboard for Grafana here: https://grafana.com/grafana/dashboards/11064/revisions 111 | 112 | ### systemd timer or cron 113 | 114 | If you want to run runrestic automatically, say once a day, the you can 115 | configure a job runner to invoke it periodically. 116 | 117 | #### systemd 118 | 119 | If you're using systemd instead of cron to run jobs, download the [sample systemd service file](https://raw.githubusercontent.com/sinnwerkstatt/runrestic/main/sample/systemd/runrestic.service) 120 | and the [sample systemd timer file](https://raw.githubusercontent.com/sinnwerkstatt/runrestic/main/sample/systemd/runrestic.timer). 121 | Then, from the directory where you downloaded them: 122 | 123 | ```bash 124 | sudo mv runrestic.service runrestic.timer /etc/systemd/system/ 125 | sudo systemctl enable runrestic.timer 126 | sudo systemctl start runrestic.timer 127 | ``` 128 | 129 | #### cron 130 | 131 | If you're using cron, download the [sample cron file](https://raw.githubusercontent.com/sinnwerkstatt/runrestic/main/sample/cron/runrestic). 132 | Then, from the directory where you downloaded it: 133 | 134 | ```bash 135 | sudo mv runrestic /etc/cron.d/runrestic 136 | sudo chmod +x /etc/cron.d/runrestic 137 | ``` 138 | 139 | ## Changelog 140 | 141 | - pre-v0.5.31 142 | - Drop support for Python 3.8 (EOL 2024-10-07) and 3.9 (EOL 2025-10) 143 | - Add and update docstrings 144 | - Add and update type hints 145 | - v0.5.30 146 | - Fix metric setting in restic runner for "check" 147 | - Support Python 3.13 148 | - Add Python 3.13 in devcontainer so that it can be used for testing 149 | - Updated Poetry lock 150 | - Enhance test coverage 151 | - Modified restic tools test to use mock file operations and shortened retry times for faster test execution 152 | - v0.5.29 153 | - Support Python 3.12 154 | - Updated devcontainer to Ubuntu 24.04 (noble) 155 | - v0.5.28 156 | - Allow jsonschema >= 4.0 157 | - v0.5.27 158 | - Fix output parsing for new restic version 0.14.0 159 | - Introduce failsafe output parser which supports default values 160 | - v0.5.26 161 | - Add output messages from `restic` and pre/post-hook commands to runrestic logs. 162 | - New CLI argument `--show-progress INTERVAL` for the restic progress update interval in seconds (default None) 163 | - v0.5.25 164 | - Drop support for Python 3.6, add support for Python 3.9 and 3.10, update dependencies 165 | - v0.5.24 166 | - Exit the script with returncode = 1 if there was an error in any of the tasks 167 | - v0.5.23 168 | - support JSON config files. 169 | - v0.5.21 170 | 171 | - fix issue where "check" does not count towards overall "errors"-metric 172 | 173 | - v**0.5**! Expect breaking changes. 174 | - metrics output is a bit different 175 | - see new `parallel` and `retry_*` options. 176 | 177 | ## Ansible 178 | 179 | @tabic wrote an ansible role, you can find it here: https://github.com/outwire/ansible-role-restic . (I have neither checked nor tested it.) 180 | 181 | ## Development 182 | 183 | This project is managed with [poetry](https://python-poetry.org/) 184 | 185 | [Install it](https://github.com/python-poetry/poetry#installation) if not already present: 186 | 187 | ```bash 188 | curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python 189 | # or 190 | pip install --user poetry 191 | ``` 192 | 193 | ### Installing dependencies 194 | 195 | ```bash 196 | poetry install 197 | ``` 198 | 199 | ### Running Tests 200 | 201 | ```bash 202 | poetry run pytest 203 | ``` 204 | 205 | ### Using VScode devcontainer 206 | 207 | The project contains a `.devcontainer` folder with the settings for VScode to [develop inside container](https://code.visualstudio.com/docs/remote/containers). The Python virtual environment 208 | created by poetry is stored outside the container in the projects path `.virtualenvs` so that it survives container rebuilds. 209 | 210 | The Ubuntu 24.04 based container uses Python 3.12 as system version and includes minimal Python 3.10 to 3.13 versions 211 | for creating virtual environments in any of those versions. 212 | 213 | It is possible to switch the Python version used by `poetry` with the command `poetry env use `, 214 | see [poetry managing environments](https://python-poetry.org/docs/managing-environments/) for more details. 215 | 216 | # Thanks 217 | 218 | This project was initially based on [borgmatic](https://github.com/witten/borgmatic/) but has since evolved into something else. 219 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "runrestic" 3 | version = "0.5.30" 4 | description = "A wrapper script for Restic backup software that inits, creates, prunes and checks backups" 5 | license = "GPL-3.0+" 6 | authors = [ "Andreas Nüßlein " ] 7 | readme = 'README.md' 8 | repository = "https://github.com/sinnwerkstatt/runrestic" 9 | homepage = "https://github.com/sinnwerkstatt/runrestic" 10 | keywords = ["backup"] 11 | classifiers = [ 12 | "Development Status :: 4 - Beta", 13 | "Environment :: Console", 14 | "Intended Audience :: System Administrators", 15 | "Programming Language :: Python", 16 | "Topic :: Security :: Cryptography", 17 | "Topic :: System :: Archiving :: Backup", 18 | ] 19 | 20 | [tool.poetry.dependencies] 21 | python = ">=3.10" 22 | toml = "^0.10" 23 | jsonschema = ">3.0" 24 | requests = ">2.27.1" 25 | 26 | [tool.poetry.group.dev.dependencies] 27 | ipdb = "*" 28 | black = { version = "*", allow-prereleases = true } 29 | mypy = "*" 30 | pytest = "*" 31 | pytest-cov = "*" 32 | pytest-subprocess = "*" 33 | importlib-metadata = { version = ">=4.0", python = ">=3.10" } 34 | types-requests = "*" 35 | types-toml = "*" 36 | 37 | [tool.poetry.scripts] 38 | runrestic = 'runrestic.runrestic.runrestic:runrestic' 39 | 40 | [tool.black] 41 | line-length = 88 42 | target-version = ['py310', 'py311', 'py312', 'py313'] 43 | include = '\.pyi?$' 44 | exclude = ''' 45 | /( 46 | \.eggs 47 | | \.git 48 | )/ 49 | ''' 50 | 51 | [tool.isort] 52 | multi_line_output = 3 53 | include_trailing_comma = true 54 | force_grid_wrap = 0 55 | use_parentheses = false 56 | line_length = 88 57 | known_first_party = 'apps' 58 | default_section = 'THIRDPARTY' 59 | sections = 'FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER' 60 | no_lines_before = 'LOCALFOLDER' 61 | 62 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | ;addopts = -rA -q --cov=runrestic tests/ 3 | addopts = --cov=runrestic --cov-report term-missing tests/ 4 | ;term-missing:skip-covered 5 | filterwarnings = 6 | ignore:open_text is deprecated:DeprecationWarning -------------------------------------------------------------------------------- /runrestic/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.5.30" 2 | -------------------------------------------------------------------------------- /runrestic/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Dict 2 | 3 | from . import prometheus 4 | 5 | 6 | def write_metrics(metrics: Dict[str, Any], config: Dict[str, Any]) -> None: 7 | configuration = config["metrics"] 8 | if "prometheus" in configuration: 9 | lines = prometheus.generate_lines(metrics, config["name"]) 10 | 11 | with open(configuration["prometheus"]["path"], "w") as file: 12 | file.writelines("".join(lines)) 13 | -------------------------------------------------------------------------------- /runrestic/metrics/prometheus.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module provides functionality to generate Prometheus-compatible metrics 3 | based on the output of various Restic commands. 4 | 5 | It defines templates for Prometheus metrics and functions to format the metrics 6 | based on the parsed Restic output. The metrics include information about backup, 7 | forget, prune, check, and stats operations. 8 | """ 9 | 10 | from typing import Any, Iterator 11 | 12 | # Prometheus metric templates for general metrics 13 | _restic_help_general = """ 14 | # HELP restic_last_run Epoch timestamp of the last run 15 | # TYPE restic_last_run gauge 16 | # HELP restic_total_duration_seconds Total duration in seconds 17 | # TYPE restic_total_duration_seconds gauge 18 | # HELP restic_total_errors Total amount of errors within the last run 19 | # TYPE restic_total_errors gauge 20 | """ 21 | _restic_general = """ 22 | restic_last_run{{config="{name}"}} {last_run} 23 | restic_total_duration_seconds{{config="{name}"}} {total_duration_seconds} 24 | restic_total_errors{{config="{name}"}} {errors} 25 | """ 26 | 27 | # Additional Prometheus metric templates for specific operations 28 | _restic_help_pre_hooks = """ 29 | # HELP restic_pre_hooks_duration_seconds Pre hooks duration in seconds 30 | # TYPE restic_pre_hooks_duration_seconds gauge 31 | # HELP restic_pre_hooks_rc Pre hooks return code 32 | # TYPE restic_pre_hooks_rc gauge 33 | """ 34 | _restic_pre_hooks = """ 35 | restic_pre_hooks_duration_seconds{{config="{name}"}} {duration_seconds} 36 | restic_pre_hooks_rc{{config="{name}"}} {rc} 37 | """ 38 | 39 | _restic_help_post_hooks = """ 40 | # HELP restic_post_hooks_duration_seconds Post hooks duration in seconds 41 | # TYPE restic_post_hooks_duration_seconds gauge 42 | # HELP restic_post_hooks_rc Post hooks return code 43 | # TYPE restic_post_hooks_rc gauge 44 | """ 45 | _restic_post_hooks = """ 46 | restic_post_hooks_duration_seconds{{config="{name}"}} {duration_seconds} 47 | restic_post_hooks_rc{{config="{name}"}} {rc} 48 | """ 49 | 50 | _restic_help_backup = """ 51 | # HELP restic_backup_files_new Number of new files 52 | # TYPE restic_backup_files_new gauge 53 | # HELP restic_backup_files_changed Number of changed files 54 | # TYPE restic_backup_files_changed gauge 55 | # HELP restic_backup_files_unmodified Number of unmodified files 56 | # TYPE restic_backup_files_unmodified gauge 57 | # HELP restic_backup_dirs_new Number of new dirs 58 | # TYPE restic_backup_dirs_new gauge 59 | # HELP restic_backup_dirs_changed Number of changed dirs 60 | # TYPE restic_backup_dirs_changed gauge 61 | # HELP restic_backup_dirs_unmodified Number of unmodified dirs 62 | # TYPE restic_backup_dirs_unmodified gauge 63 | # HELP restic_backup_processed_files Number of processed files 64 | # TYPE restic_backup_processed_files gauge 65 | # HELP restic_backup_processed_size_bytes Processed size bytes 66 | # TYPE restic_backup_processed_size_bytes gauge 67 | # HELP restic_backup_processed_duration_seconds Backup processed duration in seconds 68 | # TYPE restic_backup_processed_duration_seconds gauge 69 | # HELP restic_backup_added_to_repo Number of added to repo 70 | # TYPE restic_backup_added_to_repo gauge 71 | # HELP restic_backup_duration_seconds Backup duration in seconds 72 | # TYPE restic_backup_duration_seconds gauge 73 | # HELP restic_backup_rc Return code of the restic backup command 74 | # TYPE restic_backup_rc gauge 75 | """ 76 | _restic_backup = """ 77 | restic_backup_files_new{{config="{name}",repository="{repository}"}} {files[new]} 78 | restic_backup_files_changed{{config="{name}",repository="{repository}"}} {files[changed]} 79 | restic_backup_files_unmodified{{config="{name}",repository="{repository}"}} {files[unmodified]} 80 | restic_backup_dirs_new{{config="{name}",repository="{repository}"}} {dirs[new]} 81 | restic_backup_dirs_changed{{config="{name}",repository="{repository}"}} {dirs[changed]} 82 | restic_backup_dirs_unmodified{{config="{name}",repository="{repository}"}} {dirs[unmodified]} 83 | restic_backup_processed_files{{config="{name}",repository="{repository}"}} {processed[files]} 84 | restic_backup_processed_size_bytes{{config="{name}",repository="{repository}"}} {processed[size_bytes]} 85 | restic_backup_processed_duration_seconds{{config="{name}",repository="{repository}"}} {processed[duration_seconds]} 86 | restic_backup_added_to_repo{{config="{name}",repository="{repository}"}} {added_to_repo} 87 | restic_backup_duration_seconds{{config="{name}",repository="{repository}"}} {duration_seconds} 88 | restic_backup_rc{{config="{name}",repository="{repository}"}} {rc} 89 | """ 90 | 91 | _restic_help_forget = """ 92 | # HELP restic_forget_removed_snapshots Number of forgotten snapshots 93 | # TYPE restic_forget_removed_snapshots gauge 94 | # HELP restic_forget_duration_seconds Forget duration in seconds 95 | # TYPE restic_forget_duration_seconds gauge 96 | # HELP restic_forget_rc Return code of the restic forget command 97 | # TYPE restic_forget_rc gauge 98 | """ 99 | _restic_forget = """ 100 | restic_forget_removed_snapshots{{config="{name}",repository="{repository}"}} {removed_snapshots} 101 | restic_forget_duration_seconds{{config="{name}",repository="{repository}"}} {duration_seconds} 102 | restic_forget_rc{{config="{name}",repository="{repository}"}} {rc} 103 | """ 104 | 105 | _restic_help_prune = """ 106 | # HELP restic_prune_containing_packs_before Number of packs contained in repository before pruning 107 | # TYPE restic_prune_containing_packs_before gauge 108 | # HELP restic_prune_containing_blobs Number of blobs contained in repository before pruning 109 | # TYPE restic_prune_containing_blobs gauge 110 | # HELP restic_prune_containing_size_bytes Size in bytes contained in repository before pruning 111 | # TYPE restic_prune_containing_size_bytes gauge 112 | # HELP restic_prune_duplicate_blobs Number of duplicates found in the processed blobs 113 | # TYPE restic_prune_duplicate_blobs gauge 114 | # HELP restic_prune_duplicate_size_bytes Size in bytes of the duplicates found in the processed blobs 115 | # TYPE restic_prune_duplicate_size_bytes gauge 116 | # HELP restic_prune_in_use_blobs Number of blobs that are still in use (won't be removed) 117 | # TYPE restic_prune_in_use_blobs gauge 118 | # HELP restic_prune_removed_blobs Number of blobs to remove 119 | # TYPE restic_prune_removed_blobs gauge 120 | # HELP restic_prune_invalid_files Number of invalid files to remove 121 | # TYPE restic_prune_invalid_files gauge 122 | # HELP restic_prune_deleted_packs Number of pack to delete 123 | # TYPE restic_prune_deleted_packs gauge 124 | # HELP restic_prune_rewritten_packs Number of pack to delete 125 | # TYPE restic_prune_rewritten_packs gauge 126 | # HELP restic_prune_size_freed_bytes Size in byte freed after pack deletion 127 | # TYPE restic_prune_size_freed_bytes gauge 128 | # HELP restic_prune_removed_index_files Number of old index removed 129 | # TYPE restic_prune_removed_index_files gauge 130 | # HELP restic_prune_duration_seconds Duration in seconds 131 | # TYPE restic_prune_duration_seconds gauge 132 | # HELP restic_prune_rc Return code of the restic prune command 133 | # TYPE restic_prune_rc gauge 134 | """ 135 | _restic_prune = """ 136 | restic_prune_containing_packs_before{{config="{name}",repository="{repository}"}} {containing_packs_before} 137 | restic_prune_containing_blobs{{config="{name}",repository="{repository}"}} {containing_blobs} 138 | restic_prune_containing_size_bytes{{config="{name}",repository="{repository}"}} {containing_size_bytes} 139 | restic_prune_duplicate_blobs{{config="{name}",repository="{repository}"}} {duplicate_blobs} 140 | restic_prune_duplicate_size_bytes{{config="{name}",repository="{repository}"}} {duplicate_size_bytes} 141 | restic_prune_in_use_blobs{{config="{name}",repository="{repository}"}} {in_use_blobs} 142 | restic_prune_removed_blobs{{config="{name}",repository="{repository}"}} {removed_blobs} 143 | restic_prune_invalid_files{{config="{name}",repository="{repository}"}} {invalid_files} 144 | restic_prune_deleted_packs{{config="{name}",repository="{repository}"}} {deleted_packs} 145 | restic_prune_rewritten_packs{{config="{name}",repository="{repository}"}} {rewritten_packs} 146 | restic_prune_size_freed_bytes{{config="{name}",repository="{repository}"}} {size_freed_bytes} 147 | restic_prune_removed_index_files{{config="{name}",repository="{repository}"}} {removed_index_files} 148 | restic_prune_duration_seconds{{config="{name}",repository="{repository}"}} {duration_seconds} 149 | restic_prune_rc{{config="{name}",repository="{repository}"}} {rc} 150 | """ 151 | _restic_new_prune = """ 152 | restic_prune_to_repack_blobs{{config="{name}",repository="{repository}"}} {to_repack_blobs} 153 | restic_prune_to_repack_bytes{{config="{name}",repository="{repository}"}} {to_repack_bytes} 154 | restic_prune_removed_blobs{{config="{name}",repository="{repository}"}} {removed_blobs} 155 | restic_prune_removed_bytes{{config="{name}",repository="{repository}"}} {removed_bytes} 156 | restic_prune_to_delete_blobs{{config="{name}",repository="{repository}"}} {to_delete_blobs} 157 | restic_prune_to_delete_bytes{{config="{name}",repository="{repository}"}} {to_delete_bytes} 158 | restic_prune_total_prune_blobs{{config="{name}",repository="{repository}"}} {total_prune_blobs} 159 | restic_prune_total_prune_bytes{{config="{name}",repository="{repository}"}} {total_prune_bytes} 160 | restic_prune_remaining_blobs{{config="{name}",repository="{repository}"}} {remaining_blobs} 161 | restic_prune_remaining_bytes{{config="{name}",repository="{repository}"}} {remaining_bytes} 162 | restic_prune_remaining_unused_size{{config="{name}",repository="{repository}"}} {remaining_unused_size} 163 | restic_prune_duration_seconds{{config="{name}",repository="{repository}"}} {duration_seconds} 164 | restic_prune_rc{{config="{name}",repository="{repository}"}} {rc} 165 | """ 166 | 167 | _restic_help_check = """ 168 | # HELP restic_check_errors Boolean to tell if any error occured 169 | # TYPE restic_check_errors gauge 170 | # HELP restic_check_errors_data Boolean to tell if the pack ID does not match 171 | # TYPE restic_check_errors_data gauge 172 | # HELP restic_check_errors_snapshots Boolean to tell if any of the snapshots can not be loaded 173 | # TYPE restic_check_errors_snapshots gauge 174 | # HELP restic_check_read_data Boolean that indicates whether or not `--read-data` was pass to restic 175 | # TYPE restic_check_read_data gauge 176 | # HELP restic_check_check_unused Boolean that indicates whether or not `--check-unused` was pass to restic 177 | # TYPE restic_check_check_unused gauge 178 | # HELP restic_check_duration_seconds Duration in seconds 179 | # TYPE restic_check_duration_seconds gauge 180 | # HELP restic_check_rc Return code of the restic check command 181 | # TYPE restic_check_rc gauge 182 | """ 183 | _restic_check = """ 184 | restic_check_errors{{config="{name}",repository="{repository}"}} {errors} 185 | restic_check_errors_data{{config="{name}",repository="{repository}"}} {errors_data} 186 | restic_check_errors_snapshots{{config="{name}",repository="{repository}"}} {errors_snapshots} 187 | restic_check_read_data{{config="{name}",repository="{repository}"}} {read_data} 188 | restic_check_check_unused{{config="{name}",repository="{repository}"}} {check_unused} 189 | restic_check_duration_seconds{{config="{name}",repository="{repository}"}} {duration_seconds} 190 | restic_check_rc{{config="{name}",repository="{repository}"}} {rc} 191 | """ 192 | 193 | _restic_help_stats = """ 194 | # HELP restic_stats_total_file_count Stats for all snapshots in restore size mode - Total file count 195 | # TYPE restic_stats_total_file_count gauge 196 | # HELP restic_stats_total_size_bytes Stats for all snapshots in restore size mode - Total file size in bytes 197 | # TYPE restic_stats_total_size_bytes gauge 198 | # HELP restic_stats_duration_seconds Stats for all snapshots in restore size mode - Duration in seconds 199 | # TYPE restic_stats_duration_seconds gauge 200 | # HELP restic_stats_rc Stats for all snapshots in restore size mode - Return code of the restic stats command 201 | # TYPE restic_stats_rc gauge 202 | """ 203 | _restic_stats = """ 204 | restic_stats_total_file_count{{config="{name}",repository="{repository}"}} {total_file_count} 205 | restic_stats_total_size_bytes{{config="{name}",repository="{repository}"}} {total_size_bytes} 206 | restic_stats_duration_seconds{{config="{name}",repository="{repository}"}} {duration_seconds} 207 | restic_stats_rc{{config="{name}",repository="{repository}"}} {rc} 208 | """ 209 | 210 | 211 | def generate_lines(metrics: dict[str, Any], name: str) -> Iterator[str]: 212 | """ 213 | Generate Prometheus metrics lines for the given Restic metrics. 214 | 215 | Args: 216 | metrics (dict[str, Any]): A dictionary containing parsed Restic metrics. 217 | name (str): The configuration name for the metrics. 218 | 219 | Yields: 220 | Iterator[str]: Prometheus-formatted metric lines. 221 | """ 222 | yield _restic_help_general 223 | yield _restic_general.format(name=name, **metrics) 224 | 225 | if metrics.get("backup"): 226 | yield backup_metrics(metrics["backup"], name) 227 | if metrics.get("forget"): 228 | yield forget_metrics(metrics["forget"], name) 229 | if metrics.get("prune"): 230 | yield prune_metrics(metrics["prune"], name) 231 | if metrics.get("check"): 232 | yield check_metrics(metrics["check"], name) 233 | if metrics.get("stats"): 234 | yield stats_metrics(metrics["stats"], name) 235 | 236 | 237 | def backup_metrics(metrics: dict[str, Any], name: str) -> str: 238 | """ 239 | Generate Prometheus metrics for Restic backup operations. 240 | 241 | Args: 242 | metrics (dict[str, Any]): A dictionary containing backup metrics. 243 | name (str): The configuration name for the metrics. 244 | 245 | Returns: 246 | str: Prometheus-formatted backup metrics. 247 | """ 248 | pre_hooks = post_hooks = False 249 | retval = "" 250 | for repo, mtrx in metrics.items(): 251 | if repo == "_restic_pre_hooks": 252 | pre_hooks = True 253 | retval += _restic_pre_hooks.format(name=name, **mtrx) 254 | elif repo == "_restic_post_hooks": 255 | post_hooks = True 256 | retval += _restic_post_hooks.format(name=name, **mtrx) 257 | else: 258 | if mtrx["rc"] != 0: 259 | retval += f'restic_backup_rc{{config="{name}",repository="{repo}"}} {mtrx["rc"]}\n' 260 | else: 261 | retval += _restic_backup.format(name=name, repository=repo, **mtrx) 262 | 263 | help_text = _restic_help_backup 264 | if pre_hooks: 265 | help_text += _restic_help_pre_hooks 266 | if post_hooks: 267 | help_text += _restic_help_post_hooks 268 | return help_text + retval 269 | 270 | 271 | def forget_metrics(metrics: dict[str, Any], name: str) -> str: 272 | """ 273 | Generate Prometheus metrics for Restic forget operations. 274 | 275 | Args: 276 | metrics (dict[str, Any]): A dictionary containing forget metrics. 277 | name (str): The configuration name for the metrics. 278 | 279 | Returns: 280 | str: Prometheus-formatted forget metrics. 281 | """ 282 | retval = _restic_help_forget 283 | for repo, mtrx in metrics.items(): 284 | if mtrx["rc"] != 0: 285 | retval += f'restic_forget_rc{{config="{name}",repository="{repo}"}} {mtrx["rc"]}\n' 286 | else: 287 | retval += _restic_forget.format(name=name, repository=repo, **mtrx) 288 | return retval 289 | 290 | 291 | def prune_metrics(metrics: dict[str, Any], name: str) -> str: 292 | """ 293 | Generate Prometheus metrics for Restic prune operations. 294 | 295 | Args: 296 | metrics (dict[str, Any]): A dictionary containing prune metrics. 297 | name (str): The configuration name for the metrics. 298 | 299 | Returns: 300 | str: Prometheus-formatted prune metrics. 301 | """ 302 | retval = _restic_help_prune 303 | for repo, mtrx in metrics.items(): 304 | if mtrx["rc"] != 0: 305 | retval += ( 306 | f'restic_prune_rc{{config="{name}",repository="{repo}"}} {mtrx["rc"]}\n' 307 | ) 308 | else: 309 | try: 310 | retval += _restic_prune.format(name=name, repository=repo, **mtrx) 311 | except KeyError: 312 | retval += _restic_new_prune.format(name=name, repository=repo, **mtrx) 313 | return retval 314 | 315 | 316 | def check_metrics(metrics: dict[str, Any], name: str) -> str: 317 | """ 318 | Generate Prometheus metrics for Restic check operations. 319 | 320 | Args: 321 | metrics (dict[str, Any]): A dictionary containing check metrics. 322 | name (str): The configuration name for the metrics. 323 | 324 | Returns: 325 | str: Prometheus-formatted check metrics. 326 | """ 327 | retval = _restic_help_check 328 | for repo, mtrx in metrics.items(): 329 | if mtrx["rc"] != 0: 330 | retval += ( 331 | f'restic_check_rc{{config="{name}",repository="{repo}"}} {mtrx["rc"]}\n' 332 | ) 333 | else: 334 | retval += _restic_check.format(name=name, repository=repo, **mtrx) 335 | return retval 336 | 337 | 338 | def stats_metrics(metrics: dict[str, Any], name: str) -> str: 339 | """ 340 | Generate Prometheus metrics for Restic stats operations. 341 | 342 | Args: 343 | metrics (dict[str, Any]): A dictionary containing stats metrics. 344 | name (str): The configuration name for the metrics. 345 | 346 | Returns: 347 | str: Prometheus-formatted stats metrics. 348 | """ 349 | retval = _restic_help_stats 350 | for repo, mtrx in metrics.items(): 351 | if mtrx["rc"] != 0: 352 | retval += ( 353 | f'restic_stats_rc{{config="{name}",repository="{repo}"}} {mtrx["rc"]}\n' 354 | ) 355 | else: 356 | retval += _restic_stats.format(name=name, repository=repo, **mtrx) 357 | return retval 358 | -------------------------------------------------------------------------------- /runrestic/restic/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sinnwerkstatt/runrestic/4ff8868628e5ca4633ac666e486add0b3a8e702f/runrestic/restic/__init__.py -------------------------------------------------------------------------------- /runrestic/restic/installer.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module provides functionality to check for the presence of the Restic binary 3 | on the system and to download and install it if necessary. 4 | 5 | It interacts with the GitHub API to fetch the latest Restic release and handles 6 | the installation process, including permissions and alternative paths. 7 | """ 8 | 9 | import bz2 10 | import json 11 | import logging 12 | import os 13 | from shutil import which 14 | 15 | import requests 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | def restic_check() -> bool: 21 | """ 22 | Check if the Restic binary is available on the system. 23 | 24 | If Restic is not found, the user is prompted to install it. 25 | 26 | Returns: 27 | bool: True if Restic is available or successfully installed, False otherwise. 28 | """ 29 | if which("restic"): 30 | return True 31 | carry_on = input( 32 | "There seems to be no restic on your system. Should I install it now? [Y/n] " 33 | ) 34 | if carry_on in ["", "y", "Y"]: 35 | download_restic() 36 | return True 37 | return False 38 | 39 | 40 | def download_restic() -> None: 41 | """ 42 | Download and install the latest Restic binary. 43 | 44 | The function fetches the latest release information from the Restic GitHub repository, 45 | downloads the compressed binary, decompresses it, and installs it to `/usr/local/bin/restic`. 46 | If permissions are insufficient, the user is prompted to provide an alternative path. 47 | """ 48 | github_json = json.loads( 49 | requests.get( 50 | "https://api.github.com/repos/restic/restic/releases/latest" 51 | ).content 52 | ) 53 | 54 | download_url = "" 55 | for asset in github_json["assets"]: 56 | if "linux_amd64.bz2" in asset["name"]: 57 | download_url = asset["browser_download_url"] 58 | break 59 | 60 | file = requests.get(download_url, allow_redirects=True).content 61 | 62 | program = bz2.decompress(file) 63 | try: 64 | path = "/usr/local/bin/restic" 65 | open(path, "wb").write(program) 66 | os.chmod(path, 0o755) 67 | except PermissionError as e: 68 | print(e) 69 | print("\nTry re-running this as root.") 70 | print("Alternatively you can specify a path where I can put restic.") 71 | alt_path = input("Example: /home/you/.bin/restic. Leave blank to exit.\n") 72 | if alt_path: 73 | open(alt_path, "wb").write(program) 74 | os.chmod(alt_path, 0o755) 75 | -------------------------------------------------------------------------------- /runrestic/restic/output_parsing.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module provides functions to parse the output of various Restic commands. 3 | 4 | Each function extracts relevant information from the command output and returns it 5 | in a structured format, such as dictionaries. These functions are used to process 6 | the output of commands like `backup`, `forget`, `prune`, and `stats`. 7 | """ 8 | 9 | import json 10 | import logging 11 | import re 12 | from typing import Any 13 | 14 | from runrestic.runrestic.tools import parse_line, parse_size, parse_time 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | def parse_backup(process_infos: dict[str, Any]) -> dict[str, Any]: 20 | """ 21 | Parse the output of the Restic `backup` command. 22 | 23 | Args: 24 | process_infos (dict[str, Any]): A dictionary containing process information, 25 | including the command output and execution time. 26 | 27 | Returns: 28 | dict[str, Any]: A dictionary with parsed backup statistics, such as file counts, 29 | directory counts, processed size, and duration. 30 | """ 31 | return_code, output = process_infos["output"][-1] 32 | logger.debug("Parsing backup output: %s", output) 33 | 34 | files_new, files_changed, files_unmodified = parse_line( 35 | r"Files:\s+([0-9]+) new,\s+([0-9]+) changed,\s+([0-9]+) unmodified", 36 | output, 37 | ("0", "0", "0"), 38 | ) 39 | dirs_new, dirs_changed, dirs_unmodified = parse_line( 40 | r"Dirs:\s+([0-9]+) new,\s+([0-9]+) changed,\s+([0-9]+) unmodified", 41 | output, 42 | ("0", "0", "0"), 43 | ) 44 | added_to_the_repo = parse_line( 45 | r"Added to the repo\w*:\s+(-?[0-9.]+ [a-zA-Z]*B)", 46 | output, 47 | "0 B", 48 | ) 49 | processed_files, processed_size, processed_time = parse_line( 50 | r"processed ([0-9]+) files,\s+(-?[0-9.]+ [a-zA-Z]*B) in ([0-9]+:+[0-9]+)", 51 | output, 52 | ("0", "0 B", "00:00"), 53 | ) 54 | 55 | return { 56 | "files": { 57 | "new": files_new, 58 | "changed": files_changed, 59 | "unmodified": files_unmodified, 60 | }, 61 | "dirs": { 62 | "new": dirs_new, 63 | "changed": dirs_changed, 64 | "unmodified": dirs_unmodified, 65 | }, 66 | "processed": { 67 | "files": processed_files, 68 | "size_bytes": parse_size(processed_size), 69 | "duration_seconds": parse_time(processed_time), 70 | }, 71 | "added_to_repo": parse_size(added_to_the_repo), 72 | "duration_seconds": process_infos["time"], 73 | "rc": return_code, 74 | } 75 | 76 | 77 | def parse_forget(process_infos: dict[str, Any]) -> dict[str, Any]: 78 | """ 79 | Parse the output of the Restic `forget` command. 80 | 81 | Args: 82 | process_infos (dict[str, Any]): A dictionary containing process information, 83 | including the command output and execution time. 84 | 85 | Returns: 86 | dict[str, Any]: A dictionary with parsed forget statistics, such as the number 87 | of removed snapshots and duration. 88 | """ 89 | return_code, output = process_infos["output"][-1] 90 | re_removed_snapshots = parse_line(r"remove ([0-9]+) snapshots", output, "0") 91 | return { 92 | "removed_snapshots": re_removed_snapshots, 93 | "duration_seconds": process_infos["time"], 94 | "rc": return_code, 95 | } 96 | 97 | 98 | def parse_prune(process_infos: dict[str, Any]) -> dict[str, Any]: 99 | """ 100 | Parse the output of the Restic `prune` command. 101 | 102 | Args: 103 | process_infos (dict[str, Any]): A dictionary containing process information, 104 | including the command output and execution time. 105 | 106 | Returns: 107 | dict[str, Any]: A dictionary with parsed prune statistics, such as the number 108 | of removed blobs, freed size, and duration. 109 | """ 110 | return_code, output = process_infos["output"][-1] 111 | containing_packs, containing_blobs, containing_size = parse_line( 112 | r"repository contains ([0-9]+) packs \(([0-9]+) blobs\) with (-?[0-9.]+ ?[a-zA-Z]*B)", 113 | output, 114 | ("0", "0", "0 B"), 115 | ) 116 | duplicate_blobs, duplicate_size = parse_line( 117 | r"([0-9]+) duplicate blobs, (-?[0-9.]+ ?[a-zA-Z]*B) duplicate", 118 | output, 119 | ("0", "0 B"), 120 | ) 121 | in_use_blobs, _, removed_blobs = parse_line( 122 | r"found ([0-9]+) of ([0-9]+) data blobs still in use, removing ([0-9]+) blobs", 123 | output, 124 | ("0", "0", "0"), 125 | ) 126 | invalid_files = parse_line(r"will remove ([0-9]+) invalid files", output, ("0")) 127 | deleted_packs, rewritten_packs, size_freed = parse_line( 128 | r"will delete ([0-9]+) packs and rewrite ([0-9]+) packs, this frees (-?[0-9.]+ ?[a-zA-Z]*B)", 129 | output, 130 | ("0", "0", "0 B"), 131 | ) 132 | removed_index_files = parse_line(r"remove ([0-9]+) old index files", output, "0") 133 | 134 | return { 135 | "containing_packs_before": containing_packs, 136 | "containing_blobs": containing_blobs, 137 | "containing_size_bytes": parse_size(containing_size), 138 | "duplicate_blobs": duplicate_blobs, 139 | "duplicate_size_bytes": parse_size(duplicate_size), 140 | "in_use_blobs": in_use_blobs, 141 | "removed_blobs": removed_blobs, 142 | "invalid_files": invalid_files, 143 | "deleted_packs": deleted_packs, 144 | "rewritten_packs": rewritten_packs, 145 | "size_freed_bytes": parse_size(size_freed), 146 | "removed_index_files": removed_index_files, 147 | "duration_seconds": process_infos["time"], 148 | "rc": return_code, 149 | } 150 | 151 | 152 | def parse_new_prune(process_infos: dict[str, Any]) -> dict[str, Any]: 153 | """ 154 | Parse the output of the new Restic `prune` command. 155 | 156 | Args: 157 | process_infos (dict[str, Any]): A dictionary containing process information, 158 | including the command output and execution time. 159 | 160 | Returns: 161 | dict[str, Any]: A dictionary with parsed prune statistics, such as the number 162 | of blobs to repack, removed blobs, and remaining unused size. 163 | """ 164 | return_code, output = process_infos["output"][-1] 165 | 166 | to_repack_blobs, to_repack_bytes = parse_line( 167 | r"to repack:[\s]+([0-9]+) blobs / (-?[0-9.]+ ?[a-zA-Z]*B)", 168 | output, 169 | ("0", "0 B"), 170 | ) 171 | removed_blobs, removed_bytes = parse_line( 172 | r"this removes[:]*[\s]+([0-9]+) blobs / (-?[0-9.]+ ?[a-zA-Z]*B)", 173 | output, 174 | ("0", "0 B"), 175 | ) 176 | to_delete_blobs, to_delete_bytes = parse_line( 177 | r"to delete:[\s]+([0-9]+) blobs / (-?[0-9.]+ ?[a-zA-Z]*B)", 178 | output, 179 | ("0", "0 B"), 180 | ) 181 | total_prune_blobs, total_prune_bytes = parse_line( 182 | r"total prune:[\s]+([0-9]+) blobs / (-?[0-9.]+ ?[a-zA-Z]*B)", 183 | output, 184 | ("0", "0 B"), 185 | ) 186 | remaining_blobs, remaining_bytes = parse_line( 187 | r"remaining:[\s]+([0-9]+) blobs / (-?[0-9.]+ ?[a-zA-Z]*B)", 188 | output, 189 | ("0", "0 B"), 190 | ) 191 | remaining_unused_size = parse_line( 192 | r"unused size after prune:[\s]+(-?[0-9.]+ ?[a-zA-Z]*B)", 193 | output, 194 | "0 B", 195 | ) 196 | return { 197 | "to_repack_blobs": to_repack_blobs, 198 | "to_repack_bytes": parse_size(to_repack_bytes), 199 | "removed_blobs": removed_blobs, 200 | "removed_bytes": parse_size(removed_bytes), 201 | "to_delete_blobs": to_delete_blobs, 202 | "to_delete_bytes": parse_size(to_delete_bytes), 203 | "total_prune_blobs": total_prune_blobs, 204 | "total_prune_bytes": parse_size(total_prune_bytes), 205 | "remaining_blobs": remaining_blobs, 206 | "remaining_bytes": parse_size(remaining_bytes), 207 | "remaining_unused_size": parse_size(remaining_unused_size), 208 | "duration_seconds": process_infos["time"], 209 | "rc": return_code, 210 | } 211 | 212 | 213 | def parse_stats(process_infos: dict[str, Any]) -> dict[str, Any]: 214 | """ 215 | Parse the output of the Restic `stats` command. 216 | 217 | Args: 218 | process_infos (dict[str, Any]): A dictionary containing process information, 219 | including the command output and execution time. 220 | 221 | Returns: 222 | dict[str, Any]: A dictionary with parsed statistics, such as total file count 223 | and total size in bytes. 224 | """ 225 | return_code, output = process_infos["output"][-1] 226 | try: 227 | stats_json = json.loads(re.findall(r"(\{.*\})", output, re.MULTILINE)[0]) 228 | return { 229 | "total_file_count": stats_json["total_file_count"], 230 | "total_size_bytes": stats_json["total_size"], 231 | "duration_seconds": process_infos["time"], 232 | "rc": return_code, 233 | } 234 | except KeyError as err: 235 | logger.error("Key %s not found in output: %s", err, output) 236 | return { 237 | "total_file_count": 0, 238 | "total_size_bytes": 0, 239 | "duration_seconds": 0, 240 | "rc": return_code, 241 | } 242 | -------------------------------------------------------------------------------- /runrestic/restic/runner.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module provides the `ResticRunner` class, which is responsible for managing and executing 3 | various Restic commands such as backup, prune, check, stats, and more. It handles configuration, 4 | logging, metrics collection, and error handling for Restic operations. 5 | """ 6 | 7 | import json 8 | import logging 9 | import re 10 | import time 11 | from argparse import Namespace 12 | from datetime import datetime 13 | from typing import Any 14 | 15 | from runrestic.metrics import write_metrics 16 | from runrestic.restic.output_parsing import ( 17 | parse_backup, 18 | parse_forget, 19 | parse_new_prune, 20 | parse_prune, 21 | parse_stats, 22 | ) 23 | from runrestic.restic.tools import MultiCommand, initialize_environment, redact_password 24 | 25 | logger = logging.getLogger(__name__) 26 | 27 | 28 | class ResticRunner: 29 | """ 30 | A class to manage and execute Restic commands based on the provided configuration and arguments. 31 | 32 | Attributes: 33 | config (dict): Configuration dictionary for Restic operations. 34 | args (Namespace): Command-line arguments passed to the runner. 35 | restic_args (list): Additional arguments to pass to Restic commands. 36 | repos (list): list of repository paths to operate on. 37 | metrics (dict): dictionary to store metrics and errors for operations. 38 | log_metrics (bool): Flag to determine if metrics should be logged. 39 | pw_replacement (str): Replacement string for sensitive information in logs. 40 | """ 41 | 42 | def __init__( 43 | self, config: dict[str, Any], args: Namespace, restic_args: list[str] 44 | ) -> None: 45 | """ 46 | Initialize the ResticRunner with configuration, arguments, and Restic-specific arguments. 47 | 48 | Args: 49 | config (dict): Configuration dictionary for Restic operations. 50 | args (Namespace): Command-line arguments passed to the runner. 51 | restic_args (list): Additional arguments to pass to Restic commands. 52 | """ 53 | self.config = config 54 | self.args = args 55 | self.restic_args = restic_args 56 | 57 | self.repos: list[str] = self.config["repositories"] 58 | 59 | self.metrics: dict[str, Any] = {"errors": 0} 60 | self.log_metrics: Any = config.get("metrics") and not args.dry_run 61 | self.pw_replacement: str = ( 62 | config.get("metrics", {}) 63 | .get("prometheus", {}) 64 | .get("password_replacement", "") 65 | ) 66 | 67 | initialize_environment(self.config["environment"]) 68 | 69 | def run(self) -> int: # noqa: C901 70 | """ 71 | Execute the specified Restic actions in sequence. 72 | 73 | Returns: 74 | int: The number of errors encountered during execution. 75 | """ 76 | start_time = time.time() 77 | actions = self.args.actions 78 | 79 | if not actions and self.log_metrics: 80 | actions = ["backup", "prune", "check", "stats"] 81 | elif not actions: 82 | actions = ["backup", "prune", "check"] 83 | 84 | logger.info("Starting '%s': %s", self.config["name"], actions) 85 | for action in actions: 86 | if action == "init": 87 | self.init() 88 | elif action == "backup": 89 | self.backup() 90 | elif action == "prune": 91 | self.forget() 92 | self.prune() 93 | elif action == "check": 94 | self.check() 95 | elif action == "stats": 96 | self.stats() 97 | elif action == "unlock": 98 | self.unlock() 99 | 100 | self.metrics["last_run"] = datetime.now().timestamp() 101 | self.metrics["total_duration_seconds"] = time.time() - start_time 102 | 103 | logger.debug(json.dumps(self.metrics, indent=2)) 104 | 105 | if self.log_metrics: 106 | write_metrics(self.metrics, self.config) 107 | 108 | return self.metrics["errors"] # type: ignore[no-any-return] 109 | 110 | def init(self) -> None: 111 | """ 112 | Initialize the Restic repository for each configured repository. 113 | """ 114 | commands = [ 115 | ["restic", "-r", repo, "init", *self.restic_args] for repo in self.repos 116 | ] 117 | 118 | direct_abort_reasons = ["config file already exists"] 119 | cmd_runs = MultiCommand( 120 | commands, self.config["execution"], direct_abort_reasons 121 | ).run() 122 | 123 | for process_infos in cmd_runs: 124 | if process_infos["output"][-1][0] > 0: 125 | logger.warning(process_infos["output"]) 126 | else: 127 | logger.info(process_infos["output"]) 128 | 129 | def backup(self) -> None: 130 | """ 131 | Perform a backup operation for each configured repository, including pre- and post-hooks. 132 | """ 133 | metrics = self.metrics["backup"] = {} 134 | cfg = self.config["backup"] 135 | 136 | hooks_cfg = self.config["execution"].copy() 137 | hooks_cfg.update({"parallel": False, "shell": True}) 138 | 139 | # backup pre_hooks 140 | if cfg.get("pre_hooks"): 141 | cmd_runs = MultiCommand(cfg["pre_hooks"], config=hooks_cfg).run() 142 | metrics["_restic_pre_hooks"] = { 143 | "duration_seconds": sum([v["time"] for v in cmd_runs]), 144 | "rc": sum(x["output"][-1][0] for x in cmd_runs), 145 | } 146 | 147 | # actual backup 148 | extra_args: list[str] = [] 149 | for files_from in cfg.get("files_from", []): 150 | extra_args += ["--files-from", files_from] 151 | for exclude_pattern in cfg.get("exclude_patterns", []): 152 | extra_args += ["--exclude", exclude_pattern] 153 | for exclude_file in cfg.get("exclude_files", []): 154 | extra_args += ["--exclude-file", exclude_file] 155 | for exclude_if_present in cfg.get("exclude_if_present", []): 156 | extra_args += ["--exclude-if-present", exclude_if_present] 157 | 158 | commands = [ 159 | [ 160 | "restic", 161 | "-r", 162 | repo, 163 | "backup", 164 | *self.restic_args, 165 | *extra_args, 166 | *cfg.get("sources", []), 167 | ] 168 | for repo in self.repos 169 | ] 170 | direct_abort_reasons = [ 171 | "Fatal: unable to open config file", 172 | "Fatal: wrong password", 173 | ] 174 | cmd_runs = MultiCommand( 175 | commands, self.config["execution"], direct_abort_reasons 176 | ).run() 177 | 178 | for repo, process_infos in zip(self.repos, cmd_runs): 179 | return_code = process_infos["output"][-1][0] 180 | if return_code > 0: 181 | logger.warning(process_infos) 182 | metrics[redact_password(repo, self.pw_replacement)] = { 183 | "rc": return_code 184 | } 185 | self.metrics["errors"] += 1 186 | else: 187 | metrics[redact_password(repo, self.pw_replacement)] = parse_backup( 188 | process_infos 189 | ) 190 | 191 | # backup post_hooks 192 | if cfg.get("post_hooks"): 193 | cmd_runs = MultiCommand(cfg["post_hooks"], config=hooks_cfg).run() 194 | metrics["_restic_post_hooks"] = { 195 | "duration_seconds": sum(v["time"] for v in cmd_runs), 196 | "rc": sum(x["output"][-1][0] for x in cmd_runs), 197 | } 198 | 199 | def unlock(self) -> None: 200 | """ 201 | Unlock the Restic repository for each configured repository. 202 | """ 203 | direct_abort_reasons = [ 204 | "Fatal: unable to open config file", 205 | "Fatal: wrong password", 206 | ] 207 | commands = [ 208 | ["restic", "-r", repo, "unlock", *self.restic_args] for repo in self.repos 209 | ] 210 | 211 | cmd_runs = MultiCommand( 212 | commands, 213 | config=self.config["execution"], 214 | abort_reasons=direct_abort_reasons, 215 | ).run() 216 | for process_infos in cmd_runs: 217 | if process_infos["output"][-1][0] > 0: 218 | logger.warning(process_infos["output"]) 219 | else: 220 | logger.info(process_infos["output"]) 221 | 222 | def forget(self) -> None: 223 | """ 224 | Forget old snapshots in the Restic repository based on the pruning configuration. 225 | """ 226 | metrics = self.metrics["forget"] = {} 227 | 228 | extra_args: list[str] = [] 229 | if self.args.dry_run: 230 | extra_args += ["--dry-run"] 231 | for key, value in self.config["prune"].items(): 232 | if key.startswith("keep-"): 233 | extra_args += [f"--{key}", str(value)] 234 | if key == "group-by": 235 | extra_args += ["--group-by", value] 236 | 237 | direct_abort_reasons = [ 238 | "Fatal: unable to open config file", 239 | "Fatal: wrong password", 240 | ] 241 | commands = [ 242 | ["restic", "-r", repo, "forget", *self.restic_args, *extra_args] 243 | for repo in self.repos 244 | ] 245 | cmd_runs = MultiCommand( 246 | commands, 247 | config=self.config["execution"], 248 | abort_reasons=direct_abort_reasons, 249 | ).run() 250 | 251 | for repo, process_infos in zip(self.repos, cmd_runs): 252 | return_code = process_infos["output"][-1][0] 253 | if return_code > 0: 254 | logger.warning(process_infos["output"]) 255 | metrics[redact_password(repo, self.pw_replacement)] = { 256 | "rc": return_code 257 | } 258 | self.metrics["errors"] += 1 259 | else: 260 | metrics[redact_password(repo, self.pw_replacement)] = parse_forget( 261 | process_infos 262 | ) 263 | 264 | def prune(self) -> None: 265 | """ 266 | Prune unused data from the Restic repository. 267 | """ 268 | metrics = self.metrics["prune"] = {} 269 | 270 | direct_abort_reasons = [ 271 | "Fatal: unable to open config file", 272 | "Fatal: wrong password", 273 | ] 274 | commands = [ 275 | ["restic", "-r", repo, "prune", *self.restic_args] for repo in self.repos 276 | ] 277 | cmd_runs = MultiCommand( 278 | commands, 279 | config=self.config["execution"], 280 | abort_reasons=direct_abort_reasons, 281 | ).run() 282 | 283 | for repo, process_infos in zip(self.repos, cmd_runs): 284 | return_code = process_infos["output"][-1][0] 285 | if return_code > 0: 286 | logger.warning(process_infos["output"]) 287 | metrics[redact_password(repo, self.pw_replacement)] = { 288 | "rc": return_code 289 | } 290 | self.metrics["errors"] += 1 291 | else: 292 | try: 293 | metrics[redact_password(repo, self.pw_replacement)] = ( 294 | parse_new_prune(process_infos) 295 | ) 296 | except IndexError: 297 | # assume we're dealing with restic <0.12.0 298 | metrics[redact_password(repo, self.pw_replacement)] = parse_prune( 299 | process_infos 300 | ) 301 | 302 | def check(self) -> None: 303 | """ 304 | Perform a consistency check on the Restic repository. 305 | """ 306 | self.metrics["check"] = {} 307 | 308 | extra_args: list[str] = [] 309 | cfg = self.config.get("check") 310 | if cfg and "checks" in cfg: 311 | checks = cfg["checks"] 312 | if "check-unused" in checks: 313 | extra_args += ["--check-unused"] 314 | if "read-data" in checks: 315 | extra_args += ["--read-data"] 316 | 317 | direct_abort_reasons = [ 318 | "Fatal: unable to open config file", 319 | "Fatal: wrong password", 320 | ] 321 | commands = [ 322 | ["restic", "-r", repo, "check", *self.restic_args, *extra_args] 323 | for repo in self.repos 324 | ] 325 | cmd_runs = MultiCommand( 326 | commands, 327 | config=self.config["execution"], 328 | abort_reasons=direct_abort_reasons, 329 | ).run() 330 | 331 | for repo, process_infos in zip(self.repos, cmd_runs): 332 | metrics = { 333 | "errors": 0, 334 | "errors_data": 0, 335 | "errors_snapshots": 0, 336 | "read_data": 1 if "--read-data" in extra_args else 0, 337 | "check_unused": 1 if "--check-unused" in extra_args else 0, 338 | } 339 | return_code, output = process_infos["output"][-1] 340 | if return_code > 0: 341 | logger.warning(process_infos["output"]) 342 | self.metrics["errors"] += 1 343 | if "error: load None: 354 | """ 355 | Collect statistics for the Restic repository. 356 | """ 357 | metrics = self.metrics["stats"] = {} 358 | 359 | direct_abort_reasons = [ 360 | "Fatal: unable to open config file", 361 | "Fatal: wrong password", 362 | ] 363 | # quiet and verbose arguments are mutually exclusive 364 | verbose = re.compile(r"^--verbose") 365 | quiet = [] if list(filter(verbose.match, self.restic_args)) else ["-q"] 366 | commands = [ 367 | ["restic", "-r", repo, "stats", "--json", *quiet, *self.restic_args] 368 | for repo in self.repos 369 | ] 370 | cmd_runs = MultiCommand( 371 | commands, 372 | config=self.config["execution"], 373 | abort_reasons=direct_abort_reasons, 374 | ).run() 375 | 376 | for repo, process_infos in zip(self.repos, cmd_runs): 377 | return_code = process_infos["output"][-1][0] 378 | if return_code > 0: 379 | logger.warning(process_infos["output"]) 380 | metrics[redact_password(repo, self.pw_replacement)] = { 381 | "rc": return_code 382 | } 383 | self.metrics["errors"] += 1 384 | else: 385 | metrics[redact_password(repo, self.pw_replacement)] = parse_stats( 386 | process_infos 387 | ) 388 | -------------------------------------------------------------------------------- /runrestic/restic/shell.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module provides functionality to interact with Restic repositories via a shell. 3 | 4 | It allows users to select a repository from a list of available configurations and spawns 5 | a new shell with the appropriate environment variables set for Restic operations. 6 | """ 7 | 8 | import logging 9 | import os 10 | import pty 11 | import sys 12 | from typing import Any 13 | 14 | from runrestic.restic.tools import initialize_environment 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | def restic_shell(configs: list[dict[str, Any]]) -> None: 20 | """ 21 | Launch a shell with environment variables set for a selected Restic repository. 22 | 23 | If only one repository is available, it is automatically selected. Otherwise, the user 24 | is prompted to choose a repository from the available configurations. 25 | 26 | Args: 27 | configs (list[dict[str, Any]]): A list of configuration dictionaries, each containing 28 | repository information and environment variables. 29 | 30 | Raises: 31 | ValueError: If the user provides an invalid selection index. 32 | """ 33 | if len(configs) == 1 and len(configs[0]["repositories"]) == 1: 34 | logger.info("Found only one repository, using that one:\n") 35 | selected_config = configs[0] 36 | selected_repo = configs[0]["repositories"][0] 37 | else: 38 | print("The following repositories are available:") 39 | all_repos: list[tuple[dict[str, Any], str]] = [] 40 | i = 0 41 | for config in configs: 42 | for repo in config["repositories"]: 43 | print(f"[{i}] - {config['name']}:{repo}") 44 | all_repos.append((config, repo)) 45 | i += 1 46 | 47 | selection = int(input(f"Choose a repo [0-{i - 1}]: ")) 48 | selected_config, selected_repo = all_repos[selection] 49 | 50 | env: dict[str, str] = selected_config["environment"] 51 | env.update({"RESTIC_REPOSITORY": selected_repo}) 52 | 53 | print(f"Using: \033[1;92m{selected_config['name']}:{selected_repo}\033[0m") 54 | print("Spawning a new shell with the restic environment variables all set.") 55 | initialize_environment(env) 56 | print("\nTry `restic snapshots` for example.") 57 | pty.spawn(os.environ["SHELL"]) 58 | print("You've exited your restic shell.") 59 | sys.exit(0) 60 | -------------------------------------------------------------------------------- /runrestic/restic/tools.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module provides utility functions and classes to support Restic operations. 3 | 4 | It includes: 5 | - `MultiCommand` for executing multiple commands in parallel or sequentially. 6 | - Functions for logging process output, retrying commands, initializing environment variables, 7 | and redacting sensitive information from logs. 8 | """ 9 | 10 | import logging 11 | import os 12 | import re 13 | import time 14 | from concurrent.futures import Future 15 | from concurrent.futures.process import ProcessPoolExecutor 16 | from subprocess import PIPE, STDOUT, Popen 17 | from typing import Any, Sequence 18 | 19 | from runrestic.runrestic.tools import parse_time 20 | 21 | logger = logging.getLogger(__name__) 22 | 23 | 24 | class MultiCommand: 25 | """ 26 | A class to execute multiple commands in parallel or sequentially, with support for retries and abort conditions. 27 | 28 | Attributes: 29 | commands (Sequence[list[str] | str]): List of commands to execute. 30 | config (dict): Configuration dictionary for command execution. 31 | abort_reasons (list[str] | None): List of reasons to abort execution if found in the output. 32 | """ 33 | 34 | def __init__( 35 | self, 36 | commands: Sequence[list[str] | str], 37 | config: dict[str, Any], 38 | abort_reasons: list[str] | None = None, 39 | ) -> None: 40 | """ 41 | Initialize the MultiCommand instance. 42 | 43 | Args: 44 | commands (Sequence[list[str] | str]): List of commands to execute. 45 | config (dict): Configuration dictionary for command execution. 46 | abort_reasons (list[str] | None): List of reasons to abort execution if found in the output. 47 | """ 48 | self.processes: list[Future[dict[str, Any]]] = [] 49 | self.commands = commands 50 | self.config = config 51 | self.abort_reasons = abort_reasons 52 | self.process_pool_executor = ProcessPoolExecutor( 53 | max_workers=len(commands) if config["parallel"] else 1 54 | ) 55 | 56 | def run(self) -> list[dict[str, Any]]: 57 | """ 58 | Execute all commands and collect their results. 59 | 60 | Returns: 61 | list[dict[str, Any]]: List of results for each command. 62 | """ 63 | for command in self.commands: 64 | logger.debug("Spawning %s", command) 65 | process = self.process_pool_executor.submit( 66 | retry_process, command, self.config, self.abort_reasons 67 | ) 68 | self.processes.append(process) 69 | 70 | # result() is blocking. The function will return when all processes are done 71 | return [process.result() for process in self.processes] 72 | 73 | 74 | def log_messages(process: Any, proc_cmd: str) -> str: 75 | """ 76 | Capture the process output and generate appropriate log messages. 77 | 78 | Args: 79 | process (Any): The process object from which to capture output. 80 | proc_cmd (str): Name of the executed command (as it should appear in the logs). 81 | 82 | Returns: 83 | str: Complete process output. 84 | """ 85 | output = "" 86 | for log_out in process.stdout: 87 | if log_out.strip(): 88 | output += log_out 89 | if re.match(r"^critical|fatal", log_out, re.I): 90 | proc_log_level = logging.CRITICAL 91 | elif re.match(r"^error", log_out, re.I): 92 | proc_log_level = logging.ERROR 93 | elif re.match(r"^warning", log_out, re.I): 94 | proc_log_level = logging.WARNING 95 | elif re.match( 96 | r"^unchanged\s+/", log_out, re.I 97 | ): # unchanged files in restic output 98 | proc_log_level = logging.DEBUG 99 | else: 100 | proc_log_level = logging.INFO 101 | logger.log(proc_log_level, "[%s] %s", proc_cmd, log_out.strip()) 102 | return output 103 | 104 | 105 | def retry_process( 106 | cmd: str | list[str], 107 | config: dict[str, Any], 108 | abort_reasons: list[str] | None = None, 109 | ) -> dict[str, Any]: 110 | """ 111 | Execute a command with retries and optional abort conditions. 112 | 113 | Args: 114 | cmd (str | list[str]): Command to execute. 115 | config (dict[str, Any]): Configuration dictionary for command execution. 116 | abort_reasons (list[str] | None): List of reasons to abort execution if found in the output. 117 | 118 | Returns: 119 | dict[str, Any]: Status and output of the command execution. 120 | """ 121 | start_time = time.time() 122 | 123 | shell = config.get("shell", False) 124 | tries_total = config.get("retry_count", 0) + 1 125 | status = {"current_try": 0, "tries_total": tries_total, "output": []} 126 | proc_cmd = ( 127 | cmd[0] 128 | if isinstance(cmd, list) 129 | else os.path.basename(cmd.split(" ", maxsplit=1)[0]) 130 | ) 131 | for i in range(tries_total): 132 | status["current_try"] = i + 1 133 | 134 | with Popen( 135 | cmd, stdout=PIPE, stderr=STDOUT, shell=shell, encoding="UTF-8" 136 | ) as process: # noqa: S603 137 | output = log_messages(process, proc_cmd) 138 | returncode = process.returncode 139 | status["output"].append((returncode, output)) 140 | if returncode == 0: 141 | break 142 | 143 | if abort_reasons and any( 144 | abort_reason in output for abort_reason in abort_reasons 145 | ): 146 | logger.error( 147 | "Aborting '%s' because of %s", 148 | proc_cmd, 149 | [ 150 | abort_reason 151 | for abort_reason in abort_reasons 152 | if abort_reason in output 153 | ], 154 | ) 155 | break 156 | if config.get("retry_backoff"): 157 | if " " in config["retry_backoff"]: 158 | duration, strategy = config["retry_backoff"].split(" ") 159 | else: 160 | duration, strategy = config["retry_backoff"], None 161 | duration = parse_time(duration) 162 | logger.info( 163 | "Retry %s/%s command '%s' using %s strategy, duration = %s sec", 164 | i + 1, 165 | tries_total, 166 | proc_cmd, 167 | strategy, 168 | duration, 169 | ) 170 | 171 | if strategy == "linear": 172 | time.sleep(duration * (i + 1)) 173 | elif strategy == "exponential": 174 | time.sleep(duration << i) 175 | else: # strategy = "static" 176 | time.sleep(duration) 177 | else: 178 | logger.info( 179 | "Retry %s/%s command '%s'", 180 | i + 1, 181 | tries_total, 182 | proc_cmd, 183 | ) 184 | 185 | status["time"] = time.time() - start_time 186 | return status 187 | 188 | 189 | def initialize_environment(config: dict[str, Any]) -> None: 190 | """ 191 | Set environment variables based on the provided configuration. 192 | 193 | Args: 194 | config (dict[str, Any]): Dictionary of environment variables to set. 195 | """ 196 | for key, value in config.items(): 197 | os.environ[key] = value 198 | if key == "RESTIC_PASSWORD": 199 | value = "**********" 200 | logger.debug("[Environment] %s=%s", key, value) 201 | 202 | if os.geteuid() == 0 or not ( 203 | os.environ.get("HOME") or os.environ.get("XDG_CACHE_HOME") 204 | ): # pragma: no cover; if user is root, we just use system cache 205 | os.environ["XDG_CACHE_HOME"] = "/var/cache" 206 | 207 | 208 | def redact_password(repo_str: str, pw_replacement: str) -> str: 209 | """ 210 | Redact sensitive information (e.g., passwords) from a repository string. 211 | 212 | Args: 213 | repo_str (str): Repository string containing sensitive information. 214 | pw_replacement (str): Replacement string for sensitive information. 215 | 216 | Returns: 217 | str: Repository string with sensitive information redacted. 218 | """ 219 | re_repo = re.compile(r"(^(?:[s]?ftp:|rest:http[s]?:|s3:http[s]?:).*?):(\S+)(@.*$)") 220 | return ( 221 | re_repo.sub(rf"\1:{pw_replacement}\3", repo_str) 222 | if pw_replacement 223 | else re_repo.sub(r"\1\3", repo_str) 224 | ) 225 | -------------------------------------------------------------------------------- /runrestic/runrestic/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sinnwerkstatt/runrestic/4ff8868628e5ca4633ac666e486add0b3a8e702f/runrestic/runrestic/__init__.py -------------------------------------------------------------------------------- /runrestic/runrestic/configuration.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module provides functionality for parsing CLI arguments and reading configuration files. 3 | 4 | It includes utilities to handle command-line arguments, locate configuration files, and parse 5 | them into structured data. The module also validates configuration files against a predefined 6 | JSON schema to ensure correctness. 7 | """ 8 | 9 | import json 10 | import logging 11 | import os 12 | from argparse import ArgumentParser, Namespace 13 | from importlib.resources import open_text 14 | from typing import Any 15 | 16 | import jsonschema 17 | import toml 18 | 19 | from runrestic import __version__ 20 | from runrestic.runrestic.tools import deep_update 21 | 22 | logger = logging.getLogger(__name__) 23 | 24 | CONFIG_DEFAULTS: dict[str, Any] = { 25 | "execution": { 26 | "parallel": False, 27 | "exit_on_error": True, 28 | "retry_count": 0, 29 | } 30 | } 31 | with open_text("runrestic.runrestic", "schema.json", encoding="utf-8") as schema_file: 32 | SCHEMA: dict[str, Any] = json.load(schema_file) 33 | 34 | 35 | def cli_arguments(args: list[str] | None = None) -> tuple[Namespace, list[str]]: 36 | """ 37 | Parse command-line arguments for the `runrestic` application. 38 | 39 | Args: 40 | args (list[str] | None): A list of arguments to parse. If None, uses `sys.argv`. 41 | 42 | Returns: 43 | tuple[Namespace, list[str]]: A tuple containing parsed options and extra arguments. 44 | """ 45 | parser = ArgumentParser( 46 | prog="runrestic", 47 | description=""" 48 | A wrapper for restic. It runs restic based on config files and also outputs metrics. 49 | To initialize the repos, run `runrestic init`. 50 | If you don't define any actions, it will default to `backup prune check`, and `stats` if metrics are set. 51 | """, 52 | ) 53 | parser.add_argument( 54 | "actions", 55 | type=str, 56 | nargs="*", 57 | help="one or more from the following actions: [shell, init, backup, prune, check, stats, unlock]", 58 | ) 59 | parser.add_argument( 60 | "-n", 61 | "--dry-run", 62 | dest="dry_run", 63 | action="store_true", 64 | help="Apply --dry-run where applicable (i.e.: forget)", 65 | ) 66 | parser.add_argument( 67 | "-l", 68 | "--log-level", 69 | metavar="LOG_LEVEL", 70 | dest="log_level", 71 | default="info", 72 | help="Choose from: critical, error, warning, info, debug. (default: info)", 73 | ) 74 | parser.add_argument( 75 | "-c", 76 | "--config", 77 | dest="config_file", 78 | help="Use an alternative configuration file", 79 | ) 80 | parser.add_argument( 81 | "--show-progress", 82 | metavar="INTERVAL", 83 | help="Updated interval in seconds for restic progress (default: None)", 84 | ) 85 | parser.add_argument( 86 | "-v", "--version", action="version", version="%(prog)s " + __version__ 87 | ) 88 | 89 | options, extras = parser.parse_known_args(args) 90 | if extras: 91 | extras = [x for x in extras if x != "--"] 92 | else: 93 | valid_actions = ["shell", "init", "backup", "prune", "check", "stats", "unlock"] 94 | extras = [] 95 | new_actions: list[str] = [] 96 | for act in options.actions: 97 | if act in valid_actions: 98 | new_actions += [act] 99 | else: 100 | extras += [act] 101 | options.actions = new_actions 102 | return options, extras 103 | 104 | 105 | def possible_config_paths() -> list[str]: 106 | """ 107 | Generate a list of possible configuration file paths. 108 | 109 | Returns: 110 | list[str]: A list of paths where configuration files might be located. 111 | """ 112 | user_config_directory = os.getenv("XDG_CONFIG_HOME") or os.path.expandvars( 113 | os.path.join("$HOME", ".config") 114 | ) 115 | return [ 116 | "/etc/runrestic.toml", 117 | "/etc/runrestic.json", 118 | "/etc/runrestic/", 119 | f"{user_config_directory}/runrestic/", 120 | ] 121 | 122 | 123 | def configuration_file_paths() -> list[str]: 124 | """ 125 | Locate readable configuration files from possible paths. 126 | 127 | Returns: 128 | list[str]: A list of valid configuration file paths. 129 | """ 130 | paths: list[str] = [] 131 | for path in possible_config_paths(): 132 | path = os.path.realpath(path) 133 | # Check access permission, includes check for path existence 134 | if not os.access(path, os.R_OK): 135 | logger.debug("No access to path %s skipping", path) 136 | continue 137 | 138 | if os.path.isfile(path): 139 | paths += [path] 140 | continue 141 | 142 | for filename in os.listdir(path): 143 | filename = os.path.join(path, filename) 144 | if ( 145 | filename.endswith(".toml") or filename.endswith(".json") 146 | ) and os.path.isfile(filename): 147 | octal_permissions = oct(os.stat(filename).st_mode) 148 | if octal_permissions[-2:] != "00": # file permissions are too broad 149 | logger.warning( 150 | "NOT using %s.\n" 151 | "File permissions are too open (%s). " 152 | "You should set it to 0600: `chmod 0600 %s`\n", 153 | filename, 154 | octal_permissions[-4:], 155 | filename, 156 | ) 157 | continue 158 | 159 | paths += [filename] 160 | 161 | return paths 162 | 163 | 164 | def parse_configuration(config_filename: str) -> dict[str, Any]: 165 | """ 166 | Parse a configuration file and validate it against the schema. 167 | 168 | Args: 169 | config_filename (str): The path to the configuration file. 170 | 171 | Returns: 172 | dict[str, Any]: The parsed and validated configuration as a dictionary. 173 | """ 174 | logger.debug("Parsing configuration file: %s", config_filename) 175 | with open(config_filename, encoding="utf-8") as file: 176 | config: dict[str, Any] = ( 177 | toml.load(file) 178 | if str(config_filename).endswith(".toml") 179 | else json.load(file) 180 | ) 181 | config = deep_update(CONFIG_DEFAULTS, dict(config)) 182 | 183 | if "name" not in config: 184 | config["name"] = os.path.basename(config_filename) 185 | 186 | jsonschema.validate(instance=config, schema=SCHEMA) 187 | return config 188 | -------------------------------------------------------------------------------- /runrestic/runrestic/runrestic.py: -------------------------------------------------------------------------------- 1 | """ 2 | Runrestic main module. 3 | 4 | This module serves as the entry point for the `runrestic` application. It handles 5 | logging configuration, signal handling, and the execution of Restic operations 6 | based on user-provided configuration files and command-line arguments. 7 | """ 8 | 9 | import logging 10 | import os 11 | import signal 12 | import sys 13 | from typing import Any 14 | 15 | from runrestic.restic.installer import restic_check 16 | from runrestic.restic.runner import ResticRunner 17 | from runrestic.restic.shell import restic_shell 18 | from runrestic.runrestic.configuration import ( 19 | cli_arguments, 20 | configuration_file_paths, 21 | parse_configuration, 22 | possible_config_paths, 23 | ) 24 | 25 | logger = logging.getLogger(__name__) 26 | 27 | 28 | def configure_logging(level: str) -> None: 29 | """ 30 | Configure logging for the application. 31 | 32 | Args: 33 | level (str): The logging level as a string (e.g., "info", "debug"). 34 | """ 35 | level = logging.getLevelName(level.upper()) 36 | log = logging.getLogger("runrestic") 37 | log.setLevel(level) 38 | handler = logging.StreamHandler() 39 | handler.setLevel(level) 40 | formatter = logging.Formatter("%(message)s") 41 | handler.setFormatter(formatter) 42 | log.addHandler(handler) 43 | 44 | 45 | def configure_signals() -> None: 46 | """ 47 | Configure signal handling for the application. 48 | 49 | This function ensures that the application properly handles termination signals 50 | by killing the entire process group. 51 | """ 52 | 53 | def kill_the_group(signal_number: signal.Signals, _frame: Any) -> None: 54 | """ 55 | Kill the entire process group when a signal is received. 56 | 57 | Args: 58 | signal_number (signal.Signals): The signal received. 59 | _frame (Any): The current stack frame (unused). 60 | """ 61 | os.killpg(os.getpgrp(), signal_number) 62 | 63 | signals = [ 64 | signal.SIGINT, 65 | signal.SIGHUP, 66 | signal.SIGTERM, 67 | signal.SIGUSR1, 68 | signal.SIGUSR2, 69 | ] 70 | 71 | _ = [signal.signal(sig, kill_the_group) for sig in signals] # type: ignore[arg-type] 72 | 73 | 74 | def runrestic() -> None: 75 | """ 76 | Main function for the `runrestic` application. 77 | 78 | This function checks for the Restic binary, parses command-line arguments, 79 | configures logging and signals, loads configuration files, and executes 80 | the specified Restic actions. 81 | """ 82 | if not restic_check(): 83 | return 84 | 85 | args, extras = cli_arguments() 86 | configure_logging(args.log_level) 87 | configure_signals() 88 | 89 | if args.config_file: 90 | config_file_paths = [args.config_file] 91 | else: 92 | config_file_paths = list(configuration_file_paths()) 93 | 94 | if not config_file_paths: 95 | raise FileNotFoundError( 96 | f"Error: No configuration files found in {possible_config_paths()}" 97 | ) # noqa: TRY003 98 | 99 | configs: list[dict[str, Any]] = [] 100 | for config in config_file_paths: 101 | parsed_cfg = parse_configuration(config) 102 | if parsed_cfg: 103 | configs.append(parsed_cfg) 104 | 105 | if args.show_progress: 106 | os.environ["RESTIC_PROGRESS_FPS"] = str(1 / float(args.show_progress)) 107 | 108 | if "shell" in args.actions: 109 | restic_shell(configs) 110 | return 111 | 112 | # Track the results (number of errors) per config 113 | result: list[int] = [] 114 | for config in configs: 115 | runner = ResticRunner(config, args, extras) 116 | result.append(runner.run()) 117 | 118 | if sum(result) > 0: 119 | sys.exit(1) 120 | 121 | 122 | if __name__ == "__main__": 123 | runrestic() 124 | -------------------------------------------------------------------------------- /runrestic/runrestic/schema.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$id": "https://raw.githubusercontent.com/sinnwerkstatt/runrestic/main/runrestic/config/schema.json", 4 | "title": "Runrestic Config", 5 | "description": "Schema for runrestic configuration files, written in TOML", 6 | "type": "object", 7 | "required": [ 8 | "repositories", 9 | "environment", 10 | "backup", 11 | "prune" 12 | ], 13 | "properties": { 14 | "name": {"type": "string"}, 15 | 16 | "repositories": { 17 | "type": "array", 18 | "items": {"type": "string"}, 19 | "minItems": 1, 20 | "uniqueItems": true 21 | }, 22 | 23 | "execution": { 24 | "type": "object", 25 | "properties": { 26 | "retry_count": {"type": "integer"}, 27 | "retry_backoff": {"type": "string"}, 28 | "parallel": { 29 | "type": "boolean", 30 | "default": false 31 | }, 32 | "exit_on_error": { 33 | "type": "boolean", 34 | "default": true 35 | } 36 | } 37 | }, 38 | 39 | "environment": { 40 | "type": "object", 41 | "properties": { 42 | "RESTIC_PASSWORD": {"type": "string"}, 43 | "RESTIC_PASSWORD_FILE": {"type": "string"} 44 | }, 45 | "additionalProperties": { "type": "string" }, 46 | "oneOf": [ 47 | {"required": ["RESTIC_PASSWORD"]}, 48 | {"required": ["RESTIC_PASSWORD_FILE"]} 49 | ] 50 | }, 51 | 52 | "backup": { 53 | "type": "object", 54 | "oneOf": [ 55 | {"required": ["sources"]}, 56 | {"required": ["files_from"]} 57 | ], 58 | "properties": { 59 | "sources": { 60 | "type": "array", 61 | "items": {"type": "string"}, 62 | "minItems": 1, 63 | "uniqueItems": true 64 | }, 65 | "files_from ": {"type": "array", "items": {"type": "string"}}, 66 | "exclude_patterns": {"type": "array", "items": {"type": "string"}}, 67 | "exclude_files": {"type": "array", "items": {"type": "string"}}, 68 | "exclude_if_present": {"type": "array", "items": {"type": "string"}}, 69 | "pre_hooks": {"type": "array", "items": {"type": "string"}}, 70 | "post_hooks": {"type": "array", "items": {"type": "string"}}, 71 | "continue_on_pre_hooks_error": {"type": "boolean", "default": false} 72 | } 73 | }, 74 | 75 | "prune": { 76 | "type": "object", 77 | "anyOf": [ 78 | {"required": ["keep-last"]}, 79 | {"required": ["keep-hourly"]}, 80 | {"required": ["keep-daily"]}, 81 | {"required": ["keep-weekly"]}, 82 | {"required": ["keep-monthly"]}, 83 | {"required": ["keep-yearly"]}, 84 | {"required": ["keep-within"]}, 85 | {"required": ["keep-tag"]} 86 | ], 87 | "properties": { 88 | "keep-last": {"type": "integer"}, 89 | "keep-hourly": {"type": "integer"}, 90 | "keep-daily": {"type": "integer"}, 91 | "keep-weekly": {"type": "integer"}, 92 | "keep-monthly": {"type": "integer"}, 93 | "keep-yearly": {"type": "integer"}, 94 | "keep-within": {"type": "string"}, 95 | "keep-tag": {"type": "string"}, 96 | "group-by": {"type": "string"} 97 | } 98 | }, 99 | 100 | "check": { 101 | "type": "object", 102 | "properties": { 103 | "checks": { 104 | "type": "array", 105 | "items": {"type": "string"}, 106 | "default": ["check-unused", "read-data"] 107 | } 108 | } 109 | }, 110 | 111 | "metrics": { 112 | "type": "object", 113 | "properties": { 114 | "prometheus": { 115 | "type": "object", 116 | "required": ["path"], 117 | "properties": { 118 | "path": {"type": "string"}, 119 | "pw-replacement": {"type": "string"} 120 | } 121 | } 122 | } 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /runrestic/runrestic/tools.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module provides utility functions for parsing and manipulating data related to Restic operations. 3 | 4 | It includes functions to parse sizes, times, and lines of text using regular expressions, as well as 5 | a utility to deeply update nested dictionaries. These functions are used throughout the application 6 | to process and format data. 7 | """ 8 | 9 | import logging 10 | import re 11 | from typing import Any, TypeVar 12 | 13 | logger = logging.getLogger(__name__) 14 | 15 | 16 | def make_size(size: int) -> str: 17 | """ 18 | Convert a size in bytes to a human-readable string with appropriate units. 19 | 20 | Args: 21 | size (int): The size in bytes. 22 | 23 | Returns: 24 | str: The size formatted as a human-readable string (e.g., "1.23 GiB"). 25 | """ 26 | if size > 1 << 40: 27 | return f"{size / (1 << 40):.2f} TiB" 28 | if size > 1 << 30: 29 | return f"{size / (1 << 30):.2f} GiB" 30 | if size > 1 << 20: 31 | return f"{size / (1 << 20):.2f} MiB" 32 | if size > 1 << 10: 33 | return f"{size / (1 << 10):.2f} KiB" 34 | return f"{size:.0f} B" 35 | 36 | 37 | def parse_size(size: str) -> float: 38 | """ 39 | Parse a human-readable size string into a size in bytes. 40 | 41 | Args: 42 | size (str): The size string (e.g., "1.23 GiB"). 43 | 44 | Returns: 45 | float: The size in bytes. Returns 0.0 if parsing fails. 46 | """ 47 | re_bytes = re.compile(r"([0-9.]+) ?([a-zA-Z]*B)") 48 | try: 49 | number, unit = re_bytes.findall(size)[0] 50 | except IndexError: 51 | logger.error("Failed to parse size of '%s'", size) 52 | return 0.0 53 | units = { 54 | "B": 1, 55 | "kB": 10**3, 56 | "MB": 10**6, 57 | "GB": 10**9, 58 | "TB": 10**12, 59 | "KiB": 1024, 60 | "MiB": 2**20, 61 | "GiB": 2**30, 62 | "TiB": 2**40, 63 | } 64 | return float(number) * units.get(unit, 1) 65 | 66 | 67 | def parse_time(time_str: str) -> int: 68 | """ 69 | Parse a time string in the format "HH:MM:SS" or "MM:SS" into seconds. 70 | 71 | Args: 72 | time_str (str): The time string to parse. 73 | 74 | Returns: 75 | int: The total time in seconds. Returns 0 if parsing fails. 76 | """ 77 | re_time = re.compile(r"(?:([0-9]+):)?([0-9]+):([0-9]+)") 78 | try: 79 | hours, minutes, seconds = ( 80 | int(x) if x else 0 for x in re_time.findall(time_str)[0] 81 | ) 82 | except IndexError: 83 | logger.error("Failed to parse time of '%s'", time_str) 84 | return 0 85 | if minutes: 86 | seconds += minutes * 60 87 | if hours: 88 | seconds += hours * 3600 89 | return seconds 90 | 91 | 92 | def deep_update(base: dict[Any, Any], update: dict[Any, Any]) -> dict[Any, Any]: 93 | """ 94 | Recursively update a nested dictionary with values from another dictionary. 95 | 96 | Args: 97 | base (dict[Any, Any]): The base dictionary to update. 98 | update (dict[Any, Any]): The dictionary with updates. 99 | 100 | Returns: 101 | dict[Any, Any]: A new dictionary with the updates applied. 102 | """ 103 | new = base.copy() 104 | for key, value in update.items(): 105 | base_value = new.get(key, {}) 106 | if not isinstance(base_value, dict): 107 | new[key] = value 108 | elif isinstance(value, dict): 109 | new[key] = deep_update(base_value, value) 110 | else: 111 | new[key] = value 112 | return new 113 | 114 | 115 | ParsedType = TypeVar("ParsedType", str, tuple[str, ...]) 116 | 117 | 118 | def parse_line( 119 | regex: str, 120 | output: str, 121 | default: ParsedType, 122 | ) -> ParsedType: 123 | r""" 124 | Parse a line of text using a regular expression and return matched variables. 125 | 126 | If there is no match in the output, the variables will be returned with their default values. 127 | 128 | Args: 129 | regex (str): The regular expression to match the requested variables. 130 | output (str): The text output to be parsed. 131 | default (T): Default values to return if parsing fails. 132 | 133 | Returns: 134 | ParsedType: The parsed result or the default values. 135 | 136 | Examples: 137 | >>> parse_line( 138 | ... regex=r"Files:\s+([0-9]+) new,\s+([0-9]+) changed,\s+([0-9]+) unmodified", 139 | ... output="Files: 10 new, 5 changed, 20 unmodified", 140 | ... default=("0", "0", "0") 141 | ... ) 142 | ('10', '5', '20') 143 | """ 144 | try: 145 | parsed = re.findall(regex, output)[0] 146 | except IndexError: 147 | logger.error("No match in output for regex '%s'", regex) 148 | return default 149 | if isinstance(parsed, type(default)): 150 | return parsed 151 | else: 152 | logger.error( 153 | f"The format of the parsed output '{parsed}' does not match the expected format as per default '{default}'.", 154 | ) 155 | return default 156 | -------------------------------------------------------------------------------- /sample/cron/runrestic: -------------------------------------------------------------------------------- 1 | # You can drop this file into /etc/cron.d/ to run borgmatic nightly. 2 | 3 | 0 3 * * * root PATH=$PATH:/usr/local/bin /usr/local/bin/runrestic 4 | -------------------------------------------------------------------------------- /sample/example.toml: -------------------------------------------------------------------------------- 1 | name = "postgresql backup" # optional. if not set, the filename will be used without the extension 2 | 3 | repositories = [ 4 | "/tmp/restic-repo1", 5 | "s3:s3.amazonaws.com/bucket_name" 6 | ] 7 | 8 | [execution] 9 | parallel = true 10 | retry_count = 10 11 | retry_backoff = "1:00 exponential" # 00:00 = min:sec; 00:00:00 = hour:min:sec 12 | # strategies: 13 | # - static (same duration every try) 14 | # - linear (duration * retry number) 15 | # - exponential 16 | 17 | [environment] 18 | RESTIC_PASSWORD = "CHANGEME" 19 | # or RESTIC_PASSWORD_FILE 20 | # https://restic.readthedocs.io/en/latest/040_backup.html#environment-variables 21 | 22 | [backup] 23 | sources = [ 24 | "/var/lib/postgresql", 25 | "/etc/postgresql", 26 | "/tmp/pgdump.sql" 27 | ] 28 | 29 | exclude_patterns = ['pg_stats_tmp/'] 30 | # exclude_files = [] 31 | # exclude_if_present = [] 32 | 33 | pre_hooks = ["systemctl stop postgresql"] 34 | post_hooks = ["systemctl start postgresql"] 35 | 36 | [prune] 37 | keep-last = 3 38 | keep-hourly = 5 39 | keep-weekly = 10 40 | keep-monthly = 30 41 | group-by = "host,paths" 42 | # https://restic.readthedocs.io/en/latest/060_forget.html#removing-snapshots-according-to-a-policy 43 | 44 | [check] 45 | checks = ["check-unused", "read-data"] 46 | 47 | 48 | [metrics.prometheus] 49 | path = "/var/lib/node_exporter/textfile_collector/runrestic.prom" 50 | # password_replacement = "XXX" # use this if you need to redact passwords from repos in the log file #39 51 | -------------------------------------------------------------------------------- /sample/systemd/runrestic.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=runrestic backup 3 | 4 | [Service] 5 | Type=oneshot 6 | ExecStart=/usr/local/bin/runrestic 7 | -------------------------------------------------------------------------------- /sample/systemd/runrestic.timer: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Run runrestic backup 3 | 4 | [Timer] 5 | OnCalendar=daily 6 | Persistent=true 7 | 8 | [Install] 9 | WantedBy=timers.target 10 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # This file is only here for legacy reasons. 2 | 3 | import os.path 4 | 5 | from setuptools import setup 6 | 7 | here = os.path.abspath(os.path.dirname(__file__)) 8 | readme_path = os.path.join(here, "README.md") 9 | 10 | setup( 11 | long_description=open(readme_path, "r", encoding="utf-8").read(), 12 | long_description_content_type="text/markdown", 13 | name="runrestic", 14 | version="0.5.30", 15 | description="A wrapper script for Restic backup software that inits, creates, prunes and checks backups", 16 | python_requires=">=3.10.0,<4.0", 17 | project_urls={ 18 | "homepage": "https://github.com/sinnwerkstatt/runrestic", 19 | "repository": "https://github.com/sinnwerkstatt/runrestic", 20 | }, 21 | author="Andreas Nüßlein", 22 | author_email="andreas@nuessle.in", 23 | license="GPL-3.0+", 24 | keywords="backup", 25 | classifiers=[ 26 | "Development Status :: 4 - Beta", 27 | "Environment :: Console", 28 | "Intended Audience :: System Administrators", 29 | "Programming Language :: Python", 30 | "Topic :: Security :: Cryptography", 31 | "Topic :: System :: Archiving :: Backup", 32 | ], 33 | entry_points={ 34 | "console_scripts": ["runrestic = runrestic.runrestic.runrestic:runrestic"] 35 | }, 36 | packages=[ 37 | "runrestic", 38 | "runrestic.metrics", 39 | "runrestic.restic", 40 | "runrestic.runrestic", 41 | ], 42 | package_data={"runrestic.runrestic": ["*.json"]}, 43 | install_requires=[ 44 | "jsonschema>=3.0", 45 | "requests>=2.27.1,<3.0.0", 46 | "toml>=0.10,<0.11", 47 | ], 48 | ) 49 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sinnwerkstatt/runrestic/4ff8868628e5ca4633ac666e486add0b3a8e702f/tests/__init__.py -------------------------------------------------------------------------------- /tests/metrics/test_metrics.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | from unittest import TestCase 3 | from unittest.mock import mock_open, patch 4 | 5 | from runrestic.metrics import prometheus, write_metrics 6 | 7 | 8 | class TestResticMetrics(TestCase): 9 | @patch("builtins.open", new_callable=mock_open) 10 | @patch( 11 | "runrestic.metrics.prometheus.generate_lines", 12 | return_value=["line1\n", "line2\n"], 13 | ) 14 | def test_write_metrics(self, mock_generate_lines, mock_open): 15 | cfg = { 16 | "name": "test", 17 | "metrics": {"prometheus": {"path": "/prometheus_path"}}, 18 | } 19 | metrics = { 20 | "backup": { 21 | "repo1": { 22 | "files": {"new": "1", "changed": "2", "unmodified": "3"}, 23 | }, 24 | } 25 | } 26 | write_metrics(metrics, cfg) 27 | mock_generate_lines.assert_called_once_with(metrics, "test") 28 | mock_open.assert_called_once_with("/prometheus_path", "w") 29 | fh = mock_open() 30 | fh.writelines.assert_called_once_with("line1\nline2\n") 31 | 32 | @patch( 33 | "runrestic.metrics.prometheus.generate_lines", 34 | return_value=["line1\n", "line2\n"], 35 | ) 36 | def test_write_metrics_skipped(self, mock_generate_lines): 37 | cfg = { 38 | "name": "test", 39 | "metrics": {"unknown": {"path": "/other_path"}}, 40 | } 41 | metrics = {"dummy": "metrics"} 42 | write_metrics(metrics, cfg) 43 | mock_generate_lines.assert_not_called() 44 | 45 | 46 | def mock_metrics_func(metrics, name): 47 | return f"{name}: {metrics}" 48 | 49 | 50 | class TestResticMetricsPrometheus(TestCase): 51 | def setUp(self) -> None: 52 | prometheus._restic_help_pre_hooks = "restic_help_pre_hooks|" 53 | prometheus._restic_pre_hooks = "pre:{name}:{rc}:{duration_seconds}|" 54 | prometheus._restic_help_post_hooks = "restic_help_post_hooks|" 55 | prometheus._restic_post_hooks = "post:{name}:{rc}:{duration_seconds}|" 56 | 57 | # def setUp(self) -> None: 58 | # self.temp_dir: tempfile.TemporaryDirectory = tempfile.TemporaryDirectory() 59 | # self.prometheus_path = Path(self.temp_dir.name) / "example.prom" 60 | # self.prometheus_path.mkdir() 61 | 62 | # def tearDown(self) -> None: 63 | # self.temp_dir.cleanup() 64 | 65 | @patch("runrestic.metrics.prometheus.backup_metrics", wraps=mock_metrics_func) 66 | @patch("runrestic.metrics.prometheus.forget_metrics", wraps=mock_metrics_func) 67 | @patch("runrestic.metrics.prometheus.prune_metrics", wraps=mock_metrics_func) 68 | @patch("runrestic.metrics.prometheus.check_metrics", wraps=mock_metrics_func) 69 | @patch("runrestic.metrics.prometheus.stats_metrics", wraps=mock_metrics_func) 70 | def test_generate_lines( 71 | self, 72 | mock_stats_metrics, 73 | mock_check_metrics, 74 | mock_prune_metrics, 75 | mock_forget_metrics, 76 | mock_backup_metrics, 77 | ): 78 | scenarios: list[dict[str, Any]] = [ 79 | { 80 | "name": "all_metrics", 81 | "metrics": { 82 | "backup": {"backup_metrics": 1}, 83 | "forget": {"forget_metrics": 2}, 84 | "prune": {"prune_metrics": 3}, 85 | "check": {"check_metrics": 4}, 86 | "stats": {"stats_metrics": 5}, 87 | }, 88 | }, 89 | { 90 | "name": "backup_metrics", 91 | "metrics": { 92 | "backup": {"backup_metrics": 1}, 93 | }, 94 | }, 95 | { 96 | "name": "no_metrics", 97 | "metrics": {}, 98 | }, 99 | ] 100 | genral_metrics = { 101 | "errors": 10, 102 | "last_run": 11, 103 | "total_duration_seconds": 12, 104 | } 105 | prometheus._restic_help_general = "restic_help_general" 106 | prometheus._restic_general = ( 107 | "restic_general:{name}:{last_run}:{errors}:{total_duration_seconds}" 108 | ) 109 | for sc in scenarios: 110 | with self.subTest(sc["name"]): 111 | expected_lines = [ 112 | "restic_help_general", 113 | f"restic_general:{sc['name']}:11:10:12", 114 | ] + [f"{sc['name']}: {value}" for value in sc["metrics"].values()] 115 | metrics = sc["metrics"] | genral_metrics 116 | lines = prometheus.generate_lines(metrics, sc["name"]) 117 | self.assertEqual(list(lines), expected_lines) 118 | 119 | def test_backup_metrics(self): 120 | scenarios: list[dict[str, Any]] = [ 121 | { 122 | "name": "pre_and_post_hooks", 123 | "metrics": { 124 | "_restic_pre_hooks": {"duration_seconds": 2, "rc": 0}, 125 | "_restic_post_hooks": {"duration_seconds": 4, "rc": 0}, 126 | "repo1": { 127 | "files": {"new": "1", "changed": "2", "unmodified": "3"}, 128 | "dirs": {"new": "1", "changed": "2", "unmodified": "3"}, 129 | "processed": { 130 | "files": "1", 131 | "size_bytes": 2, 132 | "duration_seconds": 3, 133 | }, 134 | "added_to_repo": 7, 135 | "duration_seconds": 9, 136 | "rc": 0, 137 | }, 138 | "repo2": { 139 | "files": {"new": "1", "changed": "2", "unmodified": "3"}, 140 | "dirs": {"new": "1", "changed": "2", "unmodified": "3"}, 141 | "processed": { 142 | "files": "1", 143 | "size_bytes": 2, 144 | "duration_seconds": 3, 145 | }, 146 | "added_to_repo": 5, 147 | "duration_seconds": 8, 148 | "rc": 1, 149 | }, 150 | }, 151 | "expected_lines": [ 152 | "restic_help_backup", 153 | "restic_help_pre_hooks", 154 | "restic_help_post_hooks", 155 | "pre:my_backup:0:2", 156 | "post:my_backup:0:4", 157 | "restic_backup_data:my_backup:7:9", 158 | 'restic_backup_rc{config="my_backup",repository="repo2"} 1\n', 159 | ], 160 | }, 161 | { 162 | "name": "without_hooks", 163 | "metrics": { 164 | "repo1": { 165 | "files": {"new": "1", "changed": "2", "unmodified": "3"}, 166 | "dirs": {"new": "1", "changed": "2", "unmodified": "3"}, 167 | "processed": { 168 | "files": "1", 169 | "size_bytes": 2, 170 | "duration_seconds": 3, 171 | }, 172 | "added_to_repo": 7, 173 | "duration_seconds": 9, 174 | "rc": 0, 175 | }, 176 | "repo2": { 177 | "files": {"new": "1", "changed": "2", "unmodified": "3"}, 178 | "dirs": {"new": "1", "changed": "2", "unmodified": "3"}, 179 | "processed": { 180 | "files": "1", 181 | "size_bytes": 2, 182 | "duration_seconds": 3, 183 | }, 184 | "added_to_repo": 5, 185 | "duration_seconds": 8, 186 | "rc": 1, 187 | }, 188 | }, 189 | "expected_lines": [ 190 | "restic_help_backup", 191 | "restic_backup_data:my_backup:7:9", 192 | 'restic_backup_rc{config="my_backup",repository="repo2"} 1\n', 193 | ], 194 | }, 195 | ] 196 | # check that backup_metrics can be called with sample metrics 197 | # this validates that the `_restic_backup` template matches the data 198 | _lines = prometheus.backup_metrics(scenarios[0]["metrics"], "my_backup") 199 | # check call with simplified output 200 | prometheus._restic_help_backup = "restic_help_backup|" 201 | prometheus._restic_backup = ( 202 | "restic_backup_data:{name}:{added_to_repo}:{duration_seconds}|" 203 | ) 204 | for sc in scenarios: 205 | with self.subTest(sc["name"]): 206 | lines = prometheus.backup_metrics(sc["metrics"], "my_backup") 207 | self.assertEqual( 208 | lines, 209 | "|".join(sc["expected_lines"]), 210 | ) 211 | 212 | def test_forget_metrics(self): 213 | metrics = { 214 | "repo1": { 215 | "removed_snapshots": "7", 216 | "duration_seconds": 9, 217 | "rc": 0, 218 | }, 219 | "repo2": { 220 | "removed_snapshots": "2", 221 | "duration_seconds": 4.4, 222 | "rc": 1, 223 | }, 224 | } 225 | # check that forget_metrics can be called with sample metrics 226 | _lines = prometheus.forget_metrics(metrics, "my_forget") 227 | # check call with simplified output 228 | prometheus._restic_help_forget = "restic_help_forget|" 229 | prometheus._restic_forget = ( 230 | "restic_forget_data:{name}:{removed_snapshots}:{duration_seconds}|" 231 | ) 232 | lines = prometheus.forget_metrics(metrics, "my_forget") 233 | self.assertEqual( 234 | lines, 235 | "|".join( 236 | [ 237 | "restic_help_forget", 238 | "restic_forget_data:my_forget:7:9", 239 | 'restic_forget_rc{config="my_forget",repository="repo2"} 1\n', 240 | ] 241 | ), 242 | ) 243 | 244 | def test_new_prune_metrics(self): 245 | metrics = { 246 | "/tmp/restic-repo1": { # noqa: S108 247 | "containing_packs_before": "576", 248 | "containing_blobs": "95060", 249 | "containing_size_bytes": 2764885196.8, 250 | "duplicate_blobs": "0", 251 | "duplicate_size_bytes": 0.0, 252 | "in_use_blobs": "95055", 253 | "removed_blobs": "5", 254 | "invalid_files": "0", 255 | "deleted_packs": "2", 256 | "rewritten_packs": "0", 257 | "size_freed_bytes": 16679.936, 258 | "removed_index_files": "2", 259 | "duration_seconds": 4.2, 260 | "rc": 0, 261 | }, 262 | # data block with old prune metrics 263 | "/tmp/restic-repo2": { # noqa: S108 264 | "to_repack_blobs": "864", 265 | # "containing_blobs": "95060", 266 | "to_repack_bytes": 2764885196.8, 267 | "removed_blobs": "11", 268 | "removed_bytes": 42.0, 269 | "to_delete_blobs": "96358", 270 | "to_delete_bytes": 5249.936, 271 | "total_prune_blobs": "5", 272 | "total_prune_bytes": 85176.225, 273 | "remaining_blobs": "2", 274 | "remaining_bytes": 52.244, 275 | "remaining_unused_size": 16679.936, 276 | "duration_seconds": 7.3, 277 | "rc": 0, 278 | }, 279 | # data block with new prune metrics and rc > 0 280 | "/tmp/restic-repo3": { # noqa: S108 281 | "containing_packs_before": "575", 282 | "containing_blobs": "95052", 283 | "containing_size_bytes": 2765958938.624, 284 | "duplicate_blobs": "0", 285 | "duplicate_size_bytes": 0.0, 286 | "in_use_blobs": "95047", 287 | "removed_blobs": "5", 288 | "invalid_files": "0", 289 | "deleted_packs": "2", 290 | "rewritten_packs": "0", 291 | "size_freed_bytes": 16613.376, 292 | "removed_index_files": "2", 293 | "duration_seconds": 4.281890153884888, 294 | "rc": 1, 295 | }, 296 | } 297 | # check that prune_metrics can be called with sample metrics 298 | _lines = prometheus.prune_metrics(metrics, "my_prune") 299 | # check call with simplified output 300 | prometheus._restic_help_prune = "restic_help_prune|" 301 | prometheus._restic_prune = ( 302 | "restic_prune_data:{name}:{containing_packs_before}:{duration_seconds}|" 303 | ) 304 | prometheus._restic_new_prune = ( 305 | "restic_prune_data:{name}:{to_repack_blobs}:{duration_seconds}|" 306 | ) 307 | lines = prometheus.prune_metrics(metrics, "my_prune") 308 | self.assertEqual( 309 | lines, 310 | "|".join( 311 | [ 312 | "restic_help_prune", 313 | "restic_prune_data:my_prune:576:4.2", 314 | "restic_prune_data:my_prune:864:7.3", 315 | 'restic_prune_rc{config="my_prune",repository="/tmp/restic-repo3"} 1\n', 316 | ] 317 | ), 318 | ) 319 | 320 | def test_check_metrics(self): 321 | metrics = { 322 | "/tmp/restic-repo1": { # noqa: S108 323 | "errors": 0, 324 | "errors_data": 0, 325 | "errors_snapshots": 7, 326 | "read_data": 1, 327 | "check_unused": 1, 328 | "duration_seconds": 9, 329 | "rc": 0, 330 | }, 331 | "/tmp/restic-repo2": { # noqa: S108 332 | "errors": 0, 333 | "errors_data": 0, 334 | "errors_snapshots": 0, 335 | "read_data": 1, 336 | "check_unused": 1, 337 | "duration_seconds": 28.380418062210083, 338 | "rc": 1, 339 | }, 340 | } 341 | # check that check_metrics can be called with sample metrics 342 | _lines = prometheus.check_metrics(metrics, "my_check") 343 | # check call with simplified output 344 | prometheus._restic_help_check = "restic_help_check|" 345 | prometheus._restic_check = ( 346 | "restic_check_data:{name}:{errors_snapshots}:{duration_seconds}|" 347 | ) 348 | lines = prometheus.check_metrics(metrics, "my_check") 349 | self.assertEqual( 350 | lines, 351 | "|".join( 352 | [ 353 | "restic_help_check", 354 | "restic_check_data:my_check:7:9", 355 | 'restic_check_rc{config="my_check",repository="/tmp/restic-repo2"} 1\n', 356 | ] 357 | ), 358 | ) 359 | 360 | def test_stats_metrics(self): 361 | metrics = { 362 | "/tmp/restic-repo1": { # noqa: S108 363 | "total_file_count": 7, 364 | "total_size_bytes": 18148185424, 365 | "duration_seconds": 9, 366 | "rc": 0, 367 | }, 368 | "/tmp/restic-repo2": { # noqa: S108 369 | "total_file_count": 885276, 370 | "total_size_bytes": 18148185424, 371 | "duration_seconds": 20.466784715652466, 372 | "rc": 1, 373 | }, 374 | } 375 | # stats that stats_metrics can be called with sample metrics 376 | _lines = prometheus.stats_metrics(metrics, "my_stats") 377 | # stats call with simplified output 378 | prometheus._restic_help_stats = "restic_help_stats|" 379 | prometheus._restic_stats = ( 380 | "restic_stats_data:{name}:{total_file_count}:{duration_seconds}|" 381 | ) 382 | lines = prometheus.stats_metrics(metrics, "my_stats") 383 | self.assertEqual( 384 | lines, 385 | "|".join( 386 | [ 387 | "restic_help_stats", 388 | "restic_stats_data:my_stats:7:9", 389 | 'restic_stats_rc{config="my_stats",repository="/tmp/restic-repo2"} 1\n', 390 | ] 391 | ), 392 | ) 393 | -------------------------------------------------------------------------------- /tests/restic/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sinnwerkstatt/runrestic/4ff8868628e5ca4633ac666e486add0b3a8e702f/tests/restic/__init__.py -------------------------------------------------------------------------------- /tests/restic/test_installer.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from unittest.mock import patch, mock_open 3 | 4 | from runrestic.restic import installer 5 | 6 | 7 | class TestInstaller(TestCase): 8 | def test_restic_check_is_installed(self): 9 | # Mock the `which` function to simulate restic being installed 10 | with patch( 11 | "runrestic.restic.installer.which", return_value="/usr/local/bin/restic" 12 | ): 13 | self.assertTrue(installer.restic_check()) 14 | 15 | def test_restic_check_do_install(self): 16 | # Mock the `input` function to simulate user input for installation 17 | with ( 18 | patch("runrestic.restic.installer.which", return_value=None), 19 | patch("runrestic.restic.installer.download_restic"), 20 | ): 21 | with patch("builtins.input", return_value="y"): 22 | self.assertTrue(installer.restic_check()) 23 | with patch("builtins.input", return_value="n"): 24 | self.assertFalse(installer.restic_check()) 25 | 26 | def test_download_restic(self): 27 | # Mock the requests.get method to simulate a successful response 28 | with patch("runrestic.restic.installer.requests.get") as mock_get: 29 | mock_get.return_value.status_code = 200 30 | mock_get.return_value.content = b'{"assets": [{"name": "restic_linux_amd64.bz2", "browser_download_url": "https://example.com/restic_linux_amd64.bz2"}]}' 31 | with ( 32 | patch( 33 | "runrestic.restic.installer.bz2.decompress", 34 | return_value=b"dummy_program", 35 | ), 36 | patch("runrestic.restic.installer.os.chmod") as mock_chmod, 37 | patch("builtins.open", new_callable=mock_open) as mock_file_open, 38 | ): 39 | installer.download_restic() 40 | mock_get.assert_any_call( 41 | "https://api.github.com/repos/restic/restic/releases/latest", 42 | ) 43 | mock_get.assert_called_with( 44 | "https://example.com/restic_linux_amd64.bz2", 45 | allow_redirects=True, 46 | ) 47 | # Assert open() was called with the right path and mode 48 | mock_file_open.assert_called_once_with("/usr/local/bin/restic", "wb") 49 | mock_chmod.assert_called_once_with("/usr/local/bin/restic", 0o755) 50 | 51 | def test_download_restic_permission_error(self): 52 | # Mock the requests.get method to simulate a successful response 53 | with ( 54 | patch("runrestic.restic.installer.requests.get") as mock_get, 55 | patch( 56 | "runrestic.restic.installer.bz2.decompress", 57 | return_value=b"dummy_program", 58 | ), 59 | ): 60 | mock_get.return_value.status_code = 200 61 | mock_get.return_value.content = b'{"assets": [{"name": "restic_linux_amd64.bz2", "browser_download_url": "https://example.com/restic_linux_amd64.bz2"}]}' 62 | with ( 63 | patch("builtins.print") as mock_print, 64 | patch("builtins.open", new_callable=mock_open) as mock_file_open, 65 | patch("builtins.input", return_value=""), 66 | ): 67 | file_handle = mock_file_open() 68 | file_handle.write.side_effect = [ 69 | PermissionError, 70 | None, 71 | ] 72 | installer.download_restic() 73 | mock_print.assert_any_call("\nTry re-running this as root.") 74 | file_handle.write.assert_called_once() 75 | 76 | def test_download_restic_permission_error_alt(self): 77 | # Mock the requests.get method to simulate a successful response 78 | with ( 79 | patch("runrestic.restic.installer.requests.get") as mock_get, 80 | patch( 81 | "runrestic.restic.installer.bz2.decompress", 82 | return_value=b"dummy_program", 83 | ), 84 | ): 85 | mock_get.return_value.status_code = 200 86 | mock_get.return_value.content = b'{"assets": [{"name": "restic_linux_amd64.bz2", "browser_download_url": "https://example.com/restic_linux_amd64.bz2"}]}' 87 | with ( 88 | patch("builtins.print") as mock_print, 89 | patch("builtins.open", new_callable=mock_open) as mock_file_open, 90 | patch( 91 | "runrestic.restic.installer.os.chmod", 92 | ) as mock_chmod, 93 | patch("builtins.input", return_value="alt_path"), 94 | ): 95 | file_handle = mock_file_open() 96 | file_handle.write.side_effect = [ 97 | PermissionError, 98 | None, 99 | ] 100 | installer.download_restic() 101 | mock_print.assert_any_call("\nTry re-running this as root.") 102 | self.assertEqual(file_handle.write.call_count, 2) 103 | mock_chmod.assert_called_once_with("alt_path", 0o755) 104 | 105 | def test_download_restic_no_assets(self): 106 | # Mock the requests.get method to simulate a successful response 107 | with patch("runrestic.restic.installer.requests.get") as mock_get: 108 | mock_get.return_value.status_code = 200 109 | mock_get.return_value.content = b'{"dummy": 42}' 110 | self.assertRaises( 111 | KeyError, 112 | installer.download_restic, 113 | ) 114 | 115 | def test_download_restic_assets_no_match(self): 116 | # Mock the requests.get method to simulate a successful response 117 | with patch("runrestic.restic.installer.requests.get") as mock_get: 118 | mock_get.return_value.status_code = 200 119 | mock_get.return_value.content = b'{"assets": [{"name": "restic_fake_os.bz2", "browser_download_url": "https://example.com/restic_fake_os.bz2"}]}' 120 | mock_get.side_effect = [ 121 | # Simulate successful response for fetching release 122 | type( 123 | "Response", 124 | (object,), 125 | { 126 | "status_code": 200, 127 | "content": b'{"assets": [{"name": "restic_fake_os.bz2", "browser_download_url": "https://example.com/restic_fake_os.bz2"}]}', 128 | "raise_for_status": lambda x: True, 129 | }, 130 | )(), 131 | # Simulate exception during program download 132 | installer.requests.exceptions.MissingSchema("Invalid URL ''"), 133 | ] 134 | self.assertRaises( 135 | installer.requests.exceptions.MissingSchema, installer.download_restic 136 | ) 137 | 138 | def test_download_restic_request_exception_fetch_release(self): 139 | # Mock the requests.get method to simulate a request exception 140 | with ( 141 | patch( 142 | "runrestic.restic.installer.requests.get", 143 | side_effect=installer.requests.exceptions.RequestException( 144 | "Request failed" 145 | ), 146 | ), 147 | ): 148 | self.assertRaises( 149 | installer.requests.exceptions.RequestException, 150 | installer.download_restic, 151 | ) 152 | 153 | def test_download_restic_request_exception_download_program(self): 154 | # Mock the requests.get method to simulate a request exception 155 | with ( 156 | patch( 157 | "runrestic.restic.installer.requests.get", 158 | side_effect=[ 159 | # Simulate successful response for fetching release 160 | type( 161 | "Response", 162 | (object,), 163 | { 164 | "status_code": 200, 165 | "content": b'{"assets": [{"name": "restic_linux_amd64.bz2", "browser_download_url": "https://example.com/restic_linux_amd64.bz2"}]}', 166 | "raise_for_status": lambda x: True, 167 | }, 168 | )(), 169 | # Simulate exception during program download 170 | installer.requests.exceptions.RequestException("Request failed"), 171 | ], 172 | ) as mock_get, 173 | ): 174 | self.assertRaises( 175 | installer.requests.exceptions.RequestException, 176 | installer.download_restic, 177 | ) 178 | self.assertEqual(mock_get.call_count, 2) 179 | -------------------------------------------------------------------------------- /tests/restic/test_output_parsing.py: -------------------------------------------------------------------------------- 1 | """Test the restic output parsing""" 2 | 3 | from textwrap import dedent 4 | 5 | from runrestic.restic import output_parsing 6 | 7 | 8 | def test_parse_backup(): 9 | """Validate that all backup details are correctly captured""" 10 | output = dedent( 11 | """\ 12 | repository c2e84608 opened successfully, password is correct 13 | created new cache in /home/user/.cache/restic 14 | found 2 old cache directories in /home/user/.cache/restic, run `restic cache --cleanup` to remove them 15 | no parent snapshot found, will read all files 16 | 17 | Files: 22391 new, 42 changed, 5 unmodified 18 | Dirs: 2927 new, 1 changed, 3 unmodified 19 | Added to the repo: 259.569 MiB 20 | 21 | processed 22438 files, 302.750 MiB in 1:12 22 | snapshot 215cf0fa saved 23 | """ 24 | ) 25 | data = { 26 | "files": { 27 | "new": "22391", 28 | "changed": "42", 29 | "unmodified": "5", 30 | }, 31 | "dirs": { 32 | "new": "2927", 33 | "changed": "1", 34 | "unmodified": "3", 35 | }, 36 | "processed": { 37 | "files": "22438", 38 | "size_bytes": 302.750 * 2**20, 39 | "duration_seconds": 72, 40 | }, 41 | "added_to_repo": 259.569 * 2**20, 42 | "duration_seconds": 35.8, 43 | "rc": 0, 44 | } 45 | process_infos = {"output": [(0, output)], "time": data["duration_seconds"]} 46 | result = output_parsing.parse_backup(process_infos) 47 | assert result == data 48 | 49 | 50 | def test_parse_backup_defaults(): 51 | """Validate that all backup parsing uses defaults in case of unexpected formatting""" 52 | output = "UNEXPECTED OUTPUT" 53 | data = { 54 | "files": { 55 | "new": "0", 56 | "changed": "0", 57 | "unmodified": "0", 58 | }, 59 | "dirs": { 60 | "new": "0", 61 | "changed": "0", 62 | "unmodified": "0", 63 | }, 64 | "processed": { 65 | "files": "0", 66 | "size_bytes": 0, 67 | "duration_seconds": 0, 68 | }, 69 | "added_to_repo": 0, 70 | "duration_seconds": 123, 71 | "rc": 0, 72 | } 73 | process_infos = {"output": [(0, output)], "time": data["duration_seconds"]} 74 | result = output_parsing.parse_backup(process_infos) 75 | assert result == data 76 | 77 | 78 | def test_parse_forget(): 79 | """Validate that all forget details are correctly captured""" 80 | output = dedent( 81 | """\ 82 | repository c2e84608 opened successfully, password is correct 83 | found 2 old cache directories in /home/user/.cache/restic, run `restic cache --cleanup` to remove them 84 | Applying Policy: keep 2 latest snapshots 85 | keep 2 snapshots: 86 | ID Time Host Tags Reasons Paths 87 | -------------------------------------------------------------------------------------------------- 88 | 04ffe2e5 2022-05-27 15:01:41 cubitus last snapshot /home/user/git/runrestic 89 | 611527ee 2022-05-27 15:01:53 cubitus last snapshot /home/user/git/runrestic 90 | -------------------------------------------------------------------------------------------------- 91 | 2 snapshots 92 | 93 | remove 1 snapshots: 94 | ID Time Host Tags Paths 95 | ----------------------------------------------------------------------------------- 96 | 215cf0fa 2022-05-27 14:42:40 cubitus /home/user/git/runrestic 97 | ----------------------------------------------------------------------------------- 98 | 1 snapshots 99 | 100 | [0:00] 100.00% 1 / 1 files deleted 101 | """ 102 | ) 103 | data = { 104 | "removed_snapshots": "1", 105 | "duration_seconds": 12.7, 106 | "rc": 0, 107 | } 108 | process_infos = {"output": [(0, output)], "time": data["duration_seconds"]} 109 | result = output_parsing.parse_forget(process_infos) 110 | assert result == data 111 | 112 | 113 | def test_parse_forget_defaults(): 114 | """Validate that all forget details uses defaults in case of unexpected formatting""" 115 | output = "UNEXPECTED OUTPUT" 116 | data = { 117 | "removed_snapshots": "0", 118 | "duration_seconds": 123, 119 | "rc": 0, 120 | } 121 | process_infos = {"output": [(0, output)], "time": data["duration_seconds"]} 122 | result = output_parsing.parse_forget(process_infos) 123 | assert result == data 124 | 125 | 126 | def test_parse_prune(): 127 | """Validate that all prune details are correctly captured (version < 12.0)""" 128 | output = dedent( 129 | """\ 130 | password is correct 131 | storage ID 9babef79 132 | counting files in repo 133 | building new index for repo 134 | [2:16] 100.00% 11981 / 11981 packs 135 | repository contains 11981 packs (345057 blobs) with 56.676 GiB 136 | processed 345057 blobs: 0 duplicate blobs, 0B duplicate 137 | load all snapshots 138 | find data that is still in use for 1 snapshots 139 | [0:00] 100.00% 1 / 1 snapshots 140 | found 2 of 345057 data blobs still in use, removing 345055 blobs 141 | will remove 0 invalid files 142 | will delete 11979 packs and rewrite 0 packs, this frees 56.664 GiB 143 | counting files in repo 144 | [0:00] 100.00% 2 / 2 packs 145 | finding old index files 146 | saved new indexes as [70561784] 147 | remove 11 old index files 148 | [1:12] 100.00% 11979 / 11979 packs deleted 149 | done 150 | """ 151 | ) 152 | data = { 153 | "containing_packs_before": "11981", 154 | "containing_blobs": "345057", 155 | "containing_size_bytes": 56.676 * 2**30, 156 | "duplicate_blobs": "0", 157 | "duplicate_size_bytes": 0.0, 158 | "in_use_blobs": "2", 159 | "removed_blobs": "345055", 160 | "invalid_files": "0", 161 | "deleted_packs": "11979", 162 | "rewritten_packs": "0", 163 | "size_freed_bytes": 56.664 * 2**30, 164 | "removed_index_files": "11", 165 | "duration_seconds": 3.27, 166 | "rc": 0, 167 | } 168 | process_infos = {"output": [(0, output)], "time": data["duration_seconds"]} 169 | result = output_parsing.parse_prune(process_infos) 170 | assert result == data 171 | 172 | 173 | def test_parse_prune_defaults(): 174 | """Validate that all prune details uses defaults in case of unexpected formatting (version < 12.0)""" 175 | output = "UNEXPECTED OUTPUT" 176 | data = { 177 | "containing_packs_before": "0", 178 | "containing_blobs": "0", 179 | "containing_size_bytes": 0, 180 | "duplicate_blobs": "0", 181 | "duplicate_size_bytes": 0, 182 | "in_use_blobs": "0", 183 | "removed_blobs": "0", 184 | "invalid_files": "0", 185 | "deleted_packs": "0", 186 | "rewritten_packs": "0", 187 | "size_freed_bytes": 0, 188 | "removed_index_files": "0", 189 | "duration_seconds": 123, 190 | "rc": 0, 191 | } 192 | process_infos = {"output": [(0, output)], "time": data["duration_seconds"]} 193 | result = output_parsing.parse_prune(process_infos) 194 | assert result == data 195 | 196 | 197 | def test_parse_new_prune(): 198 | """Validate that all prune details are correctly captured (version >= 12.0)""" 199 | output = dedent( 200 | """\ 201 | repository c2e84608 opened successfully, password is correct 202 | found 2 old cache directories in /home/user/.cache/restic, run `restic cache --cleanup` to remove them 203 | loading indexes... 204 | loading all snapshots... 205 | finding data that is still in use for 1 snapshots 206 | [0:00] 100.00% 1 / 1 snapshots 207 | searching used packs... 208 | collecting packs for deletion and repacking 209 | [0:00] 100.00% 65 / 65 packs processed 210 | 211 | to repack: 1 blobs / 1234 B 212 | this removes: 2 blobs / 5678 B 213 | to delete: 32 blobs / 158.830 KiB 214 | total prune: 35 blobs / 158.830 KiB 215 | remaining: 19154 blobs / 260.161 MiB 216 | unused size after prune: 0 B (0.00% of remaining size) 217 | 218 | rebuilding index 219 | [0:00] 100.00% 62 / 62 packs processed 220 | deleting obsolete index files 221 | [0:00] 100.00% 2 / 2 files deleted 222 | removing 3 old packs 223 | [0:00] 100.00% 3 / 3 files deleted 224 | done 225 | """ 226 | ) 227 | data = { 228 | "to_repack_blobs": "1", 229 | "to_repack_bytes": 1234.0, 230 | "removed_blobs": "2", 231 | "removed_bytes": 5678.0, 232 | "to_delete_blobs": "32", 233 | "to_delete_bytes": 158.830 * 2**10, 234 | "total_prune_blobs": "35", 235 | "total_prune_bytes": 158.830 * 2**10, 236 | "remaining_blobs": "19154", 237 | "remaining_bytes": 260.161 * 2**20, 238 | "remaining_unused_size": 0.0, 239 | "duration_seconds": 8.47, 240 | "rc": 0, 241 | } 242 | process_infos = {"output": [(0, output)], "time": data["duration_seconds"]} 243 | result = output_parsing.parse_new_prune(process_infos) 244 | assert result == data 245 | 246 | 247 | def test_parse_new_prune_defaults(): 248 | """Validate that all prune details uses defaults in case of unexpected formatting (version >= 12.0)""" 249 | output = "UNEXPECTED OUTPUT" 250 | data = { 251 | "to_repack_blobs": "0", 252 | "to_repack_bytes": 0, 253 | "removed_blobs": "0", 254 | "removed_bytes": 0, 255 | "to_delete_blobs": "0", 256 | "to_delete_bytes": 0, 257 | "total_prune_blobs": "0", 258 | "total_prune_bytes": 0, 259 | "remaining_blobs": "0", 260 | "remaining_bytes": 0, 261 | "remaining_unused_size": 0, 262 | "duration_seconds": 123, 263 | "rc": 0, 264 | } 265 | process_infos = {"output": [(0, output)], "time": data["duration_seconds"]} 266 | result = output_parsing.parse_new_prune(process_infos) 267 | assert result == data 268 | 269 | 270 | def test_parse_stats_quiet(): 271 | """Validate that all stats are correctly captured""" 272 | output = '{"total_size":317458353,"total_file_count":50653}' 273 | data = { 274 | "total_file_count": 50653, 275 | "total_size_bytes": 317458353, 276 | "duration_seconds": 1.57, 277 | "rc": 0, 278 | } 279 | process_infos = {"output": [(0, output)], "time": data["duration_seconds"]} 280 | result = output_parsing.parse_stats(process_infos) 281 | assert result == data 282 | 283 | 284 | def test_parse_stats_verbose(): 285 | """Validate that all stats are correctly captured""" 286 | output = dedent( 287 | """\ 288 | found 2 old cache directories in /home/user/.cache/restic, run `restic cache --cleanup` to remove them 289 | {"total_size":317458353,"total_file_count":50653} 290 | """ 291 | ) 292 | data = { 293 | "total_file_count": 50653, 294 | "total_size_bytes": 317458353, 295 | "duration_seconds": 1.57, 296 | "rc": 0, 297 | } 298 | process_infos = {"output": [(0, output)], "time": data["duration_seconds"]} 299 | result = output_parsing.parse_stats(process_infos) 300 | assert result == data 301 | 302 | 303 | def test_parse_stats_quiet_defaults(): 304 | """Validate that all stats uses defaults in case of unexpected formatting""" 305 | output = '{"UNEXPECTED": "OUTPUT"}' 306 | data = { 307 | "total_file_count": 0, 308 | "total_size_bytes": 0, 309 | "duration_seconds": 0, 310 | "rc": 0, 311 | } 312 | process_infos = {"output": [(0, output)], "time": data["duration_seconds"]} 313 | result = output_parsing.parse_stats(process_infos) 314 | assert result == data 315 | -------------------------------------------------------------------------------- /tests/restic/test_restic_tools.py: -------------------------------------------------------------------------------- 1 | import io 2 | import logging 3 | import os 4 | import time 5 | from time import sleep 6 | from typing import Any, Dict, List, Optional, Union 7 | from unittest.mock import MagicMock, call, patch 8 | 9 | import pytest 10 | 11 | from runrestic.restic.tools import ( 12 | MultiCommand, 13 | initialize_environment, 14 | redact_password, 15 | retry_process, 16 | ) 17 | 18 | 19 | def fake_retry_process( 20 | cmd: Union[str, List[str]], 21 | config: Dict[str, Any], 22 | abort_reasons: Optional[List[str]] = None, 23 | ) -> Dict[str, Any]: 24 | """Fake retry_process function to simulate command execution.""" 25 | # Simulate different outputs per command 26 | if isinstance(cmd, list): 27 | cmd = cmd[0] 28 | try: 29 | count = int(cmd[-1]) 30 | except ValueError: 31 | count = 1 32 | retry_count = config.get("retry_count", 0) 33 | sleep(count / 10) # Simulate some processing time 34 | return { 35 | "current_try": count, 36 | "tries_total": 3, 37 | "output": [(1, f"fail{i}") for i in range(1, count)] 38 | + ([(0, "pass")] if retry_count + 1 >= count else []), 39 | "time": count / 10, 40 | } 41 | 42 | 43 | def fake_process(returncode: int, stdout_text: str) -> MagicMock: 44 | """Helper to create a fake Popen-like process object.""" 45 | proc = MagicMock() 46 | proc.__enter__.return_value = proc 47 | proc.__exit__.return_value = None 48 | proc.stdout = io.StringIO(stdout_text) 49 | proc.returncode = returncode 50 | return proc 51 | 52 | 53 | @pytest.mark.parametrize( 54 | "popen_results, retry_count, expected_output, expected_current, expected_total", 55 | [ 56 | ( # Test immediate success with no no reties allowed 57 | [fake_process(0, "pass"), fake_process(0, "extra")], 58 | 0, 59 | [(0, "pass")], 60 | 1, 61 | 1, 62 | ), 63 | ( # Test immediate success with 1 retry allowed 64 | [fake_process(0, "pass"), fake_process(0, "extra")], 65 | 1, 66 | [(0, "pass")], 67 | 1, 68 | 2, 69 | ), 70 | ( # Test 1 failure with 1 retry allowed, finally passing 71 | [ 72 | fake_process(1, "fail1"), 73 | fake_process(0, "pass"), 74 | fake_process(0, "extra"), 75 | ], 76 | 1, 77 | [(1, "fail1"), (0, "pass")], 78 | 2, 79 | 2, 80 | ), 81 | ( # Test 2 failures with 1 retry allowed, finally failing 82 | [ 83 | fake_process(1, "fail1"), 84 | fake_process(1, "fail2"), 85 | fake_process(0, "extra"), 86 | ], 87 | 1, 88 | [(1, "fail1"), (1, "fail2")], 89 | 2, 90 | 2, 91 | ), 92 | ( # Test 4 failures and 5th success with 4 retries allowed, finally passing 93 | [ 94 | fake_process(1, "fail1"), 95 | fake_process(1, "fail2"), 96 | fake_process(1, "fail3"), 97 | fake_process(1, "fail4"), 98 | fake_process(0, "pass"), 99 | fake_process(0, "extra"), 100 | ], 101 | 4, 102 | [ 103 | (1, "fail1"), 104 | (1, "fail2"), 105 | (1, "fail3"), 106 | (1, "fail4"), 107 | (0, "pass"), 108 | ], 109 | 5, 110 | 5, 111 | ), 112 | ( # Test 3 failures with 2 retries allowed, finally failing 113 | [ 114 | fake_process(1, "fail1"), 115 | fake_process(1, "fail2"), 116 | fake_process(1, "fail3"), 117 | fake_process(0, "extra"), 118 | ], 119 | 2, 120 | [(1, "fail1"), (1, "fail2"), (1, "fail3")], 121 | 3, 122 | 3, 123 | ), 124 | ], 125 | ) 126 | @patch("runrestic.restic.tools.Popen") 127 | def test_retry_process( 128 | mock_popen: MagicMock, 129 | popen_results, 130 | retry_count, 131 | expected_output, 132 | expected_current, 133 | expected_total, 134 | ): 135 | # Arrange 136 | mock_popen.side_effect = popen_results 137 | 138 | # Act 139 | result = retry_process(["dummy_command"], {"retry_count": retry_count}) 140 | 141 | # Assert 142 | assert "time" in result, "Result should include execution time" 143 | assert result["output"] == expected_output 144 | assert result["current_try"] == expected_current 145 | assert result["tries_total"] == expected_total 146 | 147 | 148 | @pytest.mark.parametrize( 149 | "backoff, expected_sleep_args", 150 | [ 151 | ("0:01", [1, 1, 1]), 152 | ("0:01 linear", [1, 2, 3]), 153 | ("0:01 exponential", [1, 2, 4]), 154 | ], 155 | ) 156 | @patch("runrestic.restic.tools.time.sleep") 157 | @patch("runrestic.restic.tools.Popen") 158 | def test_retry_process_backoff( 159 | mock_popen: MagicMock, 160 | mock_sleep: MagicMock, 161 | backoff, 162 | expected_sleep_args, 163 | ): 164 | # Arrange 165 | mock_popen.side_effect = [fake_process(1, f"call {i + 1}/3") for i in range(3)] 166 | 167 | # Act 168 | p = retry_process( 169 | ["dummy_command"], 170 | {"retry_count": 2, "retry_backoff": backoff}, 171 | ) 172 | 173 | # Assert sleeps 174 | expected_calls = [call(arg) for arg in expected_sleep_args] 175 | mock_sleep.assert_has_calls(expected_calls) 176 | 177 | # Remove timing and output details for comparison 178 | p.pop("time") 179 | p.pop("output") 180 | assert p == {"current_try": 3, "tries_total": 3} 181 | 182 | 183 | @patch("runrestic.restic.tools.Popen") 184 | def test_retry_process_with_abort_reason(mock_popen: MagicMock): 185 | # Call the retry_process function with mocked Popen 186 | mock_popen.return_value = fake_process(99, "Abort reason: 1/10") 187 | p = retry_process( 188 | ["dummy_command"], 189 | {"retry_count": 99}, 190 | abort_reasons=[": 1/10"], 191 | ) 192 | # Validate the results 193 | assert p["current_try"] == 1 194 | assert p["tries_total"] == 100 195 | 196 | 197 | @patch("runrestic.restic.tools.retry_process", new=fake_retry_process) 198 | def test_run_multiple_commands_parallel() -> None: 199 | cmds = ["dummy_cmd3", "dummy_cmd2", "dummy_cmd1"] 200 | config = {"retry_count": 2, "parallel": True, "retry_backoff": "0:01"} 201 | start_time = time.time() 202 | aa = MultiCommand(cmds, config).run() 203 | assert 0.5 > time.time() - start_time > 0.3 204 | expected_return = [[1, 1, 0], [1, 0], [0]] 205 | 206 | for exp, cmd_ret in zip(expected_return, aa): 207 | assert [x[0] for x in cmd_ret["output"]] == exp 208 | 209 | 210 | @patch("runrestic.restic.tools.retry_process", new=fake_retry_process) 211 | def test_run_multiple_commands_serial() -> None: 212 | cmds = ["dummy_cmd3", "dummy_cmd3", "dummy_cmd4"] 213 | config = {"retry_count": 2, "parallel": False, "retry_backoff": "0:01"} 214 | start_time = time.time() 215 | aa = MultiCommand(cmds, config).run() 216 | assert 1.1 > float(time.time() - start_time) > 0.5 217 | expected_return = [[1, 1, 0], [1, 1, 0], [1, 1, 1]] 218 | 219 | for exp, cmd_ret in zip(expected_return, aa): 220 | assert [x[0] for x in cmd_ret["output"]] == exp 221 | 222 | 223 | def test_initialize_environment_pw_redact(caplog): 224 | env = {"RESTIC_PASSWORD": "my$ecr3T"} 225 | caplog.set_level(logging.DEBUG) 226 | initialize_environment(env) 227 | assert "RESTIC_PASSWORD=**********" in caplog.text 228 | assert "my$ecr3T" not in caplog.text 229 | 230 | 231 | def test_initialize_environment_no_home(monkeypatch): 232 | env = {"TEST123": "xyz"} 233 | monkeypatch.setenv("HOME", "") 234 | initialize_environment(env) 235 | assert os.environ.get("TEST123") == "xyz" 236 | assert os.environ.get("XDG_CACHE_HOME") == "/var/cache" 237 | 238 | 239 | def test_initialize_environment_user(monkeypatch): 240 | env = {"TEST456": "abc"} 241 | monkeypatch.setenv("HOME", "/home/user") 242 | monkeypatch.setenv("XDG_CACHE_HOME", "/home/user/.cache") 243 | initialize_environment(env) 244 | assert os.environ.get("TEST456") == "abc" 245 | assert os.environ.get("XDG_CACHE_HOME") == "/home/user/.cache" 246 | 247 | 248 | def test_initialize_environment_root(monkeypatch): 249 | env = {"TEST789": "qpr"} 250 | monkeypatch.setenv("HOME", "/root") 251 | monkeypatch.setenv("XDG_CACHE_HOME", "/root/.cache") 252 | monkeypatch.setattr(os, "geteuid", lambda: 0) # fake root 253 | initialize_environment(env) 254 | assert os.environ.get("TEST789") == "qpr" 255 | assert os.environ.get("XDG_CACHE_HOME") == "/var/cache" 256 | 257 | 258 | def test_redact_password(): 259 | password = "my$ecr3T" # noqa: S105 260 | repo_strings = [ 261 | "ftp://user:{}@server.com", 262 | "rest:http://user:{}@test1.something.org", 263 | "rest:https://user:{}@a-123.what.us", 264 | "s3:http://user:{}@lost.data.net", 265 | "s3:https://user:{}@island.in.the.sun.co.uk", 266 | ] 267 | pw_replacement = "******" 268 | for repo_str in repo_strings: 269 | assert redact_password( 270 | repo_str.format(password), pw_replacement 271 | ) == repo_str.format(pw_replacement) 272 | -------------------------------------------------------------------------------- /tests/restic/test_runner.py: -------------------------------------------------------------------------------- 1 | from argparse import Namespace 2 | from typing import Any 3 | from unittest import TestCase 4 | from unittest.mock import patch 5 | 6 | from runrestic.restic import runner 7 | 8 | 9 | class TestResticRunner(TestCase): 10 | @patch("runrestic.restic.runner.initialize_environment") 11 | def test_runner_class_init(self, mock_init_env): 12 | """ 13 | Test the initialization of the Runner class. 14 | """ 15 | config = { 16 | "dummy": "config", 17 | "repositories": ["dummy-repo"], 18 | "environment": {"dummy_env": "dummy_value"}, 19 | "metrics": {"prometheus": {"password_replacement": "dummy_pw"}}, 20 | } 21 | args = Namespace() 22 | args.dry_run = False 23 | restic_args = ["dummy-arg"] 24 | 25 | runner_instance = runner.ResticRunner(config, args, restic_args) 26 | self.assertEqual(runner_instance.args, args) 27 | self.assertEqual(runner_instance.restic_args, restic_args) 28 | self.assertTrue(runner_instance.log_metrics) 29 | self.assertEqual(runner_instance.pw_replacement, "dummy_pw") 30 | 31 | @patch.object(runner.ResticRunner, "init") 32 | @patch.object(runner.ResticRunner, "backup") 33 | @patch.object(runner.ResticRunner, "forget") 34 | @patch.object(runner.ResticRunner, "prune") 35 | @patch.object(runner.ResticRunner, "check") 36 | @patch.object(runner.ResticRunner, "stats") 37 | @patch.object(runner.ResticRunner, "unlock") 38 | @patch("runrestic.restic.runner.write_metrics") 39 | def test_run_dispatcher( 40 | self, 41 | mock_write_metrics, 42 | mock_unlock, 43 | mock_stats, 44 | mock_check, 45 | mock_prune, 46 | mock_forget, 47 | mock_backup, 48 | mock_init, 49 | ): 50 | """ 51 | Dispatch scenarios for run(): 52 | - all actions explicitly named 53 | - unknown action only 54 | - default actions with metrics enabled 55 | - default actions with metrics disabled 56 | """ 57 | scenarios: list[dict[str, Any]] = [ 58 | { 59 | "name": "all_actions", 60 | "config": { 61 | "name": "test", 62 | "repositories": ["repo"], 63 | "environment": {}, 64 | "execution": {}, 65 | "metrics": {"prometheus": {}}, 66 | }, 67 | "actions": ["init", "backup", "prune", "check", "stats", "unlock"], 68 | "initial_errors": 2, 69 | "expected_calls": { 70 | "init": 1, 71 | "backup": 1, 72 | "forget": 1, 73 | "prune": 1, 74 | "check": 1, 75 | "stats": 1, 76 | "unlock": 1, 77 | }, 78 | "write_metrics": True, 79 | "expected_errors": 2, 80 | }, 81 | { 82 | "name": "unknown_only", 83 | "config": { 84 | "name": "test", 85 | "repositories": ["repo"], 86 | "environment": {}, 87 | "execution": {}, 88 | # no "metrics" key → log_metrics=False 89 | }, 90 | "actions": ["unknown"], 91 | "initial_errors": 5, 92 | "expected_calls": { 93 | "init": 0, 94 | "backup": 0, 95 | "forget": 0, 96 | "prune": 0, 97 | "check": 0, 98 | "stats": 0, 99 | "unlock": 0, 100 | }, 101 | "write_metrics": False, 102 | "expected_errors": 5, 103 | }, 104 | { 105 | "name": "default_with_stats", 106 | "config": { 107 | "name": "test", 108 | "repositories": ["repo"], 109 | "environment": {}, 110 | "execution": {}, 111 | "metrics": {"prometheus": {}}, 112 | }, 113 | "actions": [], # log_metrics=True → ["backup","prune","check","stats"] 114 | "initial_errors": 0, 115 | "expected_calls": { 116 | "init": 0, 117 | "backup": 1, 118 | "forget": 1, 119 | "prune": 1, 120 | "check": 1, 121 | "stats": 1, 122 | "unlock": 0, 123 | }, 124 | "write_metrics": True, 125 | "expected_errors": 0, 126 | }, 127 | { 128 | "name": "default_no_stats", 129 | "config": { 130 | "name": "test", 131 | "repositories": ["repo"], 132 | "environment": {}, 133 | "execution": {}, 134 | # no "metrics" → log_metrics=False → ["backup","prune","check"] 135 | }, 136 | "actions": [], 137 | "initial_errors": 0, 138 | "expected_calls": { 139 | "init": 0, 140 | "backup": 1, 141 | "forget": 1, 142 | "prune": 1, 143 | "check": 1, 144 | "stats": 0, 145 | "unlock": 0, 146 | }, 147 | "write_metrics": False, 148 | "expected_errors": 0, 149 | }, 150 | ] 151 | 152 | for sc in scenarios: 153 | with self.subTest(sc["name"]): 154 | args = Namespace(actions=sc["actions"]) 155 | args.dry_run = False 156 | runner_instance = runner.ResticRunner( 157 | sc["config"], args, restic_args=[] 158 | ) 159 | runner_instance.metrics["errors"] = sc["initial_errors"] 160 | 161 | result = runner_instance.run() 162 | self.assertEqual(result, sc["expected_errors"]) 163 | 164 | # verify each method was (or wasn't) called 165 | self.assertEqual(mock_init.call_count, sc["expected_calls"]["init"]) 166 | self.assertEqual(mock_backup.call_count, sc["expected_calls"]["backup"]) 167 | self.assertEqual(mock_forget.call_count, sc["expected_calls"]["forget"]) 168 | self.assertEqual(mock_prune.call_count, sc["expected_calls"]["prune"]) 169 | self.assertEqual(mock_check.call_count, sc["expected_calls"]["check"]) 170 | self.assertEqual(mock_stats.call_count, sc["expected_calls"]["stats"]) 171 | self.assertEqual(mock_unlock.call_count, sc["expected_calls"]["unlock"]) 172 | 173 | # verify write_metrics 174 | if sc["write_metrics"]: 175 | mock_write_metrics.assert_called_once_with( 176 | runner_instance.metrics, sc["config"] 177 | ) 178 | else: 179 | mock_write_metrics.assert_not_called() 180 | 181 | # reset mocks for next scenario 182 | for m in ( 183 | mock_init, 184 | mock_backup, 185 | mock_forget, 186 | mock_prune, 187 | mock_check, 188 | mock_stats, 189 | mock_unlock, 190 | mock_write_metrics, 191 | ): 192 | m.reset_mock() 193 | 194 | @patch("runrestic.restic.runner.MultiCommand") 195 | def test_init_runs_commands(self, mock_mc): 196 | """ 197 | Test init() method runs MultiCommand with the correct parameters and processes output. 198 | """ 199 | config = { 200 | "repositories": ["repo1", "repo2"], 201 | "environment": {}, 202 | "execution": {}, 203 | } 204 | args = Namespace(dry_run=False) 205 | restic_args: list[str] = [] 206 | runner_instance = runner.ResticRunner(config, args, restic_args) 207 | 208 | # Simulate one success and one failure in init 209 | mock_mc.return_value.run.return_value = [ 210 | {"output": [(0, "repo1 initialized")], "time": 0.1}, 211 | {"output": [(1, "repo2 already initialized")], "time": 0.2}, 212 | ] 213 | 214 | runner_instance.init() 215 | 216 | # Ensure MultiCommand was called with the correct command list 217 | expected_commands = [ 218 | ["restic", "-r", "repo1", "init"], 219 | ["restic", "-r", "repo2", "init"], 220 | ] 221 | mock_mc.assert_called_once() 222 | actual_call_args = mock_mc.call_args[0][0] 223 | self.assertEqual(actual_call_args, expected_commands) 224 | 225 | # Validate run() was invoked 226 | mock_mc.return_value.run.assert_called_once() 227 | 228 | @patch("runrestic.restic.runner.MultiCommand") 229 | @patch("runrestic.restic.runner.parse_backup") 230 | @patch( 231 | "runrestic.restic.runner.redact_password", side_effect=lambda repo, repl: repo 232 | ) 233 | def test_backup_metrics(self, mock_redact, mock_parse_backup, mock_mc): 234 | """ 235 | Test backup() handles success and failure correctly and updates metrics and errors. 236 | """ 237 | config = { 238 | "repositories": ["repo1", "repo2"], 239 | "environment": {}, 240 | "execution": {}, 241 | "backup": { 242 | "sources": ["/data"], 243 | "files_from": ["/data/files.txt"], 244 | "exclude_patterns": ["*.exclude"], 245 | "exclude_files": ["dummy_file"], 246 | "exclude_if_present": ["*.present"], 247 | }, 248 | "metrics": {}, 249 | } 250 | args = Namespace(dry_run=False) 251 | restic_args = ["--opt"] 252 | runner_instance = runner.ResticRunner(config, args, restic_args) 253 | process_success = {"output": [(0, "")], "time": 0.1} 254 | process_fail = {"output": [(1, "")], "time": 0.2} 255 | mock_mc.return_value.run.return_value = [process_success, process_fail] 256 | mock_parse_backup.return_value = {"parsed": True} 257 | 258 | runner_instance.backup() 259 | 260 | # validate MultiCommand instantiation 261 | expected_commands = [ 262 | [ 263 | "restic", 264 | "-r", 265 | "repo1", 266 | "backup", 267 | "--opt", 268 | "--files-from", 269 | "/data/files.txt", 270 | "--exclude", 271 | "*.exclude", 272 | "--exclude-file", 273 | "dummy_file", 274 | "--exclude-if-present", 275 | "*.present", 276 | "/data", 277 | ], 278 | [ 279 | "restic", 280 | "-r", 281 | "repo2", 282 | "backup", 283 | "--opt", 284 | "--files-from", 285 | "/data/files.txt", 286 | "--exclude", 287 | "*.exclude", 288 | "--exclude-file", 289 | "dummy_file", 290 | "--exclude-if-present", 291 | "*.present", 292 | "/data", 293 | ], 294 | ] 295 | expected_abort = [ 296 | "Fatal: unable to open config file", 297 | "Fatal: wrong password", 298 | ] 299 | mock_mc.assert_called_once_with( 300 | expected_commands, config["execution"], expected_abort 301 | ) 302 | mock_mc.return_value.run.assert_called_once() 303 | 304 | metrics = runner_instance.metrics["backup"] 305 | self.assertEqual(metrics["repo1"], {"parsed": True}) 306 | self.assertEqual(metrics["repo2"], {"rc": 1}) 307 | self.assertEqual(runner_instance.metrics["errors"], 1) 308 | 309 | @patch("runrestic.restic.runner.MultiCommand") 310 | @patch("runrestic.restic.runner.parse_backup") 311 | @patch( 312 | "runrestic.restic.runner.redact_password", side_effect=lambda repo, repl: repo 313 | ) 314 | def test_backup_with_pre_and_post_hooks( 315 | self, mock_redact, mock_parse_backup, mock_mc 316 | ): 317 | """ 318 | Test backup() runs pre_hooks, the main backup, and post_hooks with correct arguments and metrics. 319 | """ 320 | # Arrange 321 | config: dict[str, Any] = { 322 | "repositories": ["repo"], 323 | "environment": {}, 324 | "execution": {"foo": "bar"}, 325 | "backup": { 326 | "sources": ["data"], 327 | "pre_hooks": [["echo", "pre1"], ["echo", "pre2"]], 328 | "post_hooks": [["echo", "post1"], ["echo", "post2"]], 329 | }, 330 | "metrics": {"prometheus": {"password_replacement": ""}}, 331 | } 332 | args = Namespace(dry_run=False) 333 | restic_args = ["--opt"] 334 | runner_instance = runner.ResticRunner(config, args, restic_args) 335 | 336 | # Simulate runs 337 | pre_runs = [ 338 | {"output": [(0, "")], "time": 0.5}, 339 | {"output": [(0, "")], "time": 0.2}, 340 | ] 341 | main_runs = [ 342 | {"output": [(0, "")], "time": 1.0}, 343 | ] 344 | post_runs = [ 345 | {"output": [(0, "")], "time": 0.3}, 346 | {"output": [(1, "")], "time": 0.1}, 347 | ] 348 | # run() called three times: pre, main, post 349 | mock_mc.return_value.run.side_effect = [pre_runs, main_runs, post_runs] 350 | mock_parse_backup.return_value = {"parsed": True} 351 | 352 | # Act 353 | runner_instance.backup() 354 | 355 | # Assert MultiCommand instantiations 356 | hooks_cfg = config["execution"].copy() 357 | hooks_cfg.update({"parallel": False, "shell": True}) 358 | 359 | calls = mock_mc.call_args_list 360 | # 1) pre_hooks 361 | self.assertEqual(calls[0][0][0], config["backup"]["pre_hooks"]) 362 | self.assertEqual(calls[0][1], {"config": hooks_cfg}) 363 | # 2) main backup 364 | expected_cmds = [ 365 | [ 366 | "restic", 367 | "-r", 368 | "repo", 369 | "backup", 370 | *restic_args, 371 | *config["backup"]["sources"], 372 | ] 373 | ] 374 | expected_abort = ["Fatal: unable to open config file", "Fatal: wrong password"] 375 | self.assertEqual(calls[1][0][0], expected_cmds) 376 | self.assertEqual(calls[1][0][1], config["execution"]) 377 | self.assertEqual(calls[1][0][2], expected_abort) 378 | # 3) post_hooks 379 | self.assertEqual(calls[2][0][0], config["backup"]["post_hooks"]) 380 | self.assertEqual(calls[2][1], {"config": hooks_cfg}) 381 | 382 | # Assert metrics 383 | m = runner_instance.metrics["backup"] 384 | # pre_hooks 385 | self.assertAlmostEqual(m["_restic_pre_hooks"]["duration_seconds"], 0.7) 386 | self.assertEqual(m["_restic_pre_hooks"]["rc"], 0) 387 | # main backup 388 | self.assertEqual(m["repo"], {"parsed": True}) 389 | # post_hooks 390 | self.assertAlmostEqual(m["_restic_post_hooks"]["duration_seconds"], 0.4) 391 | self.assertEqual(m["_restic_post_hooks"]["rc"], 1) 392 | # errors only increment on main backup failures (none here) 393 | self.assertEqual(runner_instance.metrics["errors"], 0) 394 | 395 | @patch("runrestic.restic.runner.MultiCommand") 396 | @patch("runrestic.restic.runner.logger.warning") 397 | @patch("runrestic.restic.runner.logger.info") 398 | def test_unlock_logs_success_and_failure(self, mock_info, mock_warning, mock_mc): 399 | """ 400 | Test that unlock() logs info for successful unlocks and warning for failures. 401 | """ 402 | config = { 403 | "repositories": ["repo1", "repo2"], 404 | "environment": {}, 405 | "execution": {}, 406 | } 407 | args = Namespace(dry_run=False) 408 | restic_args = ["--opt"] 409 | runner_instance = runner.ResticRunner(config, args, restic_args) 410 | 411 | # Simulate one successful unlock (0) and one failure (1) 412 | outputs = [ 413 | {"output": [(0, "ok")]}, 414 | {"output": [(1, "error")]}, 415 | ] 416 | mock_mc.return_value.run.return_value = outputs 417 | 418 | runner_instance.unlock() 419 | 420 | # Verify MultiCommand was constructed correctly 421 | expected_cmds = [ 422 | ["restic", "-r", "repo1", "unlock", *restic_args], 423 | ["restic", "-r", "repo2", "unlock", *restic_args], 424 | ] 425 | mock_mc.assert_called_once_with( 426 | expected_cmds, 427 | config=config["execution"], 428 | abort_reasons=[ 429 | "Fatal: unable to open config file", 430 | "Fatal: wrong password", 431 | ], 432 | ) 433 | mock_mc.return_value.run.assert_called_once() 434 | 435 | # Check logger calls 436 | mock_info.assert_called_once_with(outputs[0]["output"]) 437 | mock_warning.assert_called_once_with(outputs[1]["output"]) 438 | 439 | @patch("runrestic.restic.runner.MultiCommand") 440 | @patch("runrestic.restic.runner.parse_forget") 441 | @patch( 442 | "runrestic.restic.runner.redact_password", side_effect=lambda repo, repl: repo 443 | ) 444 | def test_forget_metrics(self, mock_redact, mock_parse_forget, mock_mc): 445 | """ 446 | Test forget() handles metrics parsing and errors correctly. 447 | """ 448 | config = { 449 | "repositories": ["repo"], 450 | "environment": {}, 451 | "execution": {}, 452 | "prune": {"keep-last": 3}, 453 | } 454 | args = Namespace(dry_run=True) 455 | restic_args: list[str] = [] 456 | runner_instance = runner.ResticRunner(config, args, restic_args) 457 | process_info = {"output": [(0, "")], "time": 0.1} 458 | mock_mc.return_value.run.return_value = [process_info] 459 | mock_parse_forget.return_value = {"forgotten": True} 460 | 461 | runner_instance.forget() 462 | metrics = runner_instance.metrics["forget"] 463 | self.assertEqual(metrics["repo"], {"forgotten": True}) 464 | self.assertEqual(runner_instance.metrics["errors"], 0) 465 | 466 | @patch("runrestic.restic.runner.MultiCommand") 467 | @patch("runrestic.restic.runner.parse_forget") 468 | @patch( 469 | "runrestic.restic.runner.redact_password", side_effect=lambda repo, repl: repo 470 | ) 471 | def test_forget_failure_increments_errors( 472 | self, 473 | mock_redact, 474 | mock_parse_forget, 475 | mock_mc, 476 | ): 477 | """ 478 | Test that forget() handles a non-zero return code by recording rc and incrementing errors, 479 | and does not call parse_forget on failure. 480 | """ 481 | config = { 482 | "repositories": ["repo"], 483 | "environment": {}, 484 | "execution": {}, 485 | "prune": {"keep-last": 2}, 486 | } 487 | args = Namespace(dry_run=True) # dry_run should add "--dry-run" 488 | restic_args: list[str] = [] 489 | runner_instance = runner.ResticRunner(config, args, restic_args) 490 | 491 | # Simulate failure return code 492 | failure_run = {"output": [(1, "error")], "time": 0.1} 493 | mock_mc.return_value.run.return_value = [failure_run] 494 | 495 | runner_instance.forget() 496 | 497 | # Should not parse on error 498 | mock_parse_forget.assert_not_called() 499 | 500 | # Metrics for repo should record rc only 501 | forget_metrics = runner_instance.metrics["forget"] 502 | self.assertEqual(forget_metrics["repo"], {"rc": 1}) 503 | # Error counter should be incremented 504 | self.assertEqual(runner_instance.metrics["errors"], 1) 505 | 506 | # Ensure "--dry-run" was included in the command 507 | expected_cmds = [ 508 | ["restic", "-r", "repo", "forget", "--dry-run", "--keep-last", "2"] 509 | ] 510 | mock_mc.assert_called_once_with( 511 | expected_cmds, 512 | config=config["execution"], 513 | abort_reasons=[ 514 | "Fatal: unable to open config file", 515 | "Fatal: wrong password", 516 | ], 517 | ) 518 | 519 | @patch("runrestic.restic.runner.MultiCommand") 520 | @patch("runrestic.restic.runner.parse_forget") 521 | @patch( 522 | "runrestic.restic.runner.redact_password", side_effect=lambda repo, repl: repo 523 | ) 524 | def test_forget_with_group_by_and_success( 525 | self, 526 | mock_redact, 527 | mock_parse_forget, 528 | mock_mc, 529 | ): 530 | """ 531 | Test that forget() includes '--group-by' when configured and calls parse_forget on success. 532 | """ 533 | config = { 534 | "repositories": ["repo"], 535 | "environment": {}, 536 | "execution": {}, 537 | "prune": {"group-by": "tag"}, 538 | } 539 | args = Namespace(dry_run=False) 540 | restic_args: list[str] = [] 541 | runner_instance = runner.ResticRunner(config, args, restic_args) 542 | 543 | # Simulate successful run 544 | success_run = {"output": [(0, "ok")], "time": 0.2} 545 | mock_mc.return_value.run.return_value = [success_run] 546 | mock_parse_forget.return_value = {"forgotten": True} 547 | 548 | runner_instance.forget() 549 | 550 | # Should parse on success 551 | mock_parse_forget.assert_called_once_with(success_run) 552 | 553 | # Metrics should include parsed value 554 | forget_metrics = runner_instance.metrics["forget"] 555 | self.assertEqual(forget_metrics["repo"], {"forgotten": True}) 556 | self.assertEqual(runner_instance.metrics["errors"], 0) 557 | 558 | # Ensure "--group-by tag" appears in the command 559 | expected_cmds = [["restic", "-r", "repo", "forget", "--group-by", "tag"]] 560 | mock_mc.assert_called_once_with( 561 | expected_cmds, 562 | config=config["execution"], 563 | abort_reasons=[ 564 | "Fatal: unable to open config file", 565 | "Fatal: wrong password", 566 | ], 567 | ) 568 | 569 | @patch("runrestic.restic.runner.MultiCommand") 570 | @patch("runrestic.restic.runner.parse_new_prune", side_effect=IndexError) 571 | @patch("runrestic.restic.runner.parse_prune") 572 | @patch( 573 | "runrestic.restic.runner.redact_password", side_effect=lambda repo, repl: repo 574 | ) 575 | def test_prune_metrics_old_prune( 576 | self, mock_redact, mock_parse_prune, mock_new_prune, mock_mc 577 | ): 578 | """ 579 | Test prune() falls back to parse_prune when parse_new_prune raises IndexError. 580 | """ 581 | config = { 582 | "repositories": ["repo"], 583 | "environment": {}, 584 | "execution": {}, 585 | } 586 | args = Namespace(dry_run=False) 587 | restic_args: list[str] = [] 588 | runner_instance = runner.ResticRunner(config, args, restic_args) 589 | process_info = {"output": [(0, "")], "time": 0.1} 590 | mock_mc.return_value.run.return_value = [process_info] 591 | mock_parse_prune.return_value = {"pruned": True} 592 | 593 | runner_instance.prune() 594 | metrics = runner_instance.metrics["prune"] 595 | self.assertEqual(metrics["repo"], {"pruned": True}) 596 | 597 | @patch("runrestic.restic.runner.MultiCommand") 598 | @patch("runrestic.restic.runner.parse_new_prune") 599 | @patch( 600 | "runrestic.restic.runner.redact_password", side_effect=lambda repo, repl: repo 601 | ) 602 | def test_prune_metrics_new_prune(self, mock_redact, mock_parse_new_prune, mock_mc): 603 | """ 604 | Test prune() uses parse_new_prune when available. 605 | """ 606 | config = { 607 | "repositories": ["repo"], 608 | "environment": {}, 609 | "execution": {}, 610 | } 611 | args = Namespace(dry_run=False) 612 | restic_args: list[str] = [] 613 | runner_instance = runner.ResticRunner(config, args, restic_args) 614 | process_info = {"output": [(0, "")], "time": 0.1} 615 | mock_mc.return_value.run.return_value = [process_info] 616 | mock_parse_new_prune.return_value = {"new_pruned": True} 617 | 618 | runner_instance.prune() 619 | metrics = runner_instance.metrics["prune"] 620 | self.assertEqual(metrics["repo"], {"new_pruned": True}) 621 | 622 | @patch("runrestic.restic.runner.MultiCommand") 623 | @patch("runrestic.restic.runner.parse_new_prune") 624 | @patch("runrestic.restic.runner.parse_prune") 625 | @patch( 626 | "runrestic.restic.runner.redact_password", side_effect=lambda repo, repl: repo 627 | ) 628 | def test_prune_failure_increments_errors( 629 | self, 630 | mock_redact, 631 | mock_parse_prune, 632 | mock_parse_new_prune, 633 | mock_mc, 634 | ): 635 | """ 636 | Test prune() handles a non-zero return code by recording rc and incrementing errors. 637 | """ 638 | # Setup 639 | config = { 640 | "repositories": ["repo"], 641 | "environment": {}, 642 | "execution": {}, 643 | } 644 | args = Namespace(dry_run=False) 645 | restic_args: list[str] = [] 646 | runner_instance = runner.ResticRunner(config, args, restic_args) 647 | 648 | # Simulate prune failure 649 | failure_run = {"output": [(1, "prune error")], "time": 0.1} 650 | mock_mc.return_value.run.return_value = [failure_run] 651 | 652 | # Execute 653 | runner_instance.prune() 654 | 655 | # Should not attempt to parse on error 656 | mock_parse_new_prune.assert_not_called() 657 | mock_parse_prune.assert_not_called() 658 | 659 | # Metrics should record the rc and errors should increment 660 | prune_metrics = runner_instance.metrics["prune"] 661 | self.assertEqual(prune_metrics["repo"], {"rc": 1}) 662 | self.assertEqual(runner_instance.metrics["errors"], 1) 663 | 664 | @patch("runrestic.restic.runner.MultiCommand") 665 | @patch( 666 | "runrestic.restic.runner.redact_password", side_effect=lambda repo, repl: repo 667 | ) 668 | @patch("runrestic.restic.runner.parse_stats") 669 | def test_stats_metrics(self, mock_parse_stats, mock_redact, mock_mc): 670 | """ 671 | Test stats() calls parse_stats and updates metrics correctly. 672 | """ 673 | config = { 674 | "repositories": ["repo"], 675 | "environment": {}, 676 | "execution": {}, 677 | } 678 | args = Namespace(dry_run=False) 679 | restic_args = ["--verbose"] 680 | runner_instance = runner.ResticRunner(config, args, restic_args) 681 | process_info = {"output": [(0, "")], "time": 0.1} 682 | mock_mc.return_value.run.return_value = [process_info] 683 | mock_parse_stats.return_value = {"stats": True} 684 | 685 | runner_instance.stats() 686 | metrics = runner_instance.metrics["stats"] 687 | self.assertEqual(metrics["repo"], {"stats": True}) 688 | self.assertEqual(runner_instance.metrics["errors"], 0) 689 | 690 | @patch("runrestic.restic.runner.MultiCommand") 691 | @patch( 692 | "runrestic.restic.runner.redact_password", side_effect=lambda repo, repl: repo 693 | ) 694 | @patch("runrestic.restic.runner.parse_stats") 695 | def test_stats_failure_increments_errors( 696 | self, mock_parse_stats, mock_redact, mock_mc 697 | ): 698 | """ 699 | Test stats() handles return_code > 0 by recording rc and incrementing errors. 700 | """ 701 | config = { 702 | "repositories": ["repo"], 703 | "environment": {}, 704 | "execution": {}, 705 | } 706 | args = Namespace(dry_run=False) 707 | restic_args = ["--verbose"] 708 | runner_instance = runner.ResticRunner(config, args, restic_args) 709 | 710 | # simulate failure 711 | process_info = {"output": [(1, "error occurred")], "time": 0.1} 712 | mock_mc.return_value.run.return_value = [process_info] 713 | 714 | runner_instance.stats() 715 | 716 | # parse_stats should NOT be called on failure 717 | mock_parse_stats.assert_not_called() 718 | 719 | metrics = runner_instance.metrics["stats"] 720 | # rc should be recorded 721 | self.assertEqual(metrics["repo"], {"rc": 1}) 722 | # errors counter should have been incremented by 1 723 | self.assertEqual(runner_instance.metrics["errors"], 1) 724 | 725 | @patch("runrestic.restic.runner.MultiCommand") 726 | def test_check_metrics_with_and_without_options(self, mock_mc): 727 | """ 728 | Test check() with and without check-options: 729 | - Verifies the --check-unused and --read-data flags 730 | - Asserts MultiCommand args and abort_reasons 731 | - Validates per-repo metrics and global error count 732 | """ 733 | scenarios: list[dict[str, Any]] = [ 734 | { 735 | "name": "base_check", 736 | "config": { 737 | "repositories": ["repo"], 738 | "environment": {}, 739 | "execution": {}, 740 | }, 741 | "expected_commands": [["restic", "-r", "repo", "check"]], 742 | "expected_stats": { 743 | "check_unused": 0, 744 | "read_data": 0, 745 | }, 746 | }, 747 | { 748 | "name": "check_unused", 749 | "config": { 750 | "repositories": ["repo"], 751 | "environment": {}, 752 | "execution": {}, 753 | "check": { 754 | "checks": ["check-unused"], 755 | }, 756 | }, 757 | "expected_commands": [ 758 | ["restic", "-r", "repo", "check", "--check-unused"] 759 | ], 760 | "expected_stats": { 761 | "check_unused": 1, 762 | "read_data": 0, 763 | }, 764 | }, 765 | { 766 | "name": "check_read_data", 767 | "config": { 768 | "repositories": ["repo"], 769 | "environment": {}, 770 | "execution": {}, 771 | "check": { 772 | "checks": ["read-data"], 773 | }, 774 | }, 775 | "expected_commands": [["restic", "-r", "repo", "check", "--read-data"]], 776 | "expected_stats": { 777 | "check_unused": 0, 778 | "read_data": 1, 779 | }, 780 | }, 781 | ] 782 | # simulate a failure output 783 | output_str = "error: load \nPack ID does not match, corrupted" 784 | process_info = {"output": [(1, output_str)], "time": 0.5} 785 | mock_mc.return_value.run.return_value = [process_info] 786 | 787 | for sc in scenarios: 788 | with self.subTest(sc["name"]): 789 | # build config 790 | 791 | args = Namespace(dry_run=False) 792 | restic_args: list[str] = [] 793 | runner_instance = runner.ResticRunner(sc["config"], args, restic_args) 794 | 795 | # clear any prior error count 796 | runner_instance.metrics["errors"] = 0 797 | 798 | # run 799 | runner_instance.check() 800 | 801 | # validate MultiCommand instantiation 802 | expected_commands = sc["expected_commands"] 803 | expected_abort = [ 804 | "Fatal: unable to open config file", 805 | "Fatal: wrong password", 806 | ] 807 | mock_mc.assert_called_once_with( 808 | expected_commands, 809 | config=sc["config"]["execution"], 810 | abort_reasons=expected_abort, 811 | ) 812 | mock_mc.return_value.run.assert_called_once() 813 | 814 | # self.assertEqual(config, base_config) 815 | # combined per-repo metrics assertion 816 | expected_stats = { 817 | "errors": 1, 818 | "errors_snapshots": 1, 819 | "errors_data": 1, 820 | "check_unused": sc["expected_stats"]["check_unused"], 821 | "read_data": sc["expected_stats"]["read_data"], 822 | "duration_seconds": 0.5, 823 | "rc": 1, 824 | } 825 | self.assertEqual( 826 | runner_instance.metrics["check"]["repo"], expected_stats 827 | ) 828 | 829 | # global errors counter 830 | self.assertEqual(runner_instance.metrics["errors"], 1) 831 | 832 | # reset between subtests 833 | mock_mc.reset_mock() 834 | 835 | @patch("runrestic.restic.runner.MultiCommand") 836 | def test_check_metrics_with_errors(self, mock_mc): 837 | """ 838 | Test check() with and different error scenarios 839 | """ 840 | config = { 841 | "repositories": ["repo"], 842 | "environment": {}, 843 | "execution": {}, 844 | } 845 | scenarios: list[dict[str, Any]] = [ 846 | { 847 | "name": "return_code", 848 | "process_info": {"output": [(1, "error occurred")], "time": 0.1}, 849 | "global_errors": 1, 850 | "check_errors": 0, 851 | }, 852 | { 853 | "name": "load_error", 854 | "process_info": { 855 | "output": [(0, "Test: error: load ")], 856 | "time": 0.1, 857 | }, 858 | "global_errors": 0, 859 | "check_errors": 1, 860 | }, 861 | { 862 | "name": "pack_id_mismatch", 863 | "process_info": { 864 | "output": [(0, "Test: Pack ID does not match, WRONG")], 865 | "time": 0.1, 866 | }, 867 | "global_errors": 0, 868 | "check_errors": 1, 869 | }, 870 | ] 871 | 872 | for sc in scenarios: 873 | with self.subTest(sc["name"]): 874 | # build config 875 | args = Namespace(dry_run=False) 876 | restic_args: list[str] = [] 877 | runner_instance = runner.ResticRunner(config, args, restic_args) 878 | # clear any prior error count 879 | runner_instance.metrics["errors"] = 0 880 | # simulate failure 881 | mock_mc.return_value.run.return_value = [sc["process_info"]] 882 | # run 883 | runner_instance.check() 884 | mock_mc.return_value.run.assert_called_once() 885 | # global errors counter 886 | self.assertEqual(runner_instance.metrics["errors"], sc["global_errors"]) 887 | # check errors counter 888 | self.assertEqual( 889 | runner_instance.metrics["check"]["repo"]["errors"], 890 | sc["check_errors"], 891 | ) 892 | # reset between subtests 893 | mock_mc.reset_mock() 894 | -------------------------------------------------------------------------------- /tests/restic/test_shell.py: -------------------------------------------------------------------------------- 1 | import os 2 | from unittest import TestCase 3 | from unittest.mock import patch 4 | 5 | from runrestic.restic import shell 6 | 7 | 8 | class TestResticShell(TestCase): 9 | def setUp(self) -> None: 10 | # Define the environment variable for testing 11 | if not os.environ.get("SHELL"): 12 | # Set a default shell for the test environment 13 | os.environ["SHELL"] = "/bin/bash" 14 | 15 | @patch("runrestic.restic.shell.logger") 16 | @patch("runrestic.restic.shell.sys.exit") 17 | def test_restic_shell_single_repo(self, mock_sys_exit, mock_logger): 18 | """ 19 | Test the restic_shell function with a single repository configuration. 20 | """ 21 | configs = [ 22 | { 23 | "name": "TestConfig", 24 | "repositories": ["test_repo"], 25 | "environment": {"TEST_ENV": "test_value"}, 26 | } 27 | ] 28 | 29 | with ( 30 | patch("builtins.print") as mock_print, 31 | patch("runrestic.restic.shell.pty.spawn") as mock_spawn, 32 | ): 33 | shell.restic_shell(configs) 34 | 35 | mock_print.assert_any_call("Using: \033[1;92mTestConfig:test_repo\033[0m") 36 | mock_print.assert_any_call( 37 | "Spawning a new shell with the restic environment variables all set." 38 | ) 39 | mock_spawn.assert_called_once_with(os.environ["SHELL"]) 40 | mock_sys_exit.assert_called_once_with(0) 41 | 42 | @patch("runrestic.restic.shell.logger") 43 | @patch("builtins.input", return_value="1") 44 | @patch("runrestic.restic.shell.sys.exit") 45 | def test_restic_shell_multi_repo(self, mock_sys_exit, mock_input, mock_logger): 46 | """ 47 | Test the restic_shell function with a multiple repositories configuration. 48 | """ 49 | configs = [ 50 | { 51 | "name": "TestConfig", 52 | "repositories": ["test_repo_1", "test_repo_2"], 53 | "environment": {"TEST_ENV": "test_value"}, 54 | } 55 | ] 56 | 57 | with ( 58 | patch("builtins.print") as mock_print, 59 | patch("runrestic.restic.shell.pty.spawn") as mock_spawn, 60 | ): 61 | shell.restic_shell(configs) 62 | 63 | mock_print.assert_any_call("Using: \033[1;92mTestConfig:test_repo_2\033[0m") 64 | mock_print.assert_any_call( 65 | "Spawning a new shell with the restic environment variables all set." 66 | ) 67 | mock_spawn.assert_called_once_with(os.environ["SHELL"]) 68 | mock_sys_exit.assert_called_once_with(0) 69 | 70 | @patch("runrestic.restic.shell.logger") 71 | @patch("builtins.input", return_value="X") 72 | @patch("runrestic.restic.shell.sys.exit") 73 | def test_restic_shell_multi_repo_invalid_selection( 74 | self, mock_sys_exit, mock_input, mock_logger 75 | ): 76 | """ 77 | Test the restic_shell function with a multiple repositories configuration with invalid user selection. 78 | """ 79 | configs = [ 80 | { 81 | "name": "TestConfig", 82 | "repositories": ["test_repo_1", "test_repo_2"], 83 | "environment": {"TEST_ENV": "test_value"}, 84 | } 85 | ] 86 | 87 | with self.assertRaises(ValueError) as context: 88 | shell.restic_shell(configs) 89 | 90 | self.assertEqual( 91 | str(context.exception), 92 | "invalid literal for int() with base 10: 'X'", 93 | ) 94 | -------------------------------------------------------------------------------- /tests/restic/test_tools_subprocess.py: -------------------------------------------------------------------------------- 1 | """Test log output for sub-processes 2 | 3 | Using pytest-subprocess plugin https://pytest-subprocess.readthedocs.io/ 4 | """ 5 | 6 | import logging 7 | import subprocess 8 | from io import StringIO 9 | 10 | from runrestic.restic import tools 11 | 12 | 13 | def test_log_messages_no_output(): 14 | """Test log messages with no output""" 15 | # Create a fake process object 16 | fake_proc = type("FakeProc", (), {})() 17 | fake_proc.stdout = StringIO(" ") 18 | assert tools.log_messages(fake_proc, "test_cmd") == "" 19 | 20 | 21 | def test_restic_logs(caplog, fp, monkeypatch): # pylint: disable=invalid-name 22 | cmd = ["restic", "-r", "test_repo", "backup"] 23 | out = [ 24 | "using parent snapshot b601066b", 25 | "Files: 5 new, 42 changed, 1842 unmodified", 26 | "Dirs: 1 new, 4 changed, 103 unmodified", 27 | "Added to the repo: 43 MiB", 28 | "processed 1888 files, 11.342 GiB in 0:00", 29 | "snapshot 1e3c30a1 saved", 30 | ] 31 | fp.register( 32 | cmd, 33 | stdout=out, 34 | returncode=0, 35 | ) 36 | monkeypatch.setattr(tools, "Popen", subprocess.Popen) 37 | caplog.set_level(logging.INFO) 38 | result = tools.retry_process( 39 | cmd, 40 | config={}, 41 | ) 42 | assert result["output"] == [(0, "\n".join([*out, ""]))] 43 | for log in out: 44 | assert log in caplog.text 45 | assert caplog.record_tuples[0] == ( 46 | "runrestic.restic.tools", 47 | 20, 48 | "[restic] using parent snapshot b601066b", 49 | ) 50 | assert caplog.record_tuples[-1] == ( 51 | "runrestic.restic.tools", 52 | 20, 53 | "[restic] snapshot 1e3c30a1 saved", 54 | ) 55 | 56 | 57 | def test_restic_abort(caplog, fp, monkeypatch): # pylint: disable=invalid-name 58 | cmd = ["restic", "-r", "test_repo", "backup"] 59 | out = ["Fatal: wrong password"] 60 | # Register 3 calls failing 61 | fp.register(cmd, stdout=out, returncode=1, occurrences=3) 62 | monkeypatch.setattr(tools, "Popen", subprocess.Popen) 63 | caplog.set_level(logging.INFO) 64 | result = tools.retry_process( 65 | cmd, config={"retry_count": 2}, abort_reasons=["Fatal: wrong password"] 66 | ) 67 | assert result["output"] == [(1, "\n".join([*out, ""]))] 68 | assert ( 69 | "runrestic.restic.tools", 70 | logging.CRITICAL, 71 | "[restic] Fatal: wrong password", 72 | ) in caplog.record_tuples 73 | assert ( 74 | "runrestic.restic.tools", 75 | logging.ERROR, 76 | "Aborting 'restic' because of ['Fatal: wrong password']", 77 | ) in caplog.record_tuples 78 | 79 | 80 | def test_retry_pass_logs(caplog, fp, monkeypatch): # pylint: disable=invalid-name 81 | cmd = ["restic", "-r", "test_repo", "backup"] 82 | out_fail = ["Fatal: something went wrong"] 83 | out_pass = ["snapshot 1e3c30a1 saved"] 84 | retries = 2 85 | # Register #'retries' calls failing 86 | fp.register(cmd, stdout=out_fail, returncode=1, occurrences=retries) 87 | # Register final call success 88 | fp.register(cmd, stdout=out_pass, returncode=0) 89 | monkeypatch.setattr(tools, "Popen", subprocess.Popen) 90 | caplog.set_level(logging.INFO) 91 | result = tools.retry_process( 92 | cmd, config={"retry_count": retries}, abort_reasons=["Fatal: wrong password"] 93 | ) 94 | assert result["output"] == [(1, out_fail[0] + "\n")] * retries + [ 95 | (0, out_pass[0] + "\n") 96 | ] 97 | assert ( 98 | "runrestic.restic.tools", 99 | logging.CRITICAL, 100 | "[restic] Fatal: something went wrong", 101 | ) in caplog.record_tuples 102 | assert ( 103 | "runrestic.restic.tools", 104 | logging.INFO, 105 | f"Retry {retries}/{retries + 1} command 'restic'", 106 | ) in caplog.record_tuples 107 | 108 | 109 | def test_retry_fail_logs(caplog, fp, monkeypatch): # pylint: disable=invalid-name 110 | cmd = ["restic", "-r", "test_repo", "backup"] 111 | out_fail = ["Fatal: something went wrong"] 112 | out_pass = ["snapshot 1e3c30a1 saved"] 113 | retries = 3 114 | # Register 2 calls failing 115 | fp.register(cmd, stdout=out_fail, returncode=1, occurrences=retries + 1) 116 | # Register 3rd call success 117 | fp.register(cmd, stdout=out_pass, returncode=0) 118 | monkeypatch.setattr(tools, "Popen", subprocess.Popen) 119 | caplog.set_level(logging.INFO) 120 | result = tools.retry_process( 121 | cmd, config={"retry_count": retries}, abort_reasons=["Fatal: wrong password"] 122 | ) 123 | assert result["output"] == [(1, out_fail[0] + "\n")] * (retries + 1) 124 | assert ( 125 | "runrestic.restic.tools", 126 | logging.CRITICAL, 127 | "[restic] Fatal: something went wrong", 128 | ) in caplog.record_tuples 129 | assert ( 130 | "runrestic.restic.tools", 131 | logging.INFO, 132 | f"Retry 2/{retries + 1} command 'restic'", 133 | ) in caplog.record_tuples 134 | 135 | 136 | def test_log_level_mapping(caplog, fp, monkeypatch): # pylint: disable=invalid-name 137 | cmd = ["restic", "-r", "test_repo", "backup"] 138 | test_messages = { 139 | "Random log message, not an ERROR": logging.INFO, 140 | "unchanged /debug/log/message": logging.DEBUG, 141 | "warning: restic had some issue": logging.WARNING, 142 | "ERROR log message upper": logging.ERROR, 143 | "Error: log message mix": logging.ERROR, 144 | "FATAL as critical message": logging.CRITICAL, 145 | "CRITICAL log message": logging.CRITICAL, 146 | } 147 | fp.register(cmd, stdout=list(test_messages.keys()), returncode=1, occurrences=3) 148 | monkeypatch.setattr(tools, "Popen", subprocess.Popen) 149 | caplog.set_level(logging.DEBUG) 150 | 151 | result = tools.retry_process( 152 | cmd, 153 | config={}, 154 | ) 155 | assert result["output"][-1] == (1, "\n".join(test_messages.keys()) + "\n") 156 | for message, level in test_messages.items(): 157 | assert ( 158 | "runrestic.restic.tools", 159 | level, 160 | f"[restic] {message}", 161 | ) in caplog.record_tuples 162 | 163 | 164 | def test_hook_log(caplog, fp, monkeypatch): # pylint: disable=invalid-name 165 | cmd = "hook_cmd --some-option 123 -v" 166 | test_messages = { 167 | "Random log message, not an ERROR": logging.INFO, 168 | "unchanged /debug/log/message": logging.DEBUG, 169 | "warning: hook cmd had some issue": logging.WARNING, 170 | "ERROR log message upper": logging.ERROR, 171 | "Error: log message mix": logging.ERROR, 172 | "FATAL as critical message": logging.CRITICAL, 173 | "CRITICAL log message": logging.CRITICAL, 174 | } 175 | fp.register(cmd, stdout=list(test_messages.keys()), returncode=0, occurrences=1) 176 | monkeypatch.setattr(tools, "Popen", subprocess.Popen) 177 | caplog.set_level(logging.DEBUG) 178 | 179 | result = tools.retry_process( 180 | cmd, 181 | config={}, 182 | ) 183 | assert result["output"] == [(0, "\n".join(test_messages.keys()) + "\n")] 184 | for message, level in test_messages.items(): 185 | assert ( 186 | "runrestic.restic.tools", 187 | level, 188 | f"[{cmd.split(' ', maxsplit=1)[0]}] {message}", 189 | ) in caplog.record_tuples 190 | -------------------------------------------------------------------------------- /tests/runrestic/test_configuration.py: -------------------------------------------------------------------------------- 1 | import os 2 | from argparse import Namespace 3 | from unittest.mock import patch 4 | 5 | import pytest 6 | from toml import TomlDecodeError 7 | 8 | from runrestic.runrestic.configuration import ( 9 | cli_arguments, 10 | configuration_file_paths, 11 | parse_configuration, 12 | possible_config_paths, 13 | ) 14 | 15 | 16 | def test_cli_arguments(): 17 | assert cli_arguments([]) == ( 18 | Namespace( 19 | actions=[], 20 | config_file=None, 21 | dry_run=False, 22 | log_level="info", 23 | show_progress=None, 24 | ), 25 | [], 26 | ) 27 | assert cli_arguments(["-l", "debug"]) == ( 28 | Namespace( 29 | actions=[], 30 | config_file=None, 31 | dry_run=False, 32 | log_level="debug", 33 | show_progress=None, 34 | ), 35 | [], 36 | ) 37 | assert cli_arguments(["backup"]) == ( 38 | Namespace( 39 | actions=["backup"], 40 | config_file=None, 41 | dry_run=False, 42 | log_level="info", 43 | show_progress=None, 44 | ), 45 | [], 46 | ) 47 | assert cli_arguments(["backup", "--", "--one-file-system"]) == ( 48 | Namespace( 49 | actions=["backup"], 50 | config_file=None, 51 | dry_run=False, 52 | log_level="info", 53 | show_progress=None, 54 | ), 55 | ["--one-file-system"], 56 | ) 57 | with pytest.raises(SystemExit): 58 | cli_arguments(["-h"]) 59 | 60 | 61 | @pytest.fixture 62 | def restic_dir(tmpdir): 63 | os.environ["XDG_CONFIG_HOME"] = str(tmpdir) 64 | return tmpdir.mkdir("runrestic") 65 | 66 | 67 | @pytest.fixture 68 | def restic_minimal_good_conf(restic_dir, request): 69 | """ 70 | Create a minimal valid Restic configuration file. 71 | 72 | Args: 73 | restic_dir: The directory where the configuration file will be created. 74 | request: A pytest fixture to access test-specific parameters. 75 | 76 | Returns: 77 | Path: The path to the created configuration file. 78 | """ 79 | p = restic_dir.join("example.toml") 80 | content = ( 81 | 'repositories = ["/tmp/restic-repo-1"]\n' 82 | "[environment]\n" 83 | 'RESTIC_PASSWORD = "CHANGEME"\n' 84 | "[backup]\n" 85 | 'sources = ["/etc"]\n' 86 | "[prune]\n" 87 | "keep-last = 10\n" 88 | ) 89 | # Optionally add the "name" field if specified in the test 90 | if hasattr(request, "param") and request.param.get("name"): 91 | content = f'name = "{request.param["name"]}"\n' + content 92 | 93 | p.write(content) 94 | os.chmod(p, 0o0600) 95 | return p 96 | 97 | 98 | @pytest.fixture 99 | def restic_minimal_broken_conf(restic_dir): 100 | p = restic_dir.join("example.toml") 101 | content = '[environment\nRESTIC_PASSWORD = {CHANGEME"' 102 | p.write(content) 103 | os.chmod(p, 0o0600) 104 | return p 105 | 106 | 107 | def test_possible_config_paths(tmpdir): 108 | os.environ["XDG_CONFIG_HOME"] = str(tmpdir) 109 | assert possible_config_paths() == [ 110 | "/etc/runrestic.toml", 111 | "/etc/runrestic.json", 112 | "/etc/runrestic/", 113 | f"{tmpdir}/runrestic/", 114 | ] 115 | 116 | 117 | def test_configuration_file_paths(restic_minimal_good_conf): 118 | paths = list(configuration_file_paths()) 119 | assert paths == [restic_minimal_good_conf] 120 | 121 | 122 | def test_configuration_file_paths_wrong_perms(caplog, restic_dir): 123 | bad_perms_file = restic_dir.join("example.toml") 124 | bad_perms_file.write("irrelevant") 125 | os.chmod(bad_perms_file, 0o0644) 126 | paths = list(configuration_file_paths()) 127 | assert paths == [] 128 | assert f"NOT using {bad_perms_file}." in caplog.text 129 | assert "File permissions are too open (0644)" in caplog.text 130 | 131 | 132 | def test_configuration_file_paths_not_exists(tmpdir): 133 | with patch("runrestic.runrestic.configuration.possible_config_paths") as mock_paths: 134 | mock_paths.return_value = [str(tmpdir.join("config.yaml"))] 135 | assert configuration_file_paths() == [] 136 | 137 | 138 | def test_configuration_file_paths_is_file(tmpdir): 139 | tmpdir.join("config.yaml").write("irrelevant") 140 | conf_paths = [str(tmpdir.join("config.yaml"))] 141 | with patch( 142 | "runrestic.runrestic.configuration.possible_config_paths" 143 | ) as mock_possible_paths: 144 | mock_possible_paths.return_value = conf_paths 145 | assert configuration_file_paths() == conf_paths 146 | 147 | 148 | def test_configuration_file_paths_exclude_dirs(tmpdir): 149 | tmpdir.join("config.yaml").mkdir() 150 | with patch( 151 | "runrestic.runrestic.configuration.possible_config_paths" 152 | ) as mock_possible_paths: 153 | mock_possible_paths.return_value = str(tmpdir) 154 | assert configuration_file_paths() == [] 155 | 156 | 157 | @pytest.mark.parametrize( 158 | "restic_minimal_good_conf, expected_name", 159 | [ 160 | ({}, "example.toml"), 161 | ({"name": "defined_name"}, "defined_name"), 162 | ], 163 | indirect=["restic_minimal_good_conf"], 164 | ) 165 | def test_parse_configuration_good_conf(restic_minimal_good_conf, expected_name): 166 | """ 167 | Test that the configuration file is parsed correctly and includes the correct 'name' field. 168 | 169 | Args: 170 | restic_minimal_good_conf: The path to the configuration file. 171 | expected_name: The expected 'name' field in the configuration. 172 | """ 173 | assert parse_configuration(restic_minimal_good_conf) == { 174 | "name": expected_name, 175 | "repositories": ["/tmp/restic-repo-1"], # noqa: S108 176 | "environment": {"RESTIC_PASSWORD": "CHANGEME"}, 177 | "execution": { 178 | "exit_on_error": True, 179 | "parallel": False, 180 | "retry_count": 0, 181 | }, 182 | "backup": {"sources": ["/etc"]}, 183 | "prune": {"keep-last": 10}, 184 | } 185 | 186 | 187 | def test_parse_configuration_broken_conf(caplog, restic_minimal_broken_conf): 188 | with pytest.raises(TomlDecodeError): 189 | parse_configuration(restic_minimal_broken_conf) 190 | 191 | 192 | def test_cli_arguments_with_extra_args(): 193 | assert cli_arguments( 194 | ["backup", "--one-file-system", "pos_arg", "--", "--more"] 195 | ) == ( 196 | Namespace( 197 | actions=["backup"], 198 | config_file=None, 199 | dry_run=False, 200 | log_level="info", 201 | show_progress=None, 202 | ), 203 | ["--one-file-system", "pos_arg", "--more"], 204 | ) 205 | 206 | 207 | # 208 | # def test_parse_configuration_broken_conf(restic_minimal_broken_conf): 209 | # with pytest.raises(jsonschema.exceptions.ValidationError): 210 | # parse_configuration(restic_minimal_broken_conf) 211 | -------------------------------------------------------------------------------- /tests/runrestic/test_runrestic.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import signal 4 | from unittest import TestCase 5 | from unittest.mock import MagicMock, patch 6 | 7 | from runrestic.runrestic import runrestic 8 | 9 | 10 | class TestRunresticRunrestic(TestCase): 11 | def test_configure_logging_levels(self): 12 | """ 13 | Parameterized test to ensure configure_logging correctly sets logger level, 14 | adds a StreamHandler at that level, and uses the "%(message)s" formatter. 15 | """ 16 | levels_to_test = ["error", "info", "debug"] 17 | logger_name = "runrestic" 18 | 19 | for level_str in levels_to_test: 20 | with self.subTest(level=level_str): 21 | logger = logging.getLogger(logger_name) 22 | 23 | # Backup and clear handlers 24 | original_handlers = logger.handlers[:] 25 | logger.handlers.clear() 26 | 27 | try: 28 | runrestic.configure_logging(level_str) 29 | 30 | expected_level = logging.getLevelName(level_str.upper()) 31 | 32 | # Check logger level 33 | self.assertEqual(logger.level, expected_level) 34 | 35 | # Check StreamHandler with correct level 36 | stream_handlers = [ 37 | h 38 | for h in logger.handlers 39 | if isinstance(h, logging.StreamHandler) 40 | and h.level == expected_level 41 | ] 42 | self.assertTrue( 43 | stream_handlers, 44 | f"No StreamHandler with level {level_str.upper()} found", 45 | ) 46 | 47 | # Check formatter format 48 | for handler in stream_handlers: 49 | formatter = handler.formatter 50 | self.assertIsNotNone(formatter, "Handler has no formatter") 51 | self.assertEqual( 52 | formatter._fmt, # type: ignore[union-attr] 53 | "%(message)s", 54 | "Formatter format is not '%(message)s'", 55 | ) 56 | finally: 57 | # Restore original handlers 58 | logger.handlers = original_handlers 59 | 60 | def test_handlers_registered_and_kill_group(self): 61 | """ 62 | Test that configure_signals registers handlers for the expected signals 63 | and that invoking each handler inside the patch context calls os.killpg 64 | with the current process group, without killing the test process. 65 | """ 66 | registered = {} 67 | 68 | def fake_signal(sig, handler): 69 | registered[sig] = handler 70 | return None 71 | 72 | fake_pgid = 12345 73 | 74 | with ( 75 | patch("signal.signal", new=fake_signal), 76 | patch("os.getpgrp", return_value=fake_pgid), 77 | patch("os.killpg") as mock_killpg, 78 | ): 79 | runrestic.configure_signals() 80 | 81 | # Ensure handlers are registered for the expected signals 82 | expected_signals = { 83 | signal.SIGINT, 84 | signal.SIGHUP, 85 | signal.SIGTERM, 86 | signal.SIGUSR1, 87 | signal.SIGUSR2, 88 | } 89 | self.assertEqual( 90 | set(registered.keys()), 91 | expected_signals, 92 | f"Expected registrations for {expected_signals}, got {set(registered.keys())}", 93 | ) 94 | 95 | # Invoke each handler inside the patch context to ensure killpg is mocked 96 | for sig in expected_signals: 97 | handler = registered[sig] 98 | self.assertTrue(callable(handler), f"Handler for {sig} is not callable") 99 | handler(sig, None) 100 | mock_killpg.assert_called_with(fake_pgid, sig) 101 | mock_killpg.reset_mock() 102 | 103 | @patch("runrestic.runrestic.runrestic.restic_check", return_value=False) 104 | def test_no_resto_binary(self, mock_check): 105 | # If restic_check() is False, runrestic returns early 106 | self.assertIsNone(runrestic.runrestic()) # type: ignore[func-returns-value] 107 | mock_check.assert_called_once() 108 | 109 | @patch( 110 | "runrestic.runrestic.runrestic.parse_configuration", 111 | side_effect=KeyError("Error"), 112 | ) 113 | @patch("runrestic.runrestic.runrestic.configure_logging") 114 | @patch("runrestic.runrestic.runrestic.restic_check", return_value=True) 115 | @patch("runrestic.runrestic.runrestic.cli_arguments") 116 | def test_config_file_flag(self, mock_cli, mock_check, mock_log, mock_parse): 117 | args = MagicMock() 118 | args.log_level = "info" 119 | args.config_file = "/tmp/config" # noqa: S108 120 | args.actions = [] 121 | args.show_progress = None 122 | extras: list[str] = [] 123 | mock_cli.return_value = (args, extras) 124 | 125 | # with patch("runrestic.runrestic.runrestic.parse_configuration", return_value={"cfg": 1}): 126 | with self.assertRaises(KeyError): 127 | runrestic.runrestic() 128 | mock_parse.assert_called_once_with("/tmp/config") # noqa: S108 129 | mock_log.assert_called_with("info") 130 | 131 | @patch("runrestic.runrestic.runrestic.restic_check", return_value=True) 132 | @patch("runrestic.runrestic.runrestic.cli_arguments") 133 | @patch("runrestic.runrestic.runrestic.configuration_file_paths", return_value=[]) 134 | @patch( 135 | "runrestic.runrestic.runrestic.possible_config_paths", 136 | return_value=["/etc/restic", "~/.restic"], 137 | ) 138 | def test_no_config_paths_raises( 139 | self, mock_possible, mock_confpaths, mock_cli, mock_check 140 | ): 141 | args = MagicMock( 142 | log_level="debug", config_file=None, actions=[], show_progress=None 143 | ) 144 | extras: list[str] = [] 145 | mock_cli.return_value = (args, extras) 146 | with self.assertRaises(FileNotFoundError): 147 | runrestic.runrestic() 148 | 149 | @patch("runrestic.runrestic.runrestic.restic_check", return_value=True) 150 | @patch("runrestic.runrestic.runrestic.cli_arguments") 151 | @patch( 152 | "runrestic.runrestic.runrestic.configuration_file_paths", return_value=["cfg1"] 153 | ) 154 | @patch("runrestic.runrestic.runrestic.parse_configuration", return_value={}) 155 | def test_show_progress_sets_env( 156 | self, mock_parse, mock_conf_paths, mock_cli, mock_check 157 | ): 158 | args = MagicMock( 159 | log_level="info", config_file=None, actions=[], show_progress="0.5" 160 | ) 161 | extras: list[str] = [] 162 | mock_cli.return_value = (args, extras) 163 | 164 | runrestic.runrestic() 165 | self.assertIn("RESTIC_PROGRESS_FPS", os.environ) 166 | self.assertEqual(os.environ["RESTIC_PROGRESS_FPS"], str(1 / float("0.5"))) 167 | 168 | @patch("runrestic.runrestic.runrestic.restic_check", return_value=True) 169 | @patch("runrestic.runrestic.runrestic.cli_arguments") 170 | @patch( 171 | "runrestic.runrestic.runrestic.configuration_file_paths", return_value=["cfg1"] 172 | ) 173 | @patch( 174 | "runrestic.runrestic.runrestic.parse_configuration", 175 | return_value={"repositories": ["dummy"], "name": "dummy", "environment": {}}, 176 | ) 177 | @patch("runrestic.runrestic.runrestic.restic_shell") 178 | def test_shell_action_invokes_shell( 179 | self, mock_shell, mock_parse, mock_confpaths, mock_cli, mock_check 180 | ): 181 | with patch( 182 | "runrestic.runrestic.runrestic.logging.getLogger" 183 | ) as _mock_get_logger: 184 | _mock_logger = MagicMock() 185 | args = MagicMock( 186 | log_level="info", 187 | config_file=None, 188 | actions=["shell"], 189 | show_progress=None, 190 | ) 191 | extras: list[str] = [] 192 | mock_cli.return_value = (args, extras) 193 | 194 | result = runrestic.runrestic() # type: ignore[func-returns-value] 195 | mock_shell.assert_called_once_with( 196 | [{"repositories": ["dummy"], "name": "dummy", "environment": {}}] 197 | ) 198 | self.assertIsNone(result) 199 | 200 | @patch("runrestic.runrestic.runrestic.restic_check", return_value=True) 201 | @patch("runrestic.runrestic.runrestic.cli_arguments") 202 | @patch( 203 | "runrestic.runrestic.runrestic.configuration_file_paths", 204 | return_value=["cfg1", "cfg2"], 205 | ) 206 | @patch("runrestic.runrestic.runrestic.parse_configuration", return_value={"a": 1}) 207 | @patch("runrestic.runrestic.runrestic.ResticRunner") 208 | def test_runner_exit_codes( 209 | self, mock_runner_cls, mock_parse, mock_confpaths, mock_cli, mock_check 210 | ): 211 | args = MagicMock( 212 | log_level="info", config_file=None, actions=[], show_progress=None 213 | ) 214 | extras: list[str] = [] 215 | mock_cli.return_value = (args, extras) 216 | 217 | # runner one returns 0, runner two returns 2 -> sum=2 >0 -> sys.exit(1) 218 | runner1 = MagicMock(run=MagicMock(return_value=0)) 219 | runner2 = MagicMock(run=MagicMock(return_value=2)) 220 | mock_runner_cls.side_effect = [runner1, runner2] 221 | 222 | with self.assertRaises(SystemExit) as cm: 223 | runrestic.runrestic() 224 | self.assertEqual(cm.exception.code, 1) 225 | -------------------------------------------------------------------------------- /tests/runrestic/test_runrestic_tools.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from runrestic.runrestic.tools import ( 4 | deep_update, 5 | make_size, 6 | parse_line, 7 | parse_size, 8 | parse_time, 9 | ) 10 | 11 | OUTPUT = """Start of the output 12 | Dummy counter: 123 something 13 | Two counters: value 1: 456, value 2: 7.89 14 | Three counters: value 1: 1.1, value 2: 22, value 3: 33.3 kB 15 | End of output 16 | """ 17 | 18 | OUTPUT_2 = """Start of the output 19 | Two counters: value 1: 456, value 2: 7.89 20 | Three counters: in next line 21 | Three counters: value 1: 1.1, value 2: 22, value 3: 33.3 kB 22 | MORE DATA 23 | Dummy counter: 123 something 24 | End of output 25 | """ 26 | 27 | 28 | def test_make_size(): 29 | assert make_size(100) == "100 B" 30 | assert make_size(1000000) == "976.56 KiB" 31 | assert make_size(1000000000) == "953.67 MiB" 32 | assert make_size(1000000000000) == "931.32 GiB" 33 | assert make_size(1000000000000000) == "909.49 TiB" 34 | with pytest.raises(TypeError): 35 | make_size("string") # type: ignore[arg-type] 36 | 37 | 38 | def test_parse_size(): 39 | assert parse_size("910 TiB") == 1024 * 1024 * 1024 * 1024 * 910 40 | assert parse_size("910 GiB") == 1024 * 1024 * 1024 * 910 41 | assert parse_size("910 MiB") == 1024 * 1024 * 910 42 | assert parse_size("910 KiB") == 1024 * 910 43 | assert parse_size("910 B") == 910 44 | with pytest.raises(TypeError): 45 | parse_size(123) # type: ignore[arg-type] 46 | # Test missing units 47 | assert parse_size("910") == 0.0 48 | 49 | 50 | def test_parse_time(): 51 | assert parse_time("0:50") == 50 52 | assert parse_time("2:20") == 2 * 60 + 20 53 | assert parse_time("2:00:00") == 2 * 60 * 60 54 | assert parse_time("23:59:59") == 24 * 60 * 60 - 1 55 | # Test wrong time format 56 | assert parse_time("42") == 0 57 | 58 | 59 | def test_deep_update(): 60 | new = deep_update({"x": {"y": "z"}, "foo": "bar"}, {"x": {"z": "y"}, "foo": "baz"}) 61 | assert new == {"x": {"y": "z", "z": "y"}, "foo": "baz"} 62 | 63 | 64 | def test_parse_line_match_one(): 65 | assert parse_line(r"Dummy counter: (\d+) something", OUTPUT, "-1") == "123" 66 | assert parse_line(r"Dummy counter: (\d+) something", OUTPUT_2, "-1") == "123" 67 | 68 | 69 | def test_parse_line_no_match_one(): 70 | default = "-1" 71 | assert ( 72 | parse_line(r"Dummy counter NONE: (\d+) something", OUTPUT, default) == default 73 | ) 74 | 75 | 76 | def test_parse_line_match_two(): 77 | assert parse_line( 78 | r"Two counters: value 1: (\d+), value 2: ([\d\.]+)", OUTPUT, ("-1", "-1") 79 | ) == ("456", "7.89") 80 | 81 | 82 | def test_parse_line_no_match_two(): 83 | default = ("0", "0.0") 84 | assert ( 85 | parse_line( 86 | r"Two counters: value 1: (\d+) NO, value 2: ([\d\.]+)", OUTPUT, default 87 | ) 88 | == default 89 | ) 90 | assert ( 91 | parse_line( 92 | r"Two counters: value 1: (\d+) NO, value 2: ([\d\.]+)", OUTPUT_2, default 93 | ) 94 | == default 95 | ) 96 | 97 | 98 | def test_parse_line_match_three(): 99 | assert parse_line( 100 | r"Three counters: value 1: ([\d\.]+), value 2: (\d+), value 3: ([\d\.]+ [kMG]?B)", 101 | OUTPUT, 102 | ("-1", "-1", "-1"), 103 | ) == ("1.1", "22", "33.3 kB") 104 | assert parse_line( 105 | r"Three counters: value 1: ([\d\.]+), value 2: (\d+), value 3: ([\d\.]+ [kMG]?B)", 106 | OUTPUT_2, 107 | ("-1", "-1", "-1"), 108 | ) == ("1.1", "22", "33.3 kB") 109 | 110 | 111 | def test_parse_line_no_match_three(): 112 | assert parse_line( 113 | r"Three counters: value missing: ([\d\.]+), value 2: (\d+), value 3: ([\d\.]+ [kMG]?B)", 114 | OUTPUT, 115 | ("-1", "-1", "-1"), 116 | ) == ("-1", "-1", "-1") 117 | --------------------------------------------------------------------------------