├── rusticlone ├── __init__.py ├── helpers │ ├── __init__.py │ ├── timer.py │ ├── requirements.py │ ├── action.py │ ├── formatting.py │ ├── rustic.py │ ├── notification.py │ ├── rclone.py │ └── custom.py ├── processing │ ├── __init__.py │ ├── atomic.py │ ├── sequential.py │ ├── parallel.py │ └── profile.py └── cli.py ├── example ├── rclone │ └── rclone.conf ├── systemd │ ├── rusticlone.service │ └── rusticlone.timer └── rustic │ ├── Documents.toml │ ├── Media.toml │ └── common.toml ├── images ├── notification.png ├── parallel-backup.png ├── process-backup.png ├── process-restore.png ├── parallel-restore.png ├── sequential-backup.png ├── sequential-restore.png └── coverage.svg ├── .gitignore ├── Makefile ├── .github └── workflows │ ├── from_commit_to_build_test.yml │ └── from_tag_to_build_release_pypi.yml ├── CHANGELOG.md ├── pyproject.toml ├── README.md ├── tests └── tests.sh └── LICENSE /rusticlone/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /rusticlone/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /rusticlone/processing/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /example/rclone/rclone.conf: -------------------------------------------------------------------------------- 1 | [local] 2 | type = local 3 | 4 | -------------------------------------------------------------------------------- /images/notification.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlphaJack/rusticlone/HEAD/images/notification.png -------------------------------------------------------------------------------- /images/parallel-backup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlphaJack/rusticlone/HEAD/images/parallel-backup.png -------------------------------------------------------------------------------- /images/process-backup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlphaJack/rusticlone/HEAD/images/process-backup.png -------------------------------------------------------------------------------- /images/process-restore.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlphaJack/rusticlone/HEAD/images/process-restore.png -------------------------------------------------------------------------------- /images/parallel-restore.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlphaJack/rusticlone/HEAD/images/parallel-restore.png -------------------------------------------------------------------------------- /images/sequential-backup.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlphaJack/rusticlone/HEAD/images/sequential-backup.png -------------------------------------------------------------------------------- /images/sequential-restore.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlphaJack/rusticlone/HEAD/images/sequential-restore.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .venv 2 | build 3 | dist 4 | *egg-info 5 | **/**.pyc 6 | tests/coverage 7 | **/.mypy_cache 8 | **/.coverage 9 | **.kate-swp 10 | -------------------------------------------------------------------------------- /example/systemd/rusticlone.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Rusticlone Backup - service 3 | 4 | [Service] 5 | Type=oneshot 6 | # replace the remote 7 | ExecStart=rusticlone --remote "gdrive:/PC" backup 8 | -------------------------------------------------------------------------------- /example/systemd/rusticlone.timer: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Rusticlone Backup - timer 3 | 4 | [Timer] 5 | # every day at midnight 6 | OnCalendar=*-*-* 00:00:00 7 | 8 | [Install] 9 | WantedBy=timers.target 10 | -------------------------------------------------------------------------------- /example/rustic/Documents.toml: -------------------------------------------------------------------------------- 1 | [global] 2 | use-profiles = ["common"] 3 | 4 | [repository] 5 | repository = "/backup/rusticlone/Documents" 6 | 7 | [[backup.snapshots]] 8 | sources = [ 9 | "/home/jack/Documents", 10 | "/home/jack/Desktop" 11 | ] 12 | -------------------------------------------------------------------------------- /example/rustic/Media.toml: -------------------------------------------------------------------------------- 1 | [global] 2 | use-profiles = ["common"] 3 | 4 | [repository] 5 | repository = "/backup/rusticlone/Media" 6 | 7 | [[backup.snapshots]] 8 | source = ["/home/jack/Pictures"] 9 | 10 | [[backup.snapshots]] 11 | source = ["/home/jack/Videos"] 12 | -------------------------------------------------------------------------------- /example/rustic/common.toml: -------------------------------------------------------------------------------- 1 | [global] 2 | log-level = "debug" 3 | log-file = "/backup/rusticlone/rusticlone.log" 4 | 5 | [global.env] 6 | RCLONE_CONFIG = "/home/jack/.config/rclone/rclone.conf" 7 | RCLONE_CONFIG_PASS = "YYYYYY" 8 | #RCLONE_PASSWORD_COMMAND = "/usr/bin/python -c \"print('YYYYYY')\"" 9 | 10 | [repository] 11 | cache-dir = "/backup/rusticlone/cache" 12 | password = "XXXXXX" 13 | 14 | [backup] 15 | label = "rusticlone" 16 | ignore-inode = true 17 | git-ignore = true 18 | init = true 19 | one-file-system = true 20 | 21 | [forget] 22 | prune = true 23 | keep-last = 1 24 | keep-daily = 7 25 | keep-weekly = 4 26 | keep-monthly = 3 27 | keep-quarter-yearly = 4 28 | keep-yearly = 1 29 | -------------------------------------------------------------------------------- /images/coverage.svg: -------------------------------------------------------------------------------- 1 | coverage: 96.43%coverage96.43% -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # usage: 2 | # make release tag=1.0.0 3 | 4 | #In case a tag has been pushed to GitHub, but the release failed, run ` 5 | # git tag --delete v1.0.0 6 | # git push --delete origin v1.0.0 7 | # and repeat the steps below 8 | 9 | install: 10 | uv sync --all-groups 11 | 12 | lint: 13 | ty check . 14 | ruff check --fix . 15 | ruff format . 16 | 17 | test: 18 | uv run bash tests/tests.sh 19 | 20 | ci: 21 | act --workflows ".github/workflows/from_commit_to_build_test.yml" 22 | 23 | toc: 24 | find * -type f ! -name 'CHANGELOG.md' -exec toc -f {} \; 2>/dev/null 25 | 26 | review: 27 | git status 28 | echo "Abort now if there are files that needs to be committed" 29 | sleep 10 30 | 31 | tag_bump: 32 | grep -q $(tag) pyproject.toml || sed -i pyproject.toml -e "s|version = .*|version = \"$(tag)\"|" 33 | 34 | tag_changelog: 35 | git tag v$(tag) -m v$(tag) 36 | # enter "v1.0.0" 37 | git-cliff -c pyproject.toml > CHANGELOG.md 38 | 39 | tag_commit_new_changelog: 40 | git tag --delete v$(tag) 41 | git add --all || true 42 | git commit -m "minor: release $(tag)" || true 43 | git tag -fa v$(tag) -m v$(tag) 44 | 45 | tag_publish:: 46 | git push --follow-tags 47 | 48 | tag: tag_bump tag_changelog tag_commit_new_changelog tag_publish 49 | 50 | release: lint toc review tag 51 | -------------------------------------------------------------------------------- /rusticlone/helpers/timer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Simple stopwatch to measure time elapsed 3 | """ 4 | 5 | # ┌───────────────────────────────────────────────────────────────┐ 6 | # │ Contents of timer.py │ 7 | # ├───────────────────────────────────────────────────────────────┘ 8 | # │ 9 | # ├── IMPORTS 10 | # ├── CLASSES 11 | # │ 12 | # └─────────────────────────────────────────────────────────────── 13 | 14 | # ################################################################ IMPORTS 15 | 16 | # stopwatch 17 | import time 18 | 19 | # rusticlone 20 | from rusticlone.helpers.formatting import print_stats 21 | 22 | # ################################################################ CLASSES 23 | 24 | 25 | class Timer: 26 | """ 27 | Simple stopwatch to measure time elapsed 28 | """ 29 | 30 | def __init__(self, parallel: bool = False) -> None: 31 | """ 32 | Initializes the timer using the time.perf_counter() function. 33 | """ 34 | self.start_time = time.perf_counter() 35 | self.parallel = parallel 36 | self.stop_time = self.start_time 37 | self.duration = "0s" 38 | 39 | def stop(self, text: str = "Duration") -> None: 40 | """ 41 | Stop the timer and print the result 42 | """ 43 | self.stop_time = time.perf_counter() 44 | self.duration = str(round(self.stop_time - self.start_time)) + "s" 45 | print_stats(text + ":", self.duration, parallel=self.parallel) 46 | -------------------------------------------------------------------------------- /rusticlone/helpers/requirements.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility functions for version checking 3 | """ 4 | 5 | # ┌───────────────────────────────────────────────────────────────┐ 6 | # │ Contents of requirements.py │ 7 | # ├───────────────────────────────────────────────────────────────┘ 8 | # │ 9 | # ├── IMPORTS 10 | # ├── FUNCTIONS 11 | # │ 12 | # └─────────────────────────────────────────────────────────────── 13 | 14 | # ################################################################ IMPORTS 15 | 16 | # rusticlone 17 | from rusticlone.helpers.action import Action 18 | from rusticlone.helpers.rclone import Rclone 19 | from rusticlone.helpers.rustic import Rustic 20 | 21 | # ################################################################ FUNCTIONS 22 | 23 | 24 | def check_rustic_version() -> bool: 25 | """ 26 | Check that the installed Rustic version is supported 27 | """ 28 | action = Action("Checking Rustic version") 29 | rustic = Rustic("", "--version") 30 | version = ( 31 | rustic.stdout.splitlines()[0].replace("rustic", "").replace("v", "").strip() 32 | ) 33 | try: 34 | major_version = int(version.split(".")[0]) 35 | minor_version = int(version.split(".")[1]) 36 | except (ValueError, TypeError): 37 | return action.abort("Could not parse Rustic version") 38 | if major_version != 0 or minor_version != 10: 39 | return action.abort( 40 | f"Rustic {major_version}.{minor_version} is installed, but 0.10 is required" 41 | ) 42 | return action.stop() 43 | 44 | 45 | def check_rclone_version() -> bool: 46 | """ 47 | Check that the installed Rclone version is supported 48 | """ 49 | action = Action("Checking Rclone version") 50 | rclone = Rclone(default_flags=None) 51 | version = ( 52 | rclone.stdout.splitlines()[0].replace("rclone", "").replace("v", "").strip() 53 | ) 54 | try: 55 | major_version = int(version.split(".")[0]) 56 | minor_version = int(version.split(".")[1]) 57 | except (ValueError, TypeError): 58 | return action.abort("Could not parse Rclone version") 59 | if major_version <= 1 and minor_version < 67: 60 | return action.abort( 61 | f"Rclone {major_version}.{minor_version} is installed, but at least 1.67 is required" 62 | ) 63 | return action.stop() 64 | -------------------------------------------------------------------------------- /rusticlone/helpers/action.py: -------------------------------------------------------------------------------- 1 | """ 2 | Print the name of the operation, and is called again in case of success or failure 3 | """ 4 | 5 | # ┌───────────────────────────────────────────────────────────────┐ 6 | # │ Contents of action.py │ 7 | # ├───────────────────────────────────────────────────────────────┘ 8 | # │ 9 | # ├── IMPORTS 10 | # ├── CLASSES 11 | # │ 12 | # └─────────────────────────────────────────────────────────────── 13 | 14 | # ################################################################ IMPORTS 15 | 16 | # rusticlone 17 | from rusticlone.helpers.formatting import clear_line, print_stats 18 | 19 | # ################################################################ CLASSES 20 | 21 | 22 | class Action: 23 | def __init__( 24 | self, 25 | name_start: str, 26 | parallel: bool = False, 27 | status: str = "[W8]", 28 | ) -> None: 29 | """ 30 | Initializes the instance with the provided action_start parameter. 31 | 32 | Parameters: 33 | action_start (str): The action to be assigned to the instance. 34 | 35 | Returns: 36 | None 37 | """ 38 | self.name = name_start 39 | self.parallel = parallel 40 | # print(self.action) 41 | # print_stats(self.action, f'[{blink("W8")}]') 42 | print_stats(self.name, status, parallel=self.parallel) 43 | 44 | def stop(self, name_stop: str = "", status: str = "[OK]") -> bool: 45 | """ 46 | Stop the action and print the status as OK. 47 | 48 | Args: 49 | action_stop (str): The action to stop. 50 | 51 | Returns: 52 | None 53 | """ 54 | if name_stop: 55 | self.name = name_stop 56 | clear_line(parallel=self.parallel) 57 | print_stats(self.name, status, parallel=self.parallel) 58 | return True 59 | 60 | def abort(self, name_abort: str = "Aborting", status: str = "[KO]") -> bool: 61 | """ 62 | A function to abort the program, clearing the line, printing the action to abort, and exiting with status 1. 63 | Parameters: 64 | action_abort (str): The action to abort. 65 | Returns: 66 | None 67 | """ 68 | clear_line(parallel=self.parallel) 69 | print_stats(name_abort, status, parallel=self.parallel) 70 | return False 71 | -------------------------------------------------------------------------------- /.github/workflows/from_commit_to_build_test.yml: -------------------------------------------------------------------------------- 1 | # ┌───────────────────────────────────────────────────────────────┐ 2 | # │ Contents of from_commit_to_build_test.yml │ 3 | # ├───────────────────────────────────────────────────────────────┘ 4 | # │ 5 | # ├──┐From commit 6 | # │ └──┐Build and test 7 | # │ ├── Run as "root" 8 | # │ └── Run as "runner" 9 | # │ 10 | # └─────────────────────────────────────────────────────────────── 11 | 12 | # ################################################################ From commit 13 | 14 | name: Run tests on pushed commits 15 | 16 | # Run on all branches except master 17 | on: 18 | push: 19 | branches: 20 | - '**' 21 | pull_request: 22 | branches: 23 | - master 24 | 25 | permissions: 26 | contents: read 27 | 28 | concurrency: 29 | group: ${{ github.workflow }} 30 | cancel-in-progress: true 31 | 32 | jobs: 33 | 34 | # ################################ Build and test 35 | 36 | test: 37 | name: Test package 38 | timeout-minutes: 10 39 | runs-on: ubuntu-latest 40 | env: 41 | RUSTIC_VERSION: "0.10.0" 42 | steps: 43 | 44 | # ################ Run as "root" 45 | 46 | - name: Install Act dependencies 47 | if: ${{ env.ACT }} 48 | run: apt-get update && apt-get install sudo -y 49 | 50 | - name: Install RClone 51 | run: | 52 | curl -sL https://rclone.org/install.sh | sudo bash 53 | which rclone 54 | rclone --version 55 | 56 | - name: Install rustic 57 | run: | 58 | curl -sL "https://github.com/rustic-rs/rustic/releases/download/v${RUSTIC_VERSION}/rustic-v${RUSTIC_VERSION}-x86_64-unknown-linux-musl.tar.gz" | 59 | sudo tar -xz -C /usr/bin --strip-components=0 rustic 60 | which rustic 61 | rustic --version 62 | 63 | # ################ Run as "runner" 64 | 65 | - name: Install uv 66 | uses: astral-sh/setup-uv@v5 67 | 68 | - name: Access source code 69 | uses: actions/checkout@v4 70 | 71 | - name: Install package and python dependencies 72 | run: uv sync --group full 73 | 74 | - name: Run tests 75 | run: uv run bash tests/tests.sh 76 | 77 | - name: Upload coverage reports 78 | uses: actions/upload-artifact@v4 79 | if: ${{ env.ACT == '' }} 80 | with: 81 | name: coverage-reports 82 | path: /home/runner/.cache/rusticlone-tests/coverage/ 83 | retention-days: 30 84 | -------------------------------------------------------------------------------- /rusticlone/helpers/formatting.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utility functions for text formatting 3 | """ 4 | 5 | # ┌───────────────────────────────────────────────────────────────┐ 6 | # │ Contents of formatting.py │ 7 | # ├───────────────────────────────────────────────────────────────┘ 8 | # │ 9 | # ├── IMPORTS 10 | # ├── FUNCTIONS 11 | # │ 12 | # └─────────────────────────────────────────────────────────────── 13 | 14 | # ################################################################ IMPORTS 15 | 16 | # os 17 | import platform 18 | 19 | # ################################################################ FUNCTIONS 20 | 21 | 22 | def clear_line(n: int = 1, parallel: bool = False) -> None: 23 | """ 24 | Go up and clear line, replacing a "wait" with an "ok" 25 | Not being used on Windows and parallel processing 26 | """ 27 | line_up = "\033[1A" 28 | line_clear = "\x1b[2K" 29 | if platform.system() != "Windows" and not parallel: 30 | for i in range(n): 31 | print(line_up, end=line_clear) 32 | 33 | 34 | def print_stats( 35 | content_left: str, 36 | content_right: str, 37 | width_left: int = 30, 38 | width_right: int = 10, 39 | parallel: bool = False, 40 | ) -> None: 41 | """ 42 | Print left aligned and right aligned text 43 | Since it would mess the output in windows powershell, we use a workaround 44 | """ 45 | # width_left = 80 - len(content_left) 46 | # width_right = 80 - len(content_right) 47 | begin = "" 48 | end = "\n" 49 | if platform.system() == "Windows": 50 | end = "\r" 51 | if content_right != "[OK]": 52 | begin = "\n" 53 | # if not async (global) 54 | if not parallel: 55 | # if True: 56 | if content_right != "": 57 | print( 58 | f"{begin}{content_left:<{width_left}}{content_right:>{width_right}}", 59 | end=end, 60 | ) 61 | else: 62 | print(f"{begin}{content_left}") 63 | 64 | 65 | def convert_size(size: int) -> str: 66 | """ 67 | Convert a size in bytes to a human-readable format (e.g., KB, MB, GB, TB). 68 | """ 69 | sep = "" 70 | # sep = " " 71 | units = ("KB", "MB", "GB", "TB") 72 | size_list = [f"{int(size):,}{sep}B"] + [ 73 | f"{int(size) / 1024 ** (i + 1):,.0f}{sep}{u}" for i, u in enumerate(units) 74 | ] 75 | if size == 0: 76 | return f"0{sep}B" 77 | else: 78 | return [size for size in size_list if not size.startswith("0")][-1] 79 | -------------------------------------------------------------------------------- /rusticlone/helpers/rustic.py: -------------------------------------------------------------------------------- 1 | """ 2 | Launch Rustic binary with passed flags 3 | """ 4 | 5 | # ┌───────────────────────────────────────────────────────────────┐ 6 | # │ Contents of rustic.py │ 7 | # ├───────────────────────────────────────────────────────────────┘ 8 | # │ 9 | # ├── IMPORTS 10 | # ├── CLASSES 11 | # │ 12 | # └─────────────────────────────────────────────────────────────── 13 | 14 | # ################################################################ IMPORTS 15 | 16 | # typing 17 | from typing import Any 18 | 19 | # environment variables like $PATH 20 | from os import environ 21 | 22 | # launch binaries 23 | import subprocess 24 | 25 | # ################################################################ CLASSES 26 | 27 | 28 | class Rustic: 29 | def __init__(self, profile: str, action: str, *args: str, **kwargs): 30 | """ 31 | Launch a rustic command 32 | """ 33 | default_kwargs: dict[str, Any] = { 34 | "env": {}, 35 | } 36 | kwargs = default_kwargs | kwargs 37 | # Copy current environment and override with passed env variables 38 | self.env = environ.copy() 39 | self.env.update(kwargs["env"]) 40 | self.flags = ["--no-progress"] 41 | self.command = [ 42 | "rustic", 43 | *self.flags, 44 | "--use-profile", 45 | profile, 46 | action, 47 | *args, 48 | ] 49 | try: 50 | # stream output in real time 51 | # output_list = [] 52 | # with Popen(self.command, parallel=PIPE) as p: 53 | # while True: 54 | # text = p.stdout.read1().decode("utf-8") 55 | # print(text, end='', flush=True) 56 | # output_list.append(text) 57 | # self.stdout = " ".join(output_list) 58 | # print(self.stdout) 59 | # wait for completion 60 | self.subprocess = subprocess.run( 61 | self.command, check=True, capture_output=True, env=self.env 62 | ) 63 | except FileNotFoundError: 64 | print("Rustic executable not found, are you sure it is installed?") 65 | self.returncode = 1 66 | except subprocess.CalledProcessError as exception: 67 | print("Error args: '" + " ".join(self.command) + "'") 68 | print("Error status: ", exception.returncode) 69 | print(f'Error stderr:"n{exception.stderr.decode("utf-8")}') 70 | self.returncode = 1 71 | else: 72 | self.stdout = self.subprocess.stdout.decode("utf-8") 73 | self.stderr = self.subprocess.stderr.decode("utf-8") 74 | self.returncode = self.subprocess.returncode 75 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog - toc 2 | 3 | ## [1.6.0] - 2025-09-22 4 | ### Added 5 | 6 | - Add auto ignore, add lockfile 7 | - Added rclone flags to speed up upload process 8 | - Added lockfile mechanism 9 | - Automatically ignoring invalid rustic profiles 10 | - Adding macos paths (#2) 11 | 12 | ### Various 13 | 14 | - Merge pull request #3 from AlphaJack/auto-ignore 15 | 16 | ## [1.5.0] - 2025-09-13 17 | ### Added 18 | 19 | - Using forget and prune options from rustic config 20 | - Added support for rustic == 0.10 21 | - Exit on failure, add --version 22 | 23 | ## [1.4.0] - 2024-11-22 24 | ### Added 25 | 26 | - Added support for environment variables 27 | - Improved apprise error messages 28 | - Added support for notifications via apprise 29 | 30 | ### Documentation 31 | 32 | - Added notification documentation and screenshot 33 | - Added coverage badge 34 | 35 | ### Tests 36 | 37 | - Renamed from 'make tests' to 'make test' 38 | 39 | ### Various 40 | 41 | - Merge pull request #1 from AlphaJack/notifications 42 | - Apprise Notifications 43 | 44 | ## [1.3.0] - 2024-10-03 45 | ### Added 46 | 47 | - Added support for rustic == 0.9 48 | 49 | ### Documentation 50 | 51 | - Updated example profiles to rustic 0.9 52 | 53 | ### Tests 54 | 55 | - Updated test profiles to rustic 0.9 56 | 57 | ## [1.2.1] - 2024-09-22 58 | ### Documentation 59 | 60 | - Added known issues on windows 61 | 62 | ### Fixed 63 | 64 | - Removing "v" when checking for compatible rustic and rclone versions 65 | - Creating log file for parallel operations 66 | 67 | ### Tests 68 | 69 | - Added a restore after the final backup 70 | 71 | ## [1.2.0] - 2024-08-28 72 | ### Added 73 | 74 | - Added support for multiple sources per profile 75 | - Added support for rustic >= 0.8 76 | - Parsing rustic toml instead of relying on regex 77 | - Passing all environment variables to rustic and rclone 78 | 79 | ### Changed 80 | 81 | - Requiring python >= 3.11 82 | 83 | ### Documentation 84 | 85 | - Removed known limitations of older Rusticlone versions 86 | 87 | ### Tests 88 | 89 | - Added multiple sources 90 | - Added encrypted rclone config 91 | 92 | ## [1.1.1] - 2024-08-26 93 | ### Changed 94 | 95 | - Supporting only rustic == 0.7, requiring both rustic and rclone to be installed 96 | 97 | ## [1.1.0] - 2024-08-20 98 | ### Added 99 | 100 | - Added rustic and rclone version check 101 | 102 | ### Changed 103 | 104 | - Enforcing minimum rustic and rclone versions 105 | - Supportig rclone >=1.67 106 | 107 | ## [1.0.1] - 2024-05-17 108 | ### Added 109 | 110 | - Added support to backup individual files 111 | 112 | ### Changed 113 | 114 | - Not parsing /etc/rustic if profiles were found in ~/.config/rustic 115 | 116 | ## [1.0.0] - 2024-05-15 117 | ### Various 118 | 119 | - Initial commit 120 | 121 | -------------------------------------------------------------------------------- /rusticlone/helpers/notification.py: -------------------------------------------------------------------------------- 1 | """ 2 | Notify user using apprise 3 | """ 4 | 5 | # ┌───────────────────────────────────────────────────────────────┐ 6 | # │ Contents of notification.py │ 7 | # ├───────────────────────────────────────────────────────────────┘ 8 | # │ 9 | # ├── IMPORTS 10 | # ├── CLASSES 11 | # ├── FUNCTIONS 12 | # │ 13 | # └─────────────────────────────────────────────────────────────── 14 | 15 | # ################################################################ IMPORTS 16 | 17 | # push notifications 18 | try: 19 | from apprise import Apprise 20 | except (ImportError, ModuleNotFoundError): 21 | HAS_APPRISE = False 22 | else: 23 | HAS_APPRISE = True 24 | 25 | # rusticlone 26 | from rusticlone.helpers.formatting import print_stats 27 | 28 | # ################################################################ CLASSES 29 | 30 | 31 | class Result: 32 | """ 33 | Result of an atomic operation 34 | """ 35 | 36 | def __init__( 37 | self, profile: str, operation: str, success: bool, duration: str 38 | ) -> None: 39 | """ 40 | Initialize the result 41 | """ 42 | self.profile = profile 43 | self.operation = operation 44 | self.success = success 45 | self.duration = duration 46 | 47 | 48 | # ################################################################ FUNCTIONS 49 | 50 | 51 | def notify_user(results: dict[str, Result], apprise_url: str) -> None: 52 | """ 53 | Check if apprise is installed 54 | """ 55 | if HAS_APPRISE: 56 | notification = create_notification(results) 57 | send_notification(notification, apprise_url) 58 | else: 59 | print("Please install apprise to send notifications") 60 | 61 | 62 | def create_notification(results: dict[str, Result]) -> str: 63 | """ 64 | Destructure results to create a single notification 65 | """ 66 | lines = [] 67 | for result in results.values(): 68 | status = "✅" if result.success else "🟥" 69 | status = "🟨" if result.duration == "skipped" else status 70 | lines.append( 71 | f"{status} {result.operation} {result.profile} ({result.duration})" 72 | ) 73 | notification = "\n".join(lines) 74 | return notification 75 | 76 | 77 | def send_notification(notification: str, apprise_url: str) -> None: 78 | """ 79 | Send the notification to the notification services using Apprise 80 | """ 81 | dispatcher = Apprise() 82 | if not dispatcher.add(apprise_url): 83 | print(f"Invalid Apprise URL: {apprise_url}") 84 | else: 85 | service = apprise_url.split("://")[0] 86 | if not dispatcher.notify( 87 | title="Rusticlone results:", 88 | body=notification, 89 | ): 90 | print_stats("Notification not sent", f"{service}") 91 | print(f"\n{notification}") 92 | else: 93 | print_stats("Notification sent", f"{service}") 94 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | # ┌───────────────────────────────────────────────────────────────┐ 2 | # │ Contents of pyproject.toml │ 3 | # ├───────────────────────────────────────────────────────────────┘ 4 | # │ 5 | # ├── Build system 6 | # ├── Project 7 | # ├──┐Tools 8 | # │ └── Git-Cliff 9 | # │ 10 | # └─────────────────────────────────────────────────────────────── 11 | 12 | # ################################################################ Build system 13 | 14 | [build-system] 15 | requires = ["setuptools"] 16 | build-backend = "setuptools.build_meta" 17 | 18 | [tool.setuptools] 19 | packages = ["rusticlone", "rusticlone.helpers", "rusticlone.processing"] 20 | 21 | # ################################################################ Project 22 | 23 | [project] 24 | name = "rusticlone" 25 | version = "1.6.0" 26 | authors = [ 27 | { name="AlphaJack" }, 28 | ] 29 | license = "GPL-3.0-or-later" 30 | description = "3-2-1 backups using Rustic and RClone" 31 | readme = "README.md" 32 | requires-python = ">=3.11" 33 | classifiers = [ 34 | "Development Status :: 5 - Production/Stable", 35 | "Intended Audience :: End Users/Desktop", 36 | "Topic :: System :: Archiving :: Backup", 37 | "Environment :: Console", 38 | "Operating System :: OS Independent", 39 | "Programming Language :: Python :: 3" 40 | ] 41 | dependencies = [ 42 | "configargparse", 43 | "importlib-metadata", 44 | ] 45 | 46 | [dependency-groups] 47 | dev = [ 48 | "ty", 49 | "ruff", 50 | "git-cliff", 51 | "coverage", 52 | "genbadge[coverage]", 53 | "tableofcontents", 54 | ] 55 | full = [ 56 | "apprise", 57 | ] 58 | 59 | [project.scripts] 60 | rusticlone = "rusticlone.cli:main" 61 | 62 | [project.urls] 63 | Homepage = "https://github.com/AlphaJack/rusticlone" 64 | Issues = "https://github.com/AlphaJack/rusticlone/issues" 65 | Repository = "https://github.com/AlphaJack/rusticlone" 66 | Changelog = "https://github.com/AlphaJack/rusticlone/blob/master/CHANGELOG.md" 67 | 68 | # ################################################################ Tools 69 | 70 | # ################################ Git-Cliff 71 | 72 | [tool.git-cliff.git] 73 | conventional_commits = false 74 | filter_unconventional = false 75 | protect_breaking_commits = true 76 | filter_commits = false 77 | split_commits = true 78 | tag_pattern = "v[0-9].*" 79 | skip_tags = "beta|alpha" 80 | ignore_tags = "rc" 81 | sort_commits = "newest" 82 | commit_parsers = [ 83 | { message = "^(feat|[Aa]dd)", group = "Added" }, 84 | { message = "^perf", group = "Performance" }, 85 | { message = "^change", group = "Changed" }, 86 | { message = "^[Dd]oc", group = "Documentation" }, 87 | { message = "^deprecat", group = "Deprecated" }, 88 | { message = "^fix", group = "Fixed" }, 89 | { message = "^remove", group = "Removed" }, 90 | { body = ".*security", group = "Security" }, 91 | { message = "^test", group = "Tests" }, 92 | { message = "^(auto|ci|chore|minor|skip)", skip = true }, 93 | { body = ".*", group = "Various" }, 94 | ] 95 | 96 | [tool.git-cliff.changelog] 97 | trim = true 98 | header = """# Changelog - toc 99 | 100 | """ 101 | body = """ 102 | {% if version %}\ 103 | ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}\ 104 | 105 | {% else %}\ 106 | ## Work in progress\ 107 | 108 | {% endif %}\ 109 | {% for group, commits in commits | group_by(attribute="group") %} 110 | ### {{ group | upper_first }} 111 | 112 | {% for commit in commits %}\ 113 | - {{ commit.message | split(pat=':') | last | trim | upper_first }} 114 | {% endfor %}\ 115 | 116 | {% endfor %}\n 117 | """ 118 | 119 | -------------------------------------------------------------------------------- /rusticlone/cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # ┌───────────────────────────────────────────────────────────────┐ 4 | # │ Contents of cli.py │ 5 | # ├───────────────────────────────────────────────────────────────┘ 6 | # │ 7 | # ├── IMPORTS 8 | # ├── FUNCTIONS 9 | # ├── ENTRY POINT 10 | # │ 11 | # └─────────────────────────────────────────────────────────────── 12 | 13 | """ 14 | CLI script for Rusticlone 15 | """ 16 | 17 | # ################################################################ IMPORTS 18 | 19 | # accept arguments 20 | import configargparse 21 | 22 | # exit on failure 23 | import sys 24 | 25 | # version 26 | from importlib_metadata import version 27 | 28 | # rusticlone 29 | from rusticlone.helpers.custom import load_customizations 30 | from rusticlone.helpers.requirements import check_rustic_version, check_rclone_version 31 | 32 | # ################################################################ FUNCTIONS 33 | 34 | 35 | def parse_args(): 36 | """ 37 | Parse the command-line arguments and return the parsed arguments 38 | """ 39 | parser = configargparse.ArgumentParser( 40 | prog="rusticlone", 41 | description="3-2-1 backups using Rustic and RClone", 42 | ) 43 | # command = parser.add_mutually_exclusive_group(required=True) 44 | parser.add_argument( 45 | "command", 46 | help="backup (archive + upload) or restore (download + extract)", 47 | nargs=1, 48 | choices=("archive", "upload", "backup", "download", "extract", "restore"), 49 | ) 50 | parser.add_argument( 51 | "-a", 52 | "--apprise-url", 53 | type=str, 54 | env_var="APPRISE_URL", 55 | help="Apprise URL for notification", 56 | ) 57 | parser.add_argument( 58 | "-i", 59 | "--ignore", 60 | type=str, 61 | default="", 62 | help="Deprecated argument, does nothing. Will be removed in a future release", 63 | ) 64 | parser.add_argument( 65 | "-l", 66 | "--log-file", 67 | type=str, 68 | env_var="LOG_FILE", 69 | help="Log file for Rustic and RClone", 70 | ) 71 | parser.add_argument( 72 | "-p", 73 | "--parallel", 74 | action="store_true", 75 | env_var="PARALLEL", 76 | help="Process profiles in parallel", 77 | ) 78 | group = parser.add_mutually_exclusive_group() 79 | group.add_argument( 80 | "-P", 81 | "--profile", 82 | type=str, 83 | default="*", 84 | env_var="RUSTIC_PROFILE", 85 | help="Individual Rustic profile to process", 86 | ) 87 | parser.add_argument( 88 | "-r", 89 | "--remote", 90 | type=str, 91 | env_var="RCLONE_REMOTE", 92 | help="RClone remote and subdirectory", 93 | ) 94 | parser.add_argument( 95 | "-v", 96 | "--version", 97 | action="version", 98 | version="%(prog)s " + version("rusticlone"), 99 | help="Show the current version", 100 | ) 101 | args = parser.parse_args() 102 | # https://stackoverflow.com/a/19414853/13448666 103 | if args.remote is None and args.command[0] in [ 104 | "backup", 105 | "restore", 106 | "upload", 107 | "download", 108 | ]: 109 | parser.error(args.command[0] + " requires --remote") 110 | return args 111 | 112 | 113 | def main(): 114 | """ 115 | parse arguments, check log file, get a list of profiles and process them 116 | """ 117 | # parse arguments 118 | # print(sys.argv) 119 | args = parse_args() 120 | if check_rustic_version() and check_rclone_version(): 121 | load_customizations(args) 122 | else: 123 | sys.exit(1) 124 | 125 | 126 | # ################################################################ ENTRY POINT 127 | 128 | if __name__ == "__main__": 129 | main() 130 | -------------------------------------------------------------------------------- /rusticlone/helpers/rclone.py: -------------------------------------------------------------------------------- 1 | """ 2 | Launch RClone binary with passed flags 3 | """ 4 | 5 | # ┌───────────────────────────────────────────────────────────────┐ 6 | # │ Contents of rclone.py │ 7 | # ├───────────────────────────────────────────────────────────────┘ 8 | # │ 9 | # ├── IMPORTS 10 | # ├── CLASSES 11 | # │ 12 | # └─────────────────────────────────────────────────────────────── 13 | 14 | # ################################################################ IMPORTS 15 | 16 | # typing 17 | from typing import Any 18 | 19 | # environment variables like $PATH 20 | from os import environ 21 | 22 | # launch binaries 23 | import subprocess 24 | 25 | # ################################################################ CLASSES 26 | 27 | 28 | class Rclone: 29 | def __init__(self, **kwargs): 30 | """ 31 | Launch a rclone command 32 | """ 33 | default_flags = [ 34 | "--auto-confirm", 35 | "--ask-password=false", 36 | "--check-first", 37 | "--cutoff-mode=hard", 38 | "--delete-during", 39 | "--fast-list", 40 | "--links", 41 | "--human-readable", 42 | "--stats-one-line", 43 | "--transfers=10", 44 | "--verbose", 45 | "--crypt-server-side-across-configs", 46 | "--onedrive-server-side-across-configs", 47 | "--drive-server-side-across-configs", 48 | "--drive-chunk-size=128M", 49 | "--drive-acknowledge-abuse", 50 | "--drive-stop-on-upload-limit", 51 | "--no-update-modtime", 52 | "--no-update-dir-modtime", 53 | ] 54 | default_kwargs: dict[str, Any] = { 55 | "env": {}, 56 | "check_return_code": True, 57 | "action": "version", 58 | "default_flags": default_flags, 59 | "additional_flags": [], 60 | "origin": None, 61 | "destination": None, 62 | } 63 | kwargs = default_kwargs | kwargs 64 | # Copy current environment and override with passed env variables 65 | self.env = environ.copy() 66 | self.env.update(kwargs["env"]) 67 | self.check_return_code = kwargs["check_return_code"] 68 | self.action = kwargs["action"] 69 | self.origin = kwargs["origin"] 70 | self.destination = kwargs["destination"] 71 | if kwargs["default_flags"]: 72 | self.flags = [ 73 | *kwargs["default_flags"], 74 | f"--log-file={kwargs['log_file']}", 75 | # f"--config={kwargs['config']}", 76 | # f"--password-command=\"echo '{kwargs['config_pass']}'\"", 77 | *kwargs["additional_flags"], 78 | ] 79 | else: 80 | self.flags = [] 81 | self.command_entries = [ 82 | "rclone", 83 | *self.flags, 84 | self.action, 85 | self.origin, 86 | self.destination, 87 | ] 88 | self.command = [_ for _ in self.command_entries if _ is not None] 89 | 90 | try: 91 | self.subprocess = subprocess.run( 92 | self.command, 93 | check=self.check_return_code, 94 | capture_output=True, 95 | env=self.env, 96 | ) 97 | except FileNotFoundError: 98 | print("RClone executable not found, are you sure it is installed?") 99 | self.returncode = 1 100 | except subprocess.CalledProcessError as exception: 101 | print("Error env: '" + str(self.env)) 102 | print("Error args: '" + " ".join(self.command) + "'") 103 | print("Error status: ", exception.returncode) 104 | print(f"Error stderr:\n{exception.stderr.decode('utf-8')}") 105 | print("") 106 | self.returncode = 1 107 | else: 108 | self.stdout = self.subprocess.stdout.decode("utf-8") 109 | self.stderr = self.subprocess.stderr.decode("utf-8") 110 | self.returncode = self.subprocess.returncode 111 | -------------------------------------------------------------------------------- /rusticlone/processing/atomic.py: -------------------------------------------------------------------------------- 1 | """ 2 | Define which tasks are needed for archive, upload, download and extract operations 3 | """ 4 | 5 | # ┌───────────────────────────────────────────────────────────────┐ 6 | # │ Contents of atomic.py │ 7 | # ├───────────────────────────────────────────────────────────────┘ 8 | # │ 9 | # ├── IMPORTS 10 | # ├──┐FUNCTIONS 11 | # │ ├── BACKUP 12 | # │ └── RESTORE 13 | # │ 14 | # └─────────────────────────────────────────────────────────────── 15 | 16 | # ################################################################ IMPORTS 17 | 18 | # file locations 19 | from pathlib import Path 20 | 21 | # rusticlone 22 | from rusticlone.helpers.action import Action 23 | from rusticlone.helpers.timer import Timer 24 | from rusticlone.processing.profile import Profile 25 | 26 | # ################################################################ FUNCTIONS 27 | # ################################ BACKUP 28 | 29 | 30 | def profile_archive( 31 | name: str, log_file: Path, parallel: bool = False 32 | ) -> tuple[bool, str]: 33 | """ 34 | Create a snapshot of a profile in a local rustic repo 35 | """ 36 | Action(name, parallel, "[archive]") 37 | timer = Timer(parallel) 38 | profile = Profile(name, parallel) 39 | profile.parse_rustic_config() 40 | profile.set_log_file(log_file) 41 | profile.check_sources_exist() 42 | profile.check_local_repo_exists() 43 | profile.create_lockfile("archive") 44 | profile.check_local_repo_health() 45 | profile.init() 46 | profile.backup() 47 | profile.forget() 48 | profile.source_stats() 49 | profile.repo_stats() 50 | profile.delete_lockfile() 51 | timer.stop() 52 | # action.stop(" ", "") 53 | return profile.result, timer.duration 54 | 55 | 56 | def profile_upload( 57 | name: str, log_file: Path, remote_prefix: str, parallel: bool = False 58 | ) -> tuple[bool, str]: 59 | """ 60 | Sync the local rustic repo of a profile to a RClone remote 61 | """ 62 | Action(name, parallel, "[upload]") 63 | timer = Timer(parallel) 64 | profile = Profile(name, parallel) 65 | profile.parse_rustic_config() 66 | profile.set_log_file(log_file) 67 | profile.check_rclone_config_exists() 68 | profile.check_local_repo_exists() 69 | profile.create_lockfile("upload") 70 | profile.upload(remote_prefix) 71 | profile.delete_lockfile() 72 | timer.stop() 73 | # action.stop(" ", "") 74 | return profile.result, timer.duration 75 | 76 | 77 | # ################################ RESTORE 78 | 79 | 80 | def profile_download( 81 | name: str, log_file: Path, remote_prefix: str, parallel: bool = False 82 | ) -> tuple[bool, str]: 83 | """ 84 | Retrieve the RClone remote of a profile to its local rustic repo location 85 | """ 86 | Action(name, parallel, "[download]") 87 | timer = Timer(parallel) 88 | profile = Profile(name, parallel) 89 | profile.parse_rustic_config() 90 | profile.set_log_file(log_file) 91 | profile.check_rclone_config_exists() 92 | profile.check_remote_repo_exists(remote_prefix) 93 | profile.check_local_repo_exists() 94 | profile.create_lockfile("download") 95 | profile.download(remote_prefix) 96 | profile.delete_lockfile() 97 | timer.stop() 98 | # print_stats("", "") 99 | return profile.result, timer.duration 100 | 101 | 102 | def profile_extract( 103 | name: str, log_file: Path, parallel: bool = False 104 | ) -> tuple[bool, str]: 105 | """ 106 | Extract the latest snapshot of the local rustic repo of a profile to the source location 107 | """ 108 | Action(name, parallel, "[extract]") 109 | timer = Timer(parallel) 110 | profile = Profile(name, parallel) 111 | profile.parse_rustic_config() 112 | profile.set_log_file(log_file) 113 | profile.check_local_repo_exists() 114 | profile.create_lockfile("extract") 115 | profile.check_latest_snapshot() 116 | profile.check_sources_type() 117 | profile.restore() 118 | profile.delete_lockfile() 119 | timer.stop() 120 | # print_stats("", "") 121 | return profile.result, timer.duration 122 | -------------------------------------------------------------------------------- /.github/workflows/from_tag_to_build_release_pypi.yml: -------------------------------------------------------------------------------- 1 | # ┌───────────────────────────────────────────────────────────────┐ 2 | # │ Contents of from_tag_to_build_release_pypi.yml │ 3 | # ├───────────────────────────────────────────────────────────────┘ 4 | # │ 5 | # ├──┐From tag 6 | # │ └──┐Build and test 7 | # │ ├── Release to GitHub 8 | # │ ├── Deploy to PyPI 9 | # │ └── Deploy to AUR 10 | # │ 11 | # └─────────────────────────────────────────────────────────────── 12 | 13 | # ################################################################ From tag 14 | 15 | name: Create GitHub release and deploy to PyPI from a new tag 16 | 17 | # requirements: 18 | # - GitHub has an environment called "release" 19 | # - PyPI has a trusted publisher set from https://pypi.org/manage/account/publishing/ 20 | # - no need for secret PyPI token 21 | 22 | 23 | # debug: 24 | # git push --delete origin v1.1.0 && git tag --delete v1.1.0 25 | # git -a && git commit --amend 26 | # git push --force 27 | # git tag v1.1.0 && git push --tags 28 | 29 | on: 30 | push: 31 | tags: 32 | - 'v*' 33 | 34 | jobs: 35 | 36 | # ################################ Build and test 37 | 38 | build: 39 | name: Build package 40 | environment: production 41 | runs-on: ubuntu-latest 42 | # needed to push commit to repo 43 | permissions: 44 | contents: write 45 | outputs: 46 | changelog: ${{ steps.changelog.outputs.content }} 47 | steps: 48 | 49 | - name: Access source code 50 | uses: actions/checkout@v4 51 | # needed to list changes 52 | with: 53 | fetch-depth: 0 54 | token: ${{ github.token }} 55 | 56 | - name: Install uv 57 | uses: astral-sh/setup-uv@v5 58 | 59 | - name: Install package and python dependencies 60 | run: uv sync --all-groups 61 | 62 | - name: Store partial changelog for release notes 63 | id: changelog 64 | uses: orhun/git-cliff-action@v4.5.1 65 | with: 66 | config: pyproject.toml 67 | args: -vv --latest --strip all 68 | env: 69 | OUTPUT: CHANGELOG-PARTIAL.md 70 | 71 | - name: Build package 72 | run: uv build 73 | 74 | - name: Store package 75 | uses: actions/upload-artifact@v4 76 | with: 77 | name: python-dist 78 | path: dist 79 | 80 | # ################ Release to GitHub 81 | 82 | release: 83 | name: Release to GitHub 84 | needs: build 85 | runs-on: ubuntu-latest 86 | permissions: 87 | contents: write 88 | steps: 89 | 90 | - name: Access package 91 | uses: actions/download-artifact@v4 92 | with: 93 | name: python-dist 94 | path: dist 95 | 96 | # https://raw.githubusercontent.com/orhun/git-cliff/main/.github/workflows/cd.yml 97 | - name: Create release 98 | uses: softprops/action-gh-release@v2 99 | with: 100 | name: Rusticlone ${{ github.ref_name }} 101 | body: ${{ needs.build.outputs.changelog }} 102 | draft: false 103 | token: ${{ github.token }} 104 | fail_on_unmatched_files: true 105 | files: dist/*.whl 106 | 107 | # ################ Deploy to PyPI 108 | 109 | pypi: 110 | name: Deploy to PyPI 111 | needs: build 112 | runs-on: ubuntu-latest 113 | environment: release 114 | permissions: 115 | id-token: write 116 | steps: 117 | 118 | - name: Access package 119 | uses: actions/download-artifact@v4 120 | with: 121 | name: python-dist 122 | path: dist 123 | 124 | - name: Deploy package 125 | uses: pypa/gh-action-pypi-publish@v1.13.0 126 | 127 | # ################ Deploy to AUR 128 | 129 | aur: 130 | name: Deploy to AUR 131 | needs: pypi 132 | runs-on: ubuntu-latest 133 | steps: 134 | 135 | - name: Wait for the new release to be accessible from PyPI 136 | run: sleep 10 137 | 138 | - name: Update PKGBUILD 139 | uses: aksh1618/update-aur-package@v1.0.5 140 | with: 141 | tag_version_prefix: v 142 | package_name: rusticlone 143 | commit_username: "Github Action" 144 | commit_email: alphajack@aur.arch 145 | ssh_private_key: ${{ secrets.AUR_SSH_PRIVATE_KEY }} -------------------------------------------------------------------------------- /rusticlone/processing/sequential.py: -------------------------------------------------------------------------------- 1 | """ 2 | Process all the Rustic profiles in the system one at the time 3 | """ 4 | 5 | # ┌───────────────────────────────────────────────────────────────┐ 6 | # │ Contents of sequential.py │ 7 | # ├───────────────────────────────────────────────────────────────┘ 8 | # │ 9 | # ├── IMPORTS 10 | # ├──┐FUNCTIONS 11 | # │ ├── BACKUP 12 | # │ └── RESTORE 13 | # │ 14 | # └─────────────────────────────────────────────────────────────── 15 | 16 | # ################################################################ IMPORTS 17 | 18 | # file locations 19 | from pathlib import Path 20 | 21 | # rusticlone 22 | from rusticlone.helpers.action import Action 23 | from rusticlone.helpers.timer import Timer 24 | from rusticlone.helpers.notification import Result 25 | from rusticlone.helpers.formatting import print_stats 26 | from rusticlone.processing.atomic import ( 27 | profile_archive, 28 | profile_upload, 29 | profile_download, 30 | profile_extract, 31 | ) 32 | 33 | # ################################################################ FUNCTIONS 34 | # ################################ BACKUP 35 | 36 | 37 | def system_backup_sequential( 38 | profiles: list, log_file: Path, remote_prefix: str 39 | ) -> dict[str, Result]: 40 | """ 41 | Launch system_archive() and system_upload() 42 | """ 43 | # action = Action("Backing up system", status="") 44 | print("========================================") 45 | Action("System", status="[backup]") 46 | timer = Timer() 47 | archive_results = system_archive_sequential(profiles=profiles, log_file=log_file) 48 | upload_results = system_upload_sequential( 49 | profiles=profiles, 50 | log_file=log_file, 51 | remote_prefix=remote_prefix, 52 | archive_results=archive_results, 53 | ) 54 | timer.stop("System backup duration") 55 | return {**archive_results, **upload_results} 56 | 57 | 58 | def system_archive_sequential(profiles: list, log_file: Path) -> dict[str, Result]: 59 | """ 60 | For each profile, archive it 61 | 62 | Args: 63 | profiles (list): List of profiles to be archived 64 | 65 | Returns: 66 | None 67 | """ 68 | # print_stats("Creating local snapshots", "") 69 | print("________________________________________") 70 | print_stats("System", "[archive]") 71 | print_stats("", "") 72 | archive_results = {} 73 | for name in profiles: 74 | archive_success, duration = profile_archive(name=name, log_file=log_file) 75 | if archive_success: 76 | print_stats("", "") 77 | else: 78 | print_stats(f"Error archiving {name}", "") 79 | print_stats("", "") 80 | archive_results[name + "_archive"] = Result( 81 | name, "archive", archive_success, duration 82 | ) 83 | return archive_results 84 | 85 | 86 | def system_upload_sequential( 87 | profiles: list, 88 | log_file: Path, 89 | remote_prefix: str, 90 | archive_results: dict[str, Result] | None = None, 91 | ) -> dict[str, Result]: 92 | """ 93 | For each profile, upload it 94 | 95 | Args: 96 | profiles (list): List of profiles to be uploaded 97 | log_file (Path): log file for Rclone 98 | remote_prefix (str): prefix of Rclone remote 99 | 100 | Returns: 101 | None 102 | """ 103 | # print_stats("Uploading local snapshots", "") 104 | print("________________________________________") 105 | print_stats("System", "[upload]") 106 | print_stats("", "") 107 | upload_results = {} 108 | if archive_results is None: 109 | archive_results = {} 110 | for name in profiles: 111 | archive_success = True 112 | archive_result = archive_results.get(name + "_archive", None) 113 | if archive_result is not None: 114 | archive_success = archive_result.success 115 | if archive_success: 116 | upload_success, duration = profile_upload( 117 | name=name, log_file=log_file, remote_prefix=remote_prefix 118 | ) 119 | if upload_success: 120 | print_stats("", "") 121 | else: 122 | print_stats(f"Error uploading {name}", "") 123 | upload_results[name + "_upload"] = Result( 124 | name, "upload", upload_success, duration 125 | ) 126 | else: 127 | print_stats(f"Not uploading {name} as archiving failed", "") 128 | print_stats("", "") 129 | upload_results[name + "_upload"] = Result(name, "upload", False, "skipped") 130 | return upload_results 131 | 132 | 133 | # ################################ RESTORE 134 | 135 | 136 | def system_restore_sequential( 137 | profiles: list, log_file: Path, remote_prefix: str 138 | ) -> dict[str, Result]: 139 | """ 140 | Launch system_download() and system_extract() 141 | """ 142 | # action = Action("Restoring system", status="") 143 | print("========================================") 144 | Action("System", False, "[restore]") 145 | timer = Timer() 146 | download_results = system_download_sequential( 147 | profiles, log_file=log_file, remote_prefix=remote_prefix 148 | ) 149 | extract_results = system_extract_sequential( 150 | profiles=profiles, 151 | log_file=log_file, 152 | download_results=download_results, 153 | ) 154 | timer.stop("System restore duration") 155 | return {**download_results, **extract_results} 156 | 157 | 158 | def system_download_sequential( 159 | profiles: list, log_file: Path, remote_prefix: str 160 | ) -> dict[str, Result]: 161 | """ 162 | For each profile, download it to the repo location 163 | 164 | Args: 165 | profiles (list): List of profiles to be uploaded 166 | log_file (Path): log file for Rclone 167 | remote_prefix (str): prefix of Rclone remote 168 | 169 | Returns: 170 | None 171 | """ 172 | # print_stats("Downloading remote snapshots", "") 173 | print("________________________________________") 174 | print_stats("System", "[download]") 175 | print_stats("", "") 176 | download_results = {} 177 | for name in profiles: 178 | download_success, duration = profile_download( 179 | name=name, log_file=log_file, remote_prefix=remote_prefix 180 | ) 181 | if download_success: 182 | print_stats("", "") 183 | else: 184 | print_stats(f"Error downloading {name}", "") 185 | download_results[name + "_download"] = Result( 186 | name, "download", download_success, duration 187 | ) 188 | return download_results 189 | 190 | 191 | def system_extract_sequential( 192 | profiles: list, 193 | log_file: Path, 194 | download_results: dict[str, Result] | None = None, 195 | ) -> dict[str, Result]: 196 | """ 197 | For each profile, 198 | extract it to a specific folder, 199 | move the extracted snapshot to root location 200 | """ 201 | # print_stats("Extracting local snapshots", "") 202 | print("________________________________________") 203 | print_stats("System", "[extract]") 204 | print_stats("", "") 205 | if download_results is None: 206 | download_results = {} 207 | extract_results = {} 208 | for name in profiles: 209 | download_success = True 210 | download_result = download_results.get(name + "_download", None) 211 | if download_result is not None: 212 | download_success = download_result.success 213 | if download_success: 214 | extract_success, duration = profile_extract(name=name, log_file=log_file) 215 | if extract_success: 216 | print_stats("", "") 217 | else: 218 | print_stats(f"Error extracting {name}", "") 219 | extract_results[name + "_extract"] = Result( 220 | name, "extract", extract_success, duration 221 | ) 222 | else: 223 | print_stats(f"Not extracting {name} as download failed", "") 224 | extract_results[name + "_extract"] = Result( 225 | name, "extract", False, "skipped" 226 | ) 227 | return extract_results 228 | -------------------------------------------------------------------------------- /rusticlone/helpers/custom.py: -------------------------------------------------------------------------------- 1 | """ 2 | Load customizations by interpreting passed arguments 3 | """ 4 | 5 | # ┌───────────────────────────────────────────────────────────────┐ 6 | # │ Contents of custom.py │ 7 | # ├───────────────────────────────────────────────────────────────┘ 8 | # │ 9 | # ├── IMPORTS 10 | # ├── CLASSES 11 | # ├── FUNCTIONS 12 | # │ 13 | # └─────────────────────────────────────────────────────────────── 14 | 15 | # ################################################################ IMPORTS 16 | 17 | # file locations 18 | from pathlib import Path 19 | 20 | # args type 21 | from configargparse import Namespace 22 | 23 | # os and hostname 24 | import platform 25 | 26 | # exit 27 | import sys 28 | 29 | # toml parsing 30 | import tomllib 31 | 32 | # rusticlone 33 | from rusticlone.helpers.action import Action 34 | from rusticlone.helpers.rustic import Rustic 35 | from rusticlone.helpers.notification import notify_user 36 | from rusticlone.processing.parallel import ( 37 | system_backup_parallel, 38 | system_archive_parallel, 39 | system_upload_parallel, 40 | system_restore_parallel, 41 | system_download_parallel, 42 | system_extract_parallel, 43 | ) 44 | from rusticlone.processing.sequential import ( 45 | system_backup_sequential, 46 | system_archive_sequential, 47 | system_upload_sequential, 48 | system_restore_sequential, 49 | system_download_sequential, 50 | system_extract_sequential, 51 | ) 52 | from rusticlone.processing.profile import parse_repo, parse_sources 53 | 54 | # ################################################################ CLASSES 55 | 56 | 57 | class Custom: 58 | """ 59 | Set variables to args flags, or use hardcoded values 60 | """ 61 | 62 | def __init__(self, args) -> None: 63 | """ 64 | Initialize variables 65 | """ 66 | self.hostname = platform.node() 67 | self.parallel = args.parallel 68 | self.command = args.command[0] 69 | self.operating_system = platform.system() 70 | self.default_log_file = Path("rusticlone.log") 71 | self.apprise_url = "" 72 | # log file 73 | # rustic use log file from config, while from rclone it is passed from either cli or here 74 | if args.log_file is not None: 75 | self.log_file = Path(args.log_file) 76 | else: 77 | self.log_file = self.default_log_file 78 | # profiles dirs, the same where rustic reads profiles 79 | if self.operating_system == "Windows": 80 | self.profiles_dirs = [ 81 | Path.home() / "AppData/Roaming/rustic/config", 82 | Path("C:/ProgramData/rustic/config"), 83 | ] 84 | elif self.operating_system == "Darwin": 85 | self.profiles_dirs = [ 86 | Path.home() / "Library/Application Support/rustic", 87 | Path("/etc/rustic"), 88 | ] 89 | else: 90 | self.profiles_dirs = [Path.home() / ".config/rustic", Path("/etc/rustic")] 91 | # remote prefix: rclone remote + subdirectory without trailing slash 92 | if args.remote is not None: 93 | self.remote_prefix = args.remote.rstrip("/") 94 | else: 95 | self.remote_prefix = "None:/" 96 | # test profile 97 | if args.profile: 98 | self.provided_profile = args.profile 99 | else: 100 | self.provided_profile = "" 101 | if args.apprise_url: 102 | self.apprise_url = args.apprise_url 103 | 104 | def check_log_file(self) -> None: 105 | """ 106 | Create log file parent folders if missing, delete old log file 107 | """ 108 | action = Action("Checking log file") 109 | if self.log_file != self.default_log_file: 110 | used_log = str(self.log_file) 111 | self.log_file.parents[0].mkdir(parents=True, exist_ok=True) 112 | self.log_file.unlink(missing_ok=True) 113 | self.log_file.touch() 114 | else: 115 | # if not defined in Rustic profiles, "./rusticlone.log" is used 116 | used_log = "defined in Rustic profiles" 117 | action.stop(f"Log file: {used_log}", "") 118 | 119 | def check_profiles_dirs(self) -> None: 120 | """ 121 | Check that at least one profiles directory exists 122 | """ 123 | action = Action("Checking profiles directory") 124 | existing_dirs = [] 125 | old_profiles_dirs = self.profiles_dirs 126 | for profile_dir in self.profiles_dirs: 127 | if profile_dir.exists() and profile_dir.is_dir(): 128 | existing_dirs.append(profile_dir) 129 | if existing_dirs: 130 | self.profiles_dirs = existing_dirs 131 | pretty_dirs = str( 132 | [str(profiles_dir) for profiles_dir in self.profiles_dirs] 133 | ) 134 | action.stop(f"Profiles directories: {pretty_dirs}", "") 135 | else: 136 | action.abort( 137 | f"Could not find any profiles directory among {old_profiles_dirs}" 138 | ) 139 | 140 | 141 | # ################################################################ FUNCTIONS 142 | 143 | 144 | def has_needed_config_components(profile_path: Path) -> bool: 145 | """ 146 | Check if a profile should be processed by running rustic show-config 147 | """ 148 | try: 149 | profile_name = profile_path.stem 150 | rustic = Rustic(profile_name, "show-config") 151 | config = tomllib.loads(rustic.stdout) 152 | has_repo = parse_repo(config) is not None 153 | has_sources = parse_sources(config) is not None 154 | return has_repo and has_sources 155 | except (AttributeError, tomllib.TOMLDecodeError, KeyError, IndexError): 156 | return False 157 | 158 | 159 | def list_profiles(profiles_dirs: list, provided_profile: str = "*") -> list: 160 | """ 161 | Scan profiles from directories if none have been provided explicitely 162 | Don't scan from /etc/rustic if ~/.config/rustic has some profiles' 163 | """ 164 | action = Action("Reading profiles") 165 | profiles: list[str] = [] 166 | opened_files = 0 167 | if not provided_profile: 168 | provided_profile = "*" 169 | for profiles_dir in profiles_dirs: 170 | if not profiles: 171 | action.stop(f'Scanning "{profiles_dir}"', "") 172 | files = sorted(list(profiles_dir.glob(f"{provided_profile}.toml"))) 173 | for file in files: 174 | opened_files += 1 175 | if ( 176 | file.is_file() 177 | and file.stem not in profiles 178 | and has_needed_config_components(file) 179 | ): 180 | profiles.append(file.stem) 181 | # remove duplicates 182 | profiles = list(set(profiles)) 183 | if profiles: 184 | action.stop(f"Profiles: {str(profiles)}", "") 185 | return profiles 186 | else: 187 | print("") 188 | print(f"Glob pattern: {provided_profile}") 189 | print(f"Files tried: {opened_files}") 190 | action.abort("Could not find any valid profile") 191 | sys.exit(1) 192 | 193 | 194 | def process_profiles( 195 | profiles: list, 196 | parallel: bool, 197 | command: str, 198 | log_file: Path, 199 | remote_prefix: str, 200 | apprise_url: str, 201 | ) -> None: 202 | """ 203 | Process all profiles according to the command specified and parallel flag 204 | """ 205 | results = {} 206 | if parallel: 207 | match command: 208 | case "backup": 209 | system_backup_parallel( 210 | profiles=profiles, log_file=log_file, remote_prefix=remote_prefix 211 | ) 212 | case "archive": 213 | system_archive_parallel(profiles=profiles, log_file=log_file) 214 | case "upload": 215 | system_upload_parallel( 216 | profiles=profiles, log_file=log_file, remote_prefix=remote_prefix 217 | ) 218 | case "restore": 219 | system_restore_parallel( 220 | profiles=profiles, 221 | log_file=log_file, 222 | remote_prefix=remote_prefix, 223 | ) 224 | case "download": 225 | system_download_parallel( 226 | profiles=profiles, log_file=log_file, remote_prefix=remote_prefix 227 | ) 228 | case "extract": 229 | system_extract_parallel(profiles=profiles, log_file=log_file) 230 | case _: 231 | print(f"Invalid command '{command}'") 232 | else: 233 | match command: 234 | case "backup": 235 | results = system_backup_sequential( 236 | profiles=profiles, log_file=log_file, remote_prefix=remote_prefix 237 | ) 238 | case "archive": 239 | results = system_archive_sequential( 240 | profiles=profiles, log_file=log_file 241 | ) 242 | case "upload": 243 | results = system_upload_sequential( 244 | profiles=profiles, log_file=log_file, remote_prefix=remote_prefix 245 | ) 246 | case "restore": 247 | results = system_restore_sequential( 248 | profiles=profiles, 249 | log_file=log_file, 250 | remote_prefix=remote_prefix, 251 | ) 252 | case "download": 253 | results = system_download_sequential( 254 | profiles=profiles, log_file=log_file, remote_prefix=remote_prefix 255 | ) 256 | case "extract": 257 | results = system_extract_sequential( 258 | profiles=profiles, log_file=log_file 259 | ) 260 | case _: 261 | print(f"Invalid command '{command}'") 262 | sys.exit(1) 263 | if apprise_url and results: 264 | notify_user(results, apprise_url) 265 | 266 | 267 | def load_customizations(args: Namespace): 268 | """ 269 | Interpret arguments, make a list of profiles to process, and process them 270 | """ 271 | custom = Custom(args) 272 | custom.check_log_file() 273 | custom.check_profiles_dirs() 274 | profiles = list_profiles(custom.profiles_dirs, custom.provided_profile) 275 | process_profiles( 276 | profiles, 277 | custom.parallel, 278 | custom.command, 279 | custom.log_file, 280 | custom.remote_prefix, 281 | custom.apprise_url, 282 | ) 283 | -------------------------------------------------------------------------------- /rusticlone/processing/parallel.py: -------------------------------------------------------------------------------- 1 | """ 2 | Process all the Rustic profiles in the system at the same time, 3 | but archiving before uploading and downloading before extracting 4 | """ 5 | 6 | # ┌───────────────────────────────────────────────────────────────┐ 7 | # │ Contents of parallel.py │ 8 | # ├───────────────────────────────────────────────────────────────┘ 9 | # │ 10 | # ├── IMPORTS 11 | # ├──┐FUNCTIONS 12 | # │ ├── GENERIC 13 | # │ ├── BACKUP 14 | # │ └── RESTORE 15 | # │ 16 | # └─────────────────────────────────────────────────────────────── 17 | 18 | # ################################################################ IMPORTS 19 | 20 | # file locations 21 | from pathlib import Path 22 | 23 | # multithreading 24 | import concurrent.futures 25 | 26 | # rusticlone 27 | from rusticlone.helpers.action import Action 28 | from rusticlone.helpers.timer import Timer 29 | from rusticlone.helpers.formatting import print_stats 30 | from rusticlone.processing.atomic import ( 31 | profile_archive, 32 | profile_upload, 33 | profile_download, 34 | profile_extract, 35 | ) 36 | 37 | # ################################################################ FUNCTIONS 38 | # ################################ GENERIC 39 | 40 | 41 | def duration_parallel(processed_profiles: dict, command: str) -> None: 42 | """ 43 | Print duration for each atomic operation 44 | Cannot use inside system_upload_parallel() and system_extract_parallel() 45 | because they also launch a function if success 46 | """ 47 | for future in concurrent.futures.as_completed(processed_profiles): 48 | name = processed_profiles[future] 49 | try: 50 | success, duration = future.result() 51 | except Exception as exception: 52 | print_stats(f"Error {command} {name}: '{exception}'", "[KO]") 53 | else: 54 | if success: 55 | print_stats(f"{command} {name}", duration) 56 | else: 57 | print_stats(f"Failure {command} {name}", "[KO]") 58 | 59 | 60 | # ################################ BACKUP 61 | 62 | 63 | def system_backup_parallel(profiles: list, log_file: Path, remote_prefix: str) -> None: 64 | """ 65 | start a ThreadPoolExecutor and launch system_archive_parallel() 66 | Once a profile has a been archived, upload it without waiting for others 67 | """ 68 | # action = Action("Backing up system", status="") 69 | Action("System", status="[backup]") 70 | timer = Timer() 71 | with concurrent.futures.ThreadPoolExecutor( 72 | thread_name_prefix="SystemBackup" 73 | ) as executor: 74 | archived_profiles = system_archive_parallel( 75 | profiles=profiles, 76 | log_file=log_file, 77 | executor=executor, 78 | ) 79 | uploaded_profiles = system_upload_parallel( 80 | profiles=profiles, 81 | log_file=log_file, 82 | remote_prefix=remote_prefix, 83 | archived_profiles=archived_profiles, 84 | executor=executor, 85 | ) 86 | duration_parallel(uploaded_profiles, "Uploading") 87 | # print(uploaded_profiles) 88 | # print("DONE!") 89 | timer.stop("System backup duration") 90 | 91 | 92 | def system_archive_parallel(profiles: list, log_file: Path, executor=None) -> dict: 93 | """ 94 | if launched independently, start a ThreadPoolExecutor 95 | For every profile, archive it 96 | """ 97 | archived_profiles = {} 98 | # reuse executor or start a new one if launched independently 99 | if executor is not None: 100 | archived_profiles = { 101 | executor.submit( 102 | profile_archive, name=name, log_file=log_file, parallel=True 103 | ): name 104 | for name in profiles 105 | } 106 | else: 107 | with concurrent.futures.ThreadPoolExecutor( 108 | thread_name_prefix="SystemArchiveOnly" 109 | ) as executor: 110 | Action("System", status="[archive]") 111 | archived_profiles = { 112 | executor.submit( 113 | profile_archive, name=name, log_file=log_file, parallel=True 114 | ): name 115 | for name in profiles 116 | } 117 | duration_parallel(archived_profiles, "Archiving") 118 | return archived_profiles 119 | 120 | 121 | def system_upload_parallel( 122 | profiles: list, 123 | log_file: Path, 124 | remote_prefix: str, 125 | archived_profiles: dict | None = None, 126 | executor=None, 127 | ) -> dict: 128 | """ 129 | if launched independently, start a ThreadPoolExecutor 130 | otherwise check that the archival operation completed without errors 131 | For every profile, upload it it 132 | """ 133 | uploaded_profiles = {} 134 | if archived_profiles is not None and executor is not None: 135 | for future in concurrent.futures.as_completed(archived_profiles): 136 | try: 137 | success, duration = future.result() 138 | name = archived_profiles[future] 139 | except Exception as exception: 140 | print(f"Failure in archiving: '{exception}'") 141 | else: 142 | if success: 143 | uploaded_profiles[ 144 | executor.submit( 145 | profile_upload, 146 | name=name, 147 | log_file=log_file, 148 | remote_prefix=remote_prefix, 149 | parallel=True, 150 | ) 151 | ] = name 152 | print_stats(f"Archiving {name}", duration) 153 | else: 154 | print(f"Not uploading {name} due to failure in archiving") 155 | else: 156 | with concurrent.futures.ThreadPoolExecutor( 157 | thread_name_prefix="SystemUploadOnly" 158 | ) as executor: 159 | Action("System", status="[upload]") 160 | uploaded_profiles = { 161 | executor.submit( 162 | profile_upload, 163 | name=name, 164 | log_file=log_file, 165 | remote_prefix=remote_prefix, 166 | parallel=True, 167 | ): name 168 | for name in profiles 169 | } 170 | return uploaded_profiles 171 | 172 | 173 | # ################################ RESTORE 174 | 175 | 176 | def system_restore_parallel(profiles: list, log_file: Path, remote_prefix: str) -> None: 177 | """ 178 | start a ThreadPoolExecutor and launch system_download_parallel() 179 | Once a profile has a been downloaded, extract it without waiting for others 180 | """ 181 | # action = Action("Restoring system", status="") 182 | Action("System", status="[restore]") 183 | timer = Timer() 184 | with concurrent.futures.ThreadPoolExecutor( 185 | thread_name_prefix="SystemRestore" 186 | ) as executor: 187 | downloaded_profiles = system_download_parallel( 188 | profiles=profiles, 189 | log_file=log_file, 190 | remote_prefix=remote_prefix, 191 | executor=executor, 192 | ) 193 | extracted_profiles = system_extract_parallel( 194 | profiles=profiles, 195 | log_file=log_file, 196 | downloaded_profiles=downloaded_profiles, 197 | executor=executor, 198 | ) 199 | duration_parallel(extracted_profiles, "Extracting") 200 | timer.stop("System restore duration") 201 | 202 | 203 | def system_download_parallel( 204 | profiles: list, 205 | log_file: Path, 206 | remote_prefix: str, 207 | executor=None, 208 | ) -> dict: 209 | """ 210 | if launched independently, start a ThreadPoolExecutor 211 | For every profile, download it 212 | """ 213 | downloaded_profiles = {} 214 | # reuse executor or start a new one if launched independently 215 | if executor is not None: 216 | downloaded_profiles = { 217 | executor.submit( 218 | profile_download, 219 | name=name, 220 | log_file=log_file, 221 | remote_prefix=remote_prefix, 222 | parallel=True, 223 | ): name 224 | for name in profiles 225 | } 226 | else: 227 | with concurrent.futures.ThreadPoolExecutor( 228 | thread_name_prefix="SystemDownloadOnly" 229 | ) as executor: 230 | Action("System", status="[download]") 231 | downloaded_profiles = { 232 | executor.submit( 233 | profile_download, 234 | name=name, 235 | log_file=log_file, 236 | remote_prefix=remote_prefix, 237 | parallel=True, 238 | ): name 239 | for name in profiles 240 | } 241 | duration_parallel(downloaded_profiles, "Downloading") 242 | return downloaded_profiles 243 | 244 | 245 | def system_extract_parallel( 246 | profiles: list, 247 | log_file: Path, 248 | downloaded_profiles: dict | None = None, 249 | executor=None, 250 | ) -> dict: 251 | """ 252 | if launched independently, start a ThreadPoolExecutor 253 | otherwise check that the download operation completed without errors 254 | For every profile, extract it it 255 | """ 256 | extracted_profiles = {} 257 | if downloaded_profiles is not None and executor is not None: 258 | for future in concurrent.futures.as_completed(downloaded_profiles): 259 | try: 260 | success, duration = future.result() 261 | name = downloaded_profiles[future] 262 | except Exception as exception: 263 | print(f"Failure in download: '{exception}'") 264 | else: 265 | if success: 266 | extracted_profiles[ 267 | executor.submit( 268 | profile_extract, 269 | name=name, 270 | log_file=log_file, 271 | parallel=True, 272 | ) 273 | ] = name 274 | print_stats(f"Downloading {name}", duration) 275 | else: 276 | print(f"Not extracting {name} due to failure in download") 277 | else: 278 | with concurrent.futures.ThreadPoolExecutor( 279 | thread_name_prefix="SystemExtractOnly" 280 | ) as executor: 281 | Action("System", status="[extract]") 282 | extracted_profiles = { 283 | executor.submit( 284 | profile_extract, 285 | name=name, 286 | log_file=log_file, 287 | parallel=True, 288 | ): name 289 | for name in profiles 290 | } 291 | return extracted_profiles 292 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 28 | 29 | # Rusticlone 30 | 31 |
32 | PyPI Downloads 33 | Test Coverage 34 |
35 | 36 |

37 | 3-2-1 backups using Rustic and RClone 38 |

39 | 40 |
41 | backup process divided in archive and upload 42 | output of rusticlone backup parallel 43 | output of rusticlone backup sequential 44 |
45 | restore process divided in download and extract 46 | output of rusticlone restore parallel 47 | output of rusticlone restore sequential 48 |
49 | 50 | ## Motivation 51 | 52 | [Rustic](https://rustic.cli.rs/) comes with [native support](https://rustic.cli.rs/docs/commands/init/rclone.html) for [RClone](https://rclone.org/)'s built-in [Restic server](https://rclone.org/commands/rclone_serve_restic/). 53 | After trying this feature, I experienced an abysmally low backup speed, much lower than my upload bandwidth: the bottleneck was the synchronous RClone server, as Rustic was waiting for a response before sending other data. 54 | 55 | Another side effect of this feature is that Rustic does not create a local repo, meaning I would have to restore directly from the cloud in case of a disaster. 56 | 57 | Since I could not run Rustic once for all my profiles (Documents, Pictures, etc.) I came up with a tool to: 58 | 59 | - run Rustic for all my profiles 60 | - archive them to local Rustic repos 61 | - upload local repos to a RClone remote 62 | 63 | When restoring, this tool would first download a copy of the RClone remote, and then restore from local Rustic repos. 64 | By decoupling these operations, I got: 65 | 66 | - three copies of my data, two of which are local and one is remote (3-2-1 backup strategy) 67 | - the bottlenecks are now the SSD speed (for archive and extract operations) and Internet bandwidth (for upload and download operations) 68 | 69 | If it sounds interesting, keep reading! 70 | 71 | ## Installation 72 | 73 | Install [RClone](https://rclone.org/install/) >= 1.67, [Rustic](https://rustic.cli.rs/docs/installation.html) >= 0.10, [Python](https://www.python.org/downloads/) >= 3.11 and then `rusticlone`: 74 | 75 | ```bash 76 | pip install rusticlone 77 | ``` 78 | 79 | [Configure RClone](https://rclone.org/commands/rclone_config/) by adding a remote. 80 | 81 | [Create your Rustic TOML profiles](https://github.com/rustic-rs/rustic/tree/main/config) under "/etc/rustic/" or "$HOME/.config/rustic/" on Linux and MacOS. On Windows, you can put them under "%PROGRAMDATA%/rustic/config" or "%APPDATA%/rustic/config". 82 | Configure your profiles to have one or more sources. 83 | They should also have a local repository destination, without specifying the RClone remote. 84 | You can take inspiration from the profiles in the [example](example/rustic) folder. 85 | 86 | Include variables for the location (and password) of the RClone configuration: 87 | 88 | ```toml 89 | [global.env] 90 | RCLONE_CONFIG = "/home/user/.config/rclone/rclone.conf" 91 | RCLONE_CONFIG_PASS = "XXXXXX" 92 | #escape double quotes inside TOML strings 93 | #RCLONE_PASSWORD_COMMAND = "python -c \"print('YYYYYY')\"" 94 | ``` 95 | 96 | ## Usage 97 | ### Backup 98 | 99 | Let's assume you want to backup your **PC Documents** to both an **external hard drive** (HDD) and **Google Drive**. 100 | 101 | With RClone, you have configured your Google Drive as the remote. 102 | 103 | You have created the "/etc/rustic/Documents.toml" Rustic profile with: 104 | 105 | - source "/home/user/Documents" 106 | - destination "/mnt/backup/Documents" (assuming your external HDD is mounted on "/mnt") 107 | 108 | Launch Rusticlone specifying the RClone remote and the `backup` command: 109 | 110 | ```bash 111 | rusticlone -r "gdrive:/PC" backup 112 | ``` 113 | 114 | Great! You just backed up your documents to both "/mnt/backup/Documents" and ! 115 | 116 | Check the result with the following commands: 117 | 118 | ```bash 119 | #size of all your documents 120 | du -sh "/home/users/Documents" 121 | 122 | #contents of local rustic repo 123 | rustic -P "Documents" repoinfo 124 | tree "/mnt/backup/Documents" 125 | 126 | #contents of remote rustic repo 127 | rclone ncdu "gdrive:/PC/Documents" 128 | ``` 129 | 130 | ### Restore 131 | #### From the local Rustic repo 132 | 133 | In case you lose your PC, but still have your external HDD, on your new PC you need: 134 | 135 | - `rusticlone` and dependencies installed 136 | - your Rustic profiles in place 137 | - your external HDD mounted 138 | 139 | Then, run: 140 | 141 | ```bash 142 | rusticlone extract 143 | ``` 144 | 145 | Great! You just restored your documents from "/mnt/backup/Documents" to "/home/user/Documents". 146 | 147 | #### From the RClone remote 148 | 149 | In case you lose both your PC files and your external HDD, don't worry! You still have your data on the RClone remote. 150 | 151 | On your new PC you need: 152 | 153 | - `rusticlone` and dependencies installed 154 | - your RClone configuration 155 | - your Rustic profiles in place 156 | - a new external HDD mounted 157 | 158 | Then, run: 159 | 160 | ```bash 161 | rusticlone -r "gdrive:/PC" restore 162 | ``` 163 | 164 | Fantastic! You downloaded a copy of your Google Drive backup to the external HDD, 165 | and you restored your documents from the HDD to their original location. 166 | 167 | Check that everything went well: 168 | 169 | ```bash 170 | #your remote backup files are still there 171 | rclone ncdu "gdrive:/PC/Documents" 172 | 173 | #your new external HDD contains a rustic repo 174 | ls -lah "/mnt/backups/Documents" 175 | rustic -P "Documents" repoinfo 176 | 177 | #your documents have been restored 178 | du -sh "/home/users/Documents" 179 | ls -lah "/home/users/Documents" 180 | ``` 181 | 182 | You can now run `rusticlone -r "gdrive:/PC" backup` as always to keep your data safe. 183 | 184 | ### Individual commands 185 | 186 | In alternative to `backup` and `restore`, you can also run individual `rusticlone` commands: 187 | 188 | ```bash 189 | #use rustic from source to local repo 190 | rusticlone archive 191 | 192 | #use rclone from local repo to remote 193 | rusticlone -r "gdrive:/PC" upload 194 | 195 | #use rclone from remote to local repo 196 | rusticlone -r "gdrive:/PC" download 197 | 198 | #use rustic from local repo to source 199 | rusticlone extract 200 | ``` 201 | 202 | ### Push notifications 203 | 204 | Rusticlone can send a push notification with the operation results using Apprise: 205 | 206 | Push Notification 207 | 208 | Just pass the [Apprise notification URL](https://github.com/caronc/apprise?tab=readme-ov-file#supported-notifications) via the `--apprise-url` argument or `APPRISE_URL` environment variable: 209 | 210 | ```bash 211 | rusticlone --apprise-url "tgram:/XXXXXX/YYYYYY/" archive 212 | 213 | #alternative 214 | APPRISE_URL="tgram:/XXXXXX/YYYYYY/" rusticlone archive 215 | ``` 216 | 217 | ### Parallel processing 218 | 219 | You can specify the `--parallel` argument with any command to process all your profiles at the same time: 220 | 221 | ```bash 222 | rusticlone --parallel -r "gdrive:/PC" backup 223 | ``` 224 | 225 | Beware that this may fill your RAM if you have many profiles or several GB of data to archive. 226 | 227 | Parallel processing is also not (yet) compatible with push notifications. 228 | 229 | ### Exclude profiles 230 | 231 | Rustic has a handy feature: you can create additional profiles to store options shared between profiles. 232 | 233 | Let's assume this profile is called "common.toml" and contains the following: 234 | 235 | ```toml 236 | [forget] 237 | prune = true 238 | keep-last = 1 239 | keep-daily = 7 240 | keep-weekly = 4 241 | keep-monthly = 3 242 | keep-quarter-yearly = 4 243 | keep-yearly = 1 244 | ``` 245 | 246 | As it doesn't contain a "\[repository]" section, it will not be treated as a standalone profile by Rusticlone. 247 | 248 | This "common.toml" profile can be referenced from our documents by adding to "Documents.toml" the following: 249 | 250 | ```toml 251 | [global] 252 | use-profile = ["common"] 253 | 254 | # [...] 255 | ``` 256 | 257 | 258 | ### Custom log file 259 | 260 | The default behavior is that, if present, both Rustic and RClone use the log file specified in the Rustic profile configuration. 261 | 262 | A custom log file for both Rustic and RClone can be specified with `--log-file`. 263 | 264 | ```bash 265 | rusticlone --log-file "/var/log/rusticlone.log" archive 266 | ``` 267 | 268 | If no argument is passed and no log file can be found in Rustic configuration, "rusticlone.log" in the current folder is used. 269 | 270 | ### Automatic system backups 271 | 272 | Place your profiles under "/etc/rustic". 273 | If you are storing the RClone password inside the profiles, make sure the folder is only readable by root. 274 | 275 | Create a Systemd timer unit "/etc/systemd/system/rusticlone.timer" and copy inside the following: 276 | 277 | ```ini 278 | [Unit] 279 | Description=Rusticlone timer 280 | 281 | [Timer] 282 | #every day at midnight 283 | OnCalendar=*-*-* 00:00:00 284 | 285 | [Install] 286 | WantedBy=timers.target 287 | ``` 288 | 289 | Create a Systemd service unit "/etc/systemd/system/rusticlone.service" and copy inside the following: 290 | 291 | ```ini 292 | [Unit] 293 | Description=Rusticlone service 294 | 295 | [Service] 296 | Type=oneshot 297 | ExecStart=rusticlone --remote "gdrive:/PC" backup 298 | ``` 299 | 300 | Adjust your `--remote` as needed. 301 | 302 | Apply your changes and enable the timer: 303 | 304 | ```bash 305 | sudo systemctl daemon-reload 306 | sudo systemctl enable --now rusticlone.timer 307 | ``` 308 | 309 | ## Testing 310 | 311 | You can test Rusticlone with dummy files before using it for your precious data: 312 | 313 | ```bash 314 | make install 315 | make test 316 | ``` 317 | 318 | You will need `bash`, `coreutils`, `rclone`, and `rustic` installed to run the test. 319 | Before running the test, make sure that you have no important files under "$HOME/.config/rustic". 320 | 321 | At the end, you can read a test coverage report with your browser, to see which lines of the source code were run during the test. 322 | 323 | ## Known limitations 324 | 325 | - Rustic does not **save ownership and permission** for the source location, but **only for files and folders inside the source**. If you backup "/home/jack" with user "jack" and permission "0700", when you will restore it will have user "root" and permission "0755" ([intended rustic behavior](https://github.com/rustic-rs/rustic/issues/1108#issuecomment-2016584568)) 326 | - Rustic does not recognize proper Windows paths ([bug](https://github.com/rustic-rs/rustic/issues/1104)) 327 | - Rustic will not archive empty folders ([bug](https://github.com/rustic-rs/rustic/issues/1157)) 328 | - Rustic may introduce [breaking changes](https://rustic.cli.rs/docs/breaking_changes.html) 329 | 330 | ## Contribute 331 | 332 | Feel free to open an Issue to report bugs or request new features. 333 | 334 | Pull Requests are welcome, too! 335 | 336 | ## License 337 | 338 | Licensed under [GPL-3.0](LICENSE) terms. 339 | 340 | Not affiliated with Rustic or RClone. 341 | 342 | -------------------------------------------------------------------------------- /tests/tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env uv run bash 2 | 3 | # ┌───────────────────────────────────────────────────────────────┐ 4 | # │ Contents of tests.sh │ 5 | # ├───────────────────────────────────────────────────────────────┘ 6 | # │ 7 | # ├──┐VARIABLES 8 | # │ ├── MAIN 9 | # │ └── DERIVED 10 | # ├──┐FUNCTIONS 11 | # │ ├── PREPARATION 12 | # │ ├──┐RUSTICLONE BACKUP 13 | # │ │ ├── SEQUENTIAL 14 | # │ │ ├── PARALLEL 15 | # │ │ └── BACKGROUND 16 | # │ ├──┐DISASTER SIMULATION 17 | # │ │ ├── LOSING SOURCE FILES 18 | # │ │ ├── LOSING CACHE 19 | # │ │ ├── LOSING LOCAL BACKUP 20 | # │ │ └── LOSING REMOTE BACKUP 21 | # │ ├──┐RUSTICLONE RESTORE 22 | # │ │ ├── SEQUENTIAL 23 | # │ │ ├── PARALLEL 24 | # │ │ └── BACKGROUND 25 | # │ ├── RESULT 26 | # │ └── MAIN 27 | # ├── COMMANDS 28 | # │ 29 | # └─────────────────────────────────────────────────────────────── 30 | 31 | set -euo pipefail 32 | 33 | # ################################################################ VARIABLES 34 | # ################################ MAIN 35 | 36 | # must either be ~/.config/rustic or /etc/rustic 37 | RUSTIC_PROFILES_DIR="$HOME/.config/rustic" 38 | 39 | # can be any folder 40 | RUSTICLONE_TEST_DIR="$HOME/.cache/rusticlone-tests" 41 | 42 | # ################################ DERIVED 43 | 44 | mapfile -d '' profile1Content << CONTENT 45 | [global] 46 | use-profiles = ["common"] 47 | 48 | [repository] 49 | repository = "$RUSTICLONE_TEST_DIR/local/Documents" 50 | cache-dir = "$RUSTICLONE_TEST_DIR/cache" 51 | password = "XXXXXX" 52 | 53 | [[backup.snapshots]] 54 | sources = ["$RUSTICLONE_TEST_DIR/source/docs"] 55 | 56 | [forget] 57 | prune = true 58 | keep-last = 1 59 | 60 | [global.env] 61 | RCLONE_CONFIG = "$RUSTIC_PROFILES_DIR/rclone-decrypted.conf" 62 | CONTENT 63 | 64 | mapfile -d '' profile2Content << CONTENT 65 | [repository] 66 | repository = "$RUSTICLONE_TEST_DIR/local/Pictures" 67 | cache-dir = "$RUSTICLONE_TEST_DIR/cache" 68 | password = "XXXXXX" 69 | 70 | [[backup.snapshots]] 71 | sources = ["$RUSTICLONE_TEST_DIR/source/pics"] 72 | 73 | [[backup.snapshots]] 74 | sources = ["$RUSTICLONE_TEST_DIR/source/photos"] 75 | 76 | [forget] 77 | keep-daily = 10 78 | 79 | [global.env] 80 | RCLONE_CONFIG = "$RUSTIC_PROFILES_DIR/rclone-encrypted.conf" 81 | RCLONE_CONFIG_PASS = "YYYYYY" 82 | CONTENT 83 | 84 | mapfile -d '' profile3Content << CONTENT 85 | [repository] 86 | repository = "$RUSTICLONE_TEST_DIR/local/Passwords" 87 | cache-dir = "$RUSTICLONE_TEST_DIR/cache" 88 | password = "XXXXXX" 89 | 90 | [[backup.snapshots]] 91 | sources = [ 92 | "$RUSTICLONE_TEST_DIR/source/passwords.kdbx", 93 | "$RUSTICLONE_TEST_DIR/source/secrets" 94 | ] 95 | 96 | [global.env] 97 | RCLONE_CONFIG = "$RUSTIC_PROFILES_DIR/rclone-encrypted.conf" 98 | RCLONE_PASSWORD_COMMAND = "python -c \"print('YYYYYY')\"" 99 | CONTENT 100 | 101 | mapfile -d '' profileCommonContent << CONTENT 102 | [global] 103 | log-level = "debug" 104 | log-file = "$RUSTICLONE_TEST_DIR/logs/rusticlone.log" 105 | CONTENT 106 | 107 | mapfile -d '' rcloneConfContentDecrypted << CONTENT 108 | [gdrive] 109 | type = local 110 | CONTENT 111 | 112 | mapfile -d '' rcloneConfContentEncrypted << CONTENT 113 | # Encrypted rclone configuration File 114 | 115 | RCLONE_ENCRYPT_V0: 116 | LDDUg4mDyUxDwMtntnCaiUN+o9SexiohA8Y74ZYJmPD9KD8UjVtH9XYCL+3A6OGR7msabjvu0Gj2W8JRande 117 | CONTENT 118 | 119 | GOOD_APPRISE_URL="form://example.org" 120 | BAD_APPRISE_URL="moz://a" 121 | 122 | # ################################################################ FUNCTIONS 123 | # ################################ PREPARATION 124 | 125 | stopwatch_begin(){ 126 | datebegin="$EPOCHSECONDS" 127 | export datebegin 128 | } 129 | 130 | stopwatch_end(){ 131 | dateend="$EPOCHSECONDS" 132 | datediff="$((dateend-datebegin))" 133 | runtime="$(date -ud "@$datediff" -u +'%-Mm %-Ss')" 134 | echo "[OK] Tests completed succesfully in $runtime" 135 | } 136 | 137 | check_workdir(){ 138 | if [[ "tests/tests.sh" != *$1 ]]; then 139 | echo "[KO] Please run this script from the main folder as 'tests/test.sh'" 140 | exit 1 141 | fi 142 | } 143 | 144 | logecho(){ 145 | if [ "${2+isSet}" ]; then 146 | logFile="$2" 147 | else 148 | logFile="$RUSTICLONE_TEST_DIR/logs/rusticlone.log" 149 | fi 150 | echo " " 151 | echo "$1" 152 | echo " " 153 | echo " " >> "$logFile" 154 | echo "$1" >> "$logFile" 155 | } 156 | 157 | print_space(){ 158 | echo " " 159 | echo " " 160 | } 161 | 162 | print_warning(){ 163 | echo "[!!] WARNING: this script will destroy the contents of \"$RUSTIC_PROFILES_DIR\" and \"$RUSTICLONE_TEST_DIR\"" 164 | echo "[!!] Required programs: bash, coreutils, python-coverage, rustic, rclone" 165 | sleep 5 166 | } 167 | 168 | check_dependencies(){ 169 | for dep in b2sum coverage python rustic rclone; do 170 | if ! command -v "$dep" &> /dev/null; then 171 | echo "[KO] $dep could not be found" 172 | exit 1 173 | fi 174 | done 175 | } 176 | 177 | print_cleanup(){ 178 | echo "[OK] Test completed, feel free to read test coverage and remove \"$RUSTIC_PROFILES_DIR\" and \"$RUSTICLONE_TEST_DIR\"" 179 | } 180 | 181 | remove_profiles_dir(){ 182 | if [[ -d "$RUSTIC_PROFILES_DIR" ]]; then 183 | rm -r "$RUSTIC_PROFILES_DIR" 184 | fi 185 | } 186 | 187 | create_dirs(){ 188 | echo "[OK] Creating directories" 189 | remove_profiles_dir 190 | if [[ -d "$RUSTICLONE_TEST_DIR" ]]; then 191 | rm -r "$RUSTICLONE_TEST_DIR" 192 | fi 193 | mkdir -p "$RUSTIC_PROFILES_DIR" "$RUSTICLONE_TEST_DIR"/{check,source/docs,source/pics,source/photos/deeply/nested,source/secrets,logs} 194 | } 195 | 196 | create_confs(){ 197 | echo "[OK] Creating configurations" 198 | profile1Conf="$RUSTIC_PROFILES_DIR/Documents-test.toml" 199 | profile2Conf="$RUSTIC_PROFILES_DIR/Pictures-test.toml" 200 | profile3Conf="$RUSTIC_PROFILES_DIR/Passwords-test.toml" 201 | profileCommonConf="$RUSTIC_PROFILES_DIR/common.toml" 202 | rcloneConfDecrypted="$RUSTIC_PROFILES_DIR/rclone-decrypted.conf" 203 | rcloneConfEncrypted="$RUSTIC_PROFILES_DIR/rclone-encrypted.conf" 204 | echo "${profile1Content[0]}" > "$profile1Conf" 205 | echo "${profile2Content[0]}" > "$profile2Conf" 206 | echo "${profile3Content[0]}" > "$profile3Conf" 207 | echo "${profileCommonContent[0]}" > "$profileCommonConf" 208 | echo "${profileCommonContent[0]}" >> "$profile2Conf" 209 | echo "${profileCommonContent[0]}" >> "$profile3Conf" 210 | echo "${rcloneConfContentDecrypted[0]}" > "$rcloneConfDecrypted" 211 | echo "${rcloneConfContentEncrypted[0]}" > "$rcloneConfEncrypted" 212 | } 213 | 214 | create_files(){ 215 | # 10MB each 216 | echo "[OK] Creating files" 217 | head -c 10000000 /dev/urandom > "$RUSTICLONE_TEST_DIR/source/docs/important.pdf" 218 | head -c 10000000 /dev/urandom > "$RUSTICLONE_TEST_DIR/source/docs/veryimportant.pdf" 219 | head -c 10000000 /dev/urandom > "$RUSTICLONE_TEST_DIR/source/docs/notsoimportant.docx" 220 | head -c 10000000 /dev/urandom > "$RUSTICLONE_TEST_DIR/source/pics/screenshot.png" 221 | head -c 10000000 /dev/urandom > "$RUSTICLONE_TEST_DIR/source/pics/opengraph.webp" 222 | head -c 10000000 /dev/urandom > "$RUSTICLONE_TEST_DIR/source/pics/funny.gif" 223 | head -c 10000000 /dev/urandom > "$RUSTICLONE_TEST_DIR/source/photos/photo.jpeg" 224 | head -c 10000000 /dev/urandom > "$RUSTICLONE_TEST_DIR/source/photos/deeply/nested/memory.avif" 225 | sameFile="$(head -c 10000000 /dev/urandom | base64)" 226 | echo "$sameFile" > "$RUSTICLONE_TEST_DIR/source/passwords.kdbx" 227 | echo "$sameFile" "almost" > "$RUSTICLONE_TEST_DIR/source/secrets/env" 228 | chmod 0600 "$RUSTICLONE_TEST_DIR/source/passwords.kdbx" 229 | } 230 | 231 | create_new_files(){ 232 | # 10MB each 233 | echo "[OK] Creating files" 234 | head -c 10000000 /dev/urandom > "$RUSTICLONE_TEST_DIR/source/docs/important2.pdf" 235 | head -c 10000000 /dev/urandom > "$RUSTICLONE_TEST_DIR/source/docs/veryimportant2.pdf" 236 | head -c 10000000 /dev/urandom > "$RUSTICLONE_TEST_DIR/source/docs/notsoimportant2.docx" 237 | head -c 10000000 /dev/urandom > "$RUSTICLONE_TEST_DIR/source/pics/screenshot2.png" 238 | head -c 10000000 /dev/urandom > "$RUSTICLONE_TEST_DIR/source/pics/opengraph2.webp" 239 | head -c 10000000 /dev/urandom > "$RUSTICLONE_TEST_DIR/source/pics/funny2.gif" 240 | head -c 10000000 /dev/urandom > "$RUSTICLONE_TEST_DIR/source/photos/photo2.jpeg" 241 | head -c 10000000 /dev/urandom > "$RUSTICLONE_TEST_DIR/source/photos/deeply/nested/memory2.avif" 242 | } 243 | 244 | create_check_source(){ 245 | echo "[OK] Creating checksums for source files" 246 | find "$RUSTICLONE_TEST_DIR/source" -type f -exec b2sum {} \; > "$RUSTICLONE_TEST_DIR/check/source.txt" 247 | } 248 | 249 | wait_background(){ 250 | while wait -n; do : ; done; 251 | } 252 | 253 | # ################################ RUSTICLONE BACKUP 254 | # ################ SEQUENTIAL 255 | 256 | rusticlone_backup(){ 257 | logecho "[OK] Backing up with Rusticlone" 258 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" -a "$GOOD_APPRISE_URL" backup 259 | } 260 | 261 | rusticlone_archive(){ 262 | logecho "[OK] Archiving with Rusticlone" 263 | coverage run --append --module rusticlone.cli archive 264 | } 265 | 266 | rusticlone_upload(){ 267 | logecho "[OK] Uploading with Rusticlone" 268 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" upload 269 | } 270 | 271 | rusticlone_backup_flags(){ 272 | touch "$RUSTICLONE_TEST_DIR/logs/log-specified-in-args.log" 273 | logecho "[OK] Backing up from Rusticlone" "$RUSTICLONE_TEST_DIR/logs/log-specified-in-args.log" 274 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" -P "Pictures-test" --log-file "$RUSTICLONE_TEST_DIR/logs/log-specified-in-args.log" backup 275 | logecho "[OK] Backing up from Rusticlone" 276 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" -P "Documents-test" backup 277 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" -P "Passwords-test" -a "$BAD_APPRISE_URL" backup 278 | } 279 | 280 | # ################ PARALLEL 281 | 282 | rusticlone_backup_parallel(){ 283 | echo "[OK] Backing up with Rusticlone using parallel mode" 284 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" --parallel backup 285 | } 286 | 287 | rusticlone_archive_parallel(){ 288 | echo "[OK] Archiving with Rusticlone using parallel mode" 289 | coverage run --append --module rusticlone.cli --parallel archive 290 | } 291 | 292 | rusticlone_upload_parallel(){ 293 | echo "[OK] Uploading with Rusticlone using parallel mode" 294 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" --parallel upload 295 | } 296 | 297 | # ################ BACKGROUND 298 | 299 | rusticlone_backup_background(){ 300 | echo "[OK] Backing up with Rusticlone in background" 301 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" --parallel backup >/dev/null & 302 | } 303 | 304 | rusticlone_archive_background(){ 305 | echo "[OK] Archiving with Rusticlone in background" 306 | coverage run --append --module rusticlone.cli archive >/dev/null & 307 | } 308 | 309 | rusticlone_upload_background(){ 310 | echo "[OK] Uploading with Rusticlone in background" 311 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" upload >/dev/null & 312 | } 313 | 314 | # ################################ DISASTER SIMULATION 315 | # ################ LOSING SOURCE FILES 316 | 317 | destroy_source1(){ 318 | echo "[OK] Destroying documents" 319 | rm -rf "$RUSTICLONE_TEST_DIR/source/docs" 320 | } 321 | 322 | destroy_source2(){ 323 | echo "[OK] Destroying pictures" 324 | rm -rf "$RUSTICLONE_TEST_DIR/source/pics" 325 | } 326 | 327 | destroy_source3(){ 328 | echo "[OK] Destroying passwords" 329 | rm -rf "$RUSTICLONE_TEST_DIR/source/passwords.kdbx" 330 | } 331 | 332 | destroy_source12(){ 333 | echo "[OK] Destroying documents and pictures" 334 | rm -rf "$RUSTICLONE_TEST_DIR/source" 335 | } 336 | 337 | # ################ LOSING CACHE 338 | 339 | destroy_cache(){ 340 | echo "[OK] Destroying cache" 341 | rm -rf "$RUSTICLONE_TEST_DIR/cache" 342 | } 343 | 344 | # ################ LOSING LOCAL BACKUP 345 | 346 | destroy_local1(){ 347 | echo "[OK] Destroying local documents backup" 348 | rm -rf "$RUSTICLONE_TEST_DIR/local/Documents" 349 | } 350 | 351 | destroy_local2(){ 352 | echo "[OK] Destroying local pictures backup" 353 | rm -rf "$RUSTICLONE_TEST_DIR/local/Pictures" 354 | } 355 | 356 | destroy_local12(){ 357 | echo "[OK] Destroying local documents and pictures backup" 358 | rm -rf "$RUSTICLONE_TEST_DIR/local" 359 | } 360 | 361 | # ################ LOSING REMOTE BACKUP 362 | 363 | destroy_remote1(){ 364 | echo "[OK] Destroying remote documents backup" 365 | rm -rf "$RUSTICLONE_TEST_DIR/remote/Documents" 366 | } 367 | 368 | destroy_remote2(){ 369 | echo "[OK] Destroying remote pictures backup" 370 | rm -rf "$RUSTICLONE_TEST_DIR/remote/Pictures" 371 | } 372 | 373 | destroy_remote12(){ 374 | echo "[OK] Destroying remote documents and pictures backups" 375 | rm -rf "$RUSTICLONE_TEST_DIR/remote" 376 | } 377 | 378 | # ################################ RUSTICLONE RESTORE 379 | # ################ SEQUENTIAL 380 | 381 | rusticlone_restore(){ 382 | logecho "[OK] Restoring from Rusticlone" 383 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" restore 384 | } 385 | 386 | rusticlone_restore_flags(){ 387 | touch "$RUSTICLONE_TEST_DIR/logs/log-specified-in-args.log"4 388 | logecho "[OK] Restoring from Rusticlone" "$RUSTICLONE_TEST_DIR/logs/log-specified-in-args.log" 389 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" -P "Documents-test" --log-file "$RUSTICLONE_TEST_DIR/logs/log-specified-in-args.log" restore 390 | logecho "[OK] Restoring from Rusticlone" 391 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" -P "Passwords-test" restore 392 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" -P "Pictures-test" restore 393 | } 394 | 395 | rusticlone_download(){ 396 | logecho "[OK] Downloading from Rusticlone" 397 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" download 398 | } 399 | 400 | rusticlone_extract(){ 401 | logecho "[OK] Extracting from Rusticlone" 402 | coverage run --append --module rusticlone.cli extract 403 | } 404 | 405 | # ################ PARALLEL 406 | 407 | rusticlone_restore_parallel(){ 408 | echo "[OK] Restoring with Rusticlone using parallel mode" 409 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" --parallel restore 410 | } 411 | 412 | rusticlone_download_parallel(){ 413 | echo "[OK] Downloading with Rusticlone using parallel mode" 414 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" --parallel download 415 | } 416 | 417 | rusticlone_extract_parallel(){ 418 | echo "[OK] Extracting with Rusticlone using parallel mode" 419 | coverage run --append --module rusticlone.cli --parallel extract 420 | } 421 | 422 | # ################ BACKGROUND 423 | 424 | rusticlone_restore_background(){ 425 | echo "[OK] Restoring with Rusticlone in background" 426 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" --parallel restore >/dev/null & 427 | } 428 | 429 | rusticlone_download_background(){ 430 | echo "[OK] Downloading with Rusticlone in background" 431 | coverage run --append --module rusticlone.cli --remote "gdrive:/$RUSTICLONE_TEST_DIR/remote" download >/dev/null & 432 | } 433 | 434 | rusticlone_extract_background(){ 435 | echo "[OK] Extracting with Rusticlone in background" 436 | coverage run --append --module rusticlone.cli --parallel extract >/dev/null & 437 | } 438 | 439 | # ################################ RESULT 440 | 441 | check_source(){ 442 | echo "[OK] Checking sums for source files" 443 | b2sum --check "$RUSTICLONE_TEST_DIR/check/source.txt" || exit 1 444 | } 445 | 446 | create_coverage(){ 447 | coverage html 448 | coverage xml 449 | coverage report 450 | rm -rf "tests/coverage" 451 | mv "htmlcov" "$RUSTICLONE_TEST_DIR/coverage" 452 | mv "coverage.xml" "$RUSTICLONE_TEST_DIR/coverage" 453 | } 454 | 455 | create_badge(){ 456 | genbadge coverage -i "$RUSTICLONE_TEST_DIR/coverage/coverage.xml" -o "images/coverage.svg" 457 | } 458 | 459 | check_coverage(){ 460 | echo " " 461 | echo "[OK] Read the coverage report in detail by running:" 462 | echo " " 463 | echo " firefox \"$RUSTICLONE_TEST_DIR/coverage/index.html\"" 464 | echo " " 465 | } 466 | 467 | # ################################ MAIN 468 | 469 | main(){ 470 | # preparation 471 | check_workdir "$0" 472 | print_warning 473 | check_dependencies 474 | stopwatch_begin 475 | create_dirs 476 | create_confs 477 | create_files 478 | create_check_source 479 | 480 | # backup 481 | rusticlone_backup 482 | print_space 483 | destroy_local1 484 | destroy_remote2 485 | print_space 486 | rusticlone_backup_parallel 487 | print_space 488 | destroy_remote1 489 | destroy_local2 490 | print_space 491 | rusticlone_backup_background 492 | rusticlone_backup_background 493 | rusticlone_backup_parallel 494 | rusticlone_upload_background 495 | rusticlone_archive_background 496 | rusticlone_upload_background 497 | rusticlone_archive_background 498 | rusticlone_archive 499 | rusticlone_backup_background 500 | wait_background 501 | print_space 502 | destroy_cache 503 | print_space 504 | rusticlone_upload_parallel 505 | print_space 506 | destroy_remote2 507 | destroy_cache 508 | print_space 509 | rusticlone_archive_parallel 510 | rusticlone_upload 511 | print_space 512 | destroy_source12 513 | destroy_source3 514 | destroy_local2 515 | print_space 516 | 517 | # restore 518 | rusticlone_restore 519 | print_space 520 | destroy_source1 521 | destroy_local2 522 | print_space 523 | rusticlone_restore_background 524 | rusticlone_restore_parallel 525 | wait_background 526 | print_space 527 | check_source 528 | destroy_local1 529 | destroy_source2 530 | print_space 531 | rusticlone_restore_background 532 | rusticlone_download_background 533 | print_space 534 | rusticlone_download_parallel 535 | rusticlone_archive_parallel 536 | rusticlone_extract 537 | rusticlone_extract 538 | wait_background 539 | print_space 540 | check_source 541 | destroy_cache 542 | print_space 543 | rusticlone_download 544 | rusticlone_extract_background 545 | rusticlone_extract_parallel 546 | rusticlone_extract_background 547 | rusticlone_extract_parallel 548 | wait_background 549 | print_space 550 | check_source 551 | print_space 552 | destroy_cache 553 | destroy_source1 554 | destroy_local2 555 | print_space 556 | rusticlone_restore_background 557 | print_space 558 | rusticlone_restore 559 | wait_background 560 | print_space 561 | destroy_source2 562 | destroy_local1 563 | print_space 564 | rusticlone_restore_flags 565 | print_space 566 | 567 | # further run 568 | check_source 569 | create_new_files 570 | rusticlone_backup 571 | rusticlone_restore 572 | check_source 573 | print_space 574 | 575 | # results 576 | remove_profiles_dir 577 | create_coverage 578 | create_badge 579 | check_coverage 580 | stopwatch_end 581 | } 582 | 583 | # ################################################################ COMMANDS 584 | 585 | main "$@" 586 | -------------------------------------------------------------------------------- /rusticlone/processing/profile.py: -------------------------------------------------------------------------------- 1 | """ 2 | Define actions that can be run for each Rustic profile 3 | """ 4 | 5 | # ┌───────────────────────────────────────────────────────────────┐ 6 | # │ Contents of profile.py │ 7 | # ├───────────────────────────────────────────────────────────────┘ 8 | # │ 9 | # ├── IMPORTS 10 | # ├── CLASSES 11 | # ├── FUNCTIONS 12 | # │ 13 | # └─────────────────────────────────────────────────────────────── 14 | 15 | # ################################################################ IMPORTS 16 | 17 | # types 18 | from typing import Any 19 | 20 | # default timezone 21 | from datetime import timezone 22 | 23 | # file locations, path concatenation 24 | from pathlib import Path 25 | 26 | # stdout parsing 27 | import tomllib 28 | import json 29 | from datetime import datetime 30 | 31 | # hostname 32 | import platform 33 | 34 | # rusticlone 35 | from rusticlone.helpers.action import Action 36 | from rusticlone.helpers.rclone import Rclone 37 | from rusticlone.helpers.rustic import Rustic 38 | from rusticlone.helpers.formatting import clear_line, print_stats, convert_size 39 | 40 | # ################################################################ CLASSES 41 | 42 | 43 | class Profile: 44 | """ 45 | Define actions that can be run for each Rustic profile 46 | """ 47 | 48 | def __init__(self, profile: str, parallel: bool = False) -> None: 49 | """ 50 | Default values for each Rustic profile 51 | """ 52 | self.profile_name = profile 53 | self.parallel = parallel 54 | self.repo = Path("") 55 | self.lockfile = Path("rusticlone.lock") 56 | self.log_file = Path("rusticlone.log") 57 | self.env: dict[str, str] = {} 58 | self.does_forget = False 59 | self.password_provided = "" 60 | # json objects 61 | self.backup_output: list[dict[Any, Any]] = [] 62 | self.sources: list[Path] = [] 63 | self.sources_number = 0 64 | self.sources_exist: dict[Path, bool] = {} 65 | self.sources_type: dict[Path, str] = {} 66 | self.local_repo_exists = False 67 | self.snapshot_exists = False 68 | self.result = True 69 | self.hostname = platform.node() 70 | self.config: dict[str, Any] = {} 71 | self.latest_snapshot_timestamp = datetime.min.replace(tzinfo=timezone.utc) 72 | # self.repo_info = "" 73 | 74 | def parse_rustic_config(self) -> None: 75 | """ 76 | Parse the configuration to extract source, repo, log file and environment variables 77 | """ 78 | if self.result: 79 | action = Action("Parsing rustic configuration", self.parallel) 80 | rustic = Rustic(self.profile_name, "show-config") 81 | try: 82 | self.config = tomllib.loads(rustic.stdout) 83 | except (AttributeError, tomllib.TOMLDecodeError): 84 | self.result = action.abort("Could not parse rustic configuration") 85 | else: 86 | self.result = self.parse_rustic_config_component(action, "sources") 87 | self.result = self.parse_rustic_config_component(action, "repo") 88 | self.result = self.parse_rustic_config_component(action, "log") 89 | self.result = self.parse_rustic_config_component(action, "env") 90 | self.result = self.parse_rustic_config_component(action, "forget") 91 | if self.result: 92 | action.stop("Parsed rustic configuration") 93 | 94 | def parse_rustic_config_component(self, action, component: str) -> bool: 95 | """ 96 | Store values and return True if successful 97 | """ 98 | try: 99 | match component: 100 | case "sources": 101 | self.sources = parse_sources(self.config) 102 | case "repo": 103 | self.repo = parse_repo(self.config) 104 | self.lockfile = self.repo / "rusticlone.lock" 105 | case "log": 106 | self.log_file = parse_log(self.config) 107 | case "env": 108 | self.env = parse_env(self.config) 109 | case "forget": 110 | self.does_forget = parse_forget(self.config) 111 | except KeyError: 112 | match component: 113 | case "env": 114 | pass 115 | case _: 116 | action.abort( 117 | f"Could not parse {component} in config:\n", str(self.config) 118 | ) 119 | return True 120 | 121 | def check_rclone_config_exists(self) -> None: 122 | """ 123 | Parse Rustic configuration and extract rclone config, and rclone config password command. 124 | These will be passed to RClone during upload and download operations, where Rustic is not used. 125 | """ 126 | if self.result: 127 | action = Action("Checking Rclone configuration", self.parallel) 128 | try: 129 | rclone_config_file = Path(self.env["RCLONE_CONFIG"]) 130 | except (KeyError, TypeError) as exception: 131 | self.result = action.abort( 132 | "Could not parse rclone config:", str(exception) 133 | ) 134 | else: 135 | if rclone_config_file.is_file(): 136 | action.stop("Rclone configuration exists") 137 | else: 138 | self.result = action.abort( 139 | f"Rclone configuration file does not exist: {rclone_config_file}" 140 | ) 141 | 142 | def create_lockfile(self, operation: str) -> None: 143 | """ 144 | Add a lockfile to the repo containing the operation name. 145 | If the lockfile already exists, abort and print its contents. 146 | """ 147 | if self.result: 148 | action = Action("Creating lockfile", self.parallel) 149 | # Create repo directory if it doesn't exist (for new repos) 150 | if not self.local_repo_exists: 151 | self.repo.mkdir(parents=True, exist_ok=True) 152 | 153 | if self.lockfile.exists(): 154 | try: 155 | with open(self.lockfile, "r") as f: 156 | existing_operation = f.read() 157 | except Exception: 158 | self.result = action.abort("Lockfile already exists") 159 | else: 160 | self.result = action.abort( 161 | f"Found another {existing_operation} lockfile" 162 | ) 163 | else: 164 | try: 165 | with open(self.lockfile, "w") as f: 166 | f.write(operation) 167 | except Exception: 168 | self.result = action.abort("Could not create lockfile") 169 | else: 170 | action.stop("Created lockfile") 171 | 172 | def delete_lockfile(self) -> None: 173 | """ 174 | Delete the lockfile from the repo 175 | """ 176 | if self.result: 177 | action = Action("Deleting lockfile", self.parallel) 178 | if self.lockfile.exists(): 179 | self.lockfile.unlink() 180 | action.stop("Deleted lockfile") 181 | 182 | def set_log_file(self, passed_log_file: Path) -> None: 183 | """ 184 | set rclone log file 185 | if not default has been passed, use it, otherwise use the one in conf 186 | if there are no matches in conf, use the default one 187 | rustic fails if it cannot find the parent folder of the log file when parsing the conf, 188 | so it must be created before Profile.read_rustic_config() is run 189 | from now on, use self.log_file in Profile.upload() and Profile.download() 190 | """ 191 | if self.result: 192 | action = Action("Setting log file", self.parallel) 193 | if passed_log_file != Path("rusticlone.log"): 194 | self.log_file = passed_log_file 195 | if self.parallel: 196 | suffix_old = self.log_file.suffix 197 | suffix_new = "-" + self.profile_name + ".log" 198 | self.log_file = Path(str(self.log_file).replace(suffix_old, suffix_new)) 199 | # rustic fails anyway if it cannot find the path when parsing the conf 200 | self.log_file.parent.mkdir(parents=True, exist_ok=True) 201 | self.log_file.touch(exist_ok=True) 202 | action.stop("Set log file") 203 | 204 | def check_sources_exist(self) -> None: 205 | """ 206 | Check if all the file sources for the profile exist in the local filesystem 207 | """ 208 | if self.result: 209 | action = Action("Checking if sources exists", self.parallel) 210 | # print(self.source) 211 | for source in self.sources: 212 | if source.exists(): 213 | self.sources_exist[source] = True 214 | else: 215 | self.sources_exist[source] = False 216 | if all(self.sources_exist.values()): 217 | self.sources_number = len(self.sources) 218 | action.stop("All sources exist") 219 | else: 220 | self.result = action.abort("Some sources do not exist") 221 | 222 | def check_local_repo_exists(self) -> None: 223 | """ 224 | Check if the local repo folder exists using pathlib 225 | The program doesn't fail if the local repo doesn't exist, 226 | because we can create it with rustic init or by downloading it 227 | However, some restore functions depend on its existence, 228 | that's why we store the check result as a boolean variable. 229 | 230 | Skip if missing local repo: 231 | - check_local_repo_health(), init(), download() 232 | 233 | Fail if missing local repo: 234 | - repo_stats(), check_latest_snapshot(), check_source_type(), restore() 235 | """ 236 | if self.result: 237 | action = Action("Checking if local repo exists", self.parallel) 238 | repo_config_file = self.repo / "config" 239 | if repo_config_file.exists() and repo_config_file.is_file(): 240 | self.local_repo_exists = True 241 | action.stop("Local repo already exists") 242 | else: 243 | self.local_repo_exists = False 244 | action.stop("Local repo does not exist yet") 245 | 246 | def check_local_repo_health(self) -> None: 247 | """ 248 | Only check local repos for speed purposes 249 | """ 250 | if self.result: 251 | if self.local_repo_exists: 252 | action = Action("Checking repo health", self.parallel) 253 | Rustic(self.profile_name, "check", "--log-file", str(self.log_file)) 254 | action.stop("Repo is healthy") 255 | 256 | def check_remote_repo_exists(self, remote_prefix: str) -> None: 257 | """ 258 | Check remote repo folder with rclone 259 | """ 260 | if self.result: 261 | action = Action("Checking if remote repo exists", self.parallel) 262 | rclone_log_file = str(self.log_file) 263 | repo_name = str(self.repo.name) 264 | rclone_origin = remote_prefix + "/" + repo_name 265 | rclone = Rclone( 266 | env=self.env, 267 | log_file=rclone_log_file, 268 | action="lsd", 269 | origin=rclone_origin, 270 | check_return_code=False, 271 | ) 272 | # == 3 if does not exist 273 | if rclone.returncode != 0: 274 | self.result = action.abort("Remote repo does not exist") 275 | else: 276 | action.stop("Remote repo exists") 277 | 278 | def init(self) -> None: 279 | """ 280 | Initialize the repository if it does not exist, and perform the necessary setup. 281 | This method does not take any parameters and returns None, however it needs to know if 282 | the local repo already exists. 283 | We don't remove this function even if "--init" flag in backup would do the trick, 284 | as we set custom treepack and datapack sizes 285 | """ 286 | # if self.local_repo_exists: 287 | # action = Action("Using existing repo", self.parallel) 288 | # else: 289 | if self.result: 290 | if not self.local_repo_exists: 291 | action = Action("Initializing a new local repo", self.parallel) 292 | # will go interactive if [repository] password is not set 293 | rustic = Rustic( 294 | self.profile_name, 295 | "init", 296 | "--with-created", 297 | "--set-treepack-size", 298 | "50MB", 299 | "--set-datapack-size", 300 | "500MB", 301 | "--log-file", 302 | str(self.log_file), 303 | ) 304 | if rustic.returncode != 0: 305 | self.result = action.abort("Could not inizialize a new local repo") 306 | else: 307 | self.local_repo_exists = True 308 | action.stop("Initialized a new local repo") 309 | 310 | def backup(self) -> None: 311 | """ 312 | Perform a backup operation and store the output in self.backup_output. 313 | If multiple sources per profile are set, the output is the concatenation of json objects 314 | https://stackoverflow.com/a/42985887 315 | """ 316 | if self.result: 317 | action = Action("Creating snapshot", self.parallel) 318 | rustic = Rustic( 319 | self.profile_name, 320 | "backup", 321 | "--init", 322 | "--json", 323 | "--log-file", 324 | str(self.log_file), 325 | ) 326 | try: 327 | text = rustic.stdout.lstrip() 328 | while text: 329 | json_object, index = json.JSONDecoder().raw_decode(text) 330 | text = text[index:].lstrip() 331 | self.backup_output.append(json_object) 332 | except (AttributeError, json.JSONDecodeError): 333 | # print(json.loads(rustic.stdout)) 334 | self.result = action.abort("Could not create snapshot") 335 | else: 336 | action.stop("Created snapshot") 337 | 338 | def source_stats(self) -> None: 339 | """ 340 | A method to gather source statistics and print them to the console. 341 | This method does not take any parameters and does not return anything. 342 | """ 343 | if self.result: 344 | # action = Action("Retrieving source stats", self.parallel) 345 | action = Action("Retrieving stats", self.parallel) 346 | # print(self.backup_output) 347 | # print(self.backup_output) 348 | source_files = 0 349 | source_size = 0 350 | snapshot_size = 0 351 | try: 352 | for json_object in self.backup_output: 353 | summary = json_object["summary"] 354 | source_files += int(summary["total_files_processed"]) 355 | source_size += int(summary["total_bytes_processed"]) 356 | snapshot_size += int(summary["data_added"]) 357 | # else: 358 | # source_files = 0 359 | # source_size = 0 360 | # snapshot_size = 0 361 | except (KeyError, TypeError): 362 | self.result = action.abort("Could not retrieve stats") 363 | else: 364 | clear_line(parallel=self.parallel) 365 | source_text = "sources" if self.sources_number > 1 else "source" 366 | print_stats( 367 | "Number of sources:", 368 | str(self.sources_number), 369 | parallel=self.parallel, 370 | ) 371 | print_stats( 372 | f"Files in {source_text}:", 373 | str(source_files), 374 | parallel=self.parallel, 375 | ) 376 | print_stats( 377 | f"Size of {source_text}:", 378 | convert_size(source_size), 379 | parallel=self.parallel, 380 | ) 381 | print_stats( 382 | "Size of snapshot:", 383 | convert_size(snapshot_size), 384 | parallel=self.parallel, 385 | ) 386 | print_stats("", "", parallel=self.parallel) 387 | 388 | def repo_stats(self) -> None: 389 | """ 390 | Get repo stats and print the number of files and the total size of the repository. 391 | """ 392 | if self.result: 393 | # action = Action("Retrieving repo stats", self.parallel) 394 | # saving one command, even if it's run before snapshot 395 | # json_output = json.loads(self.repo_info) 396 | rustic = Rustic( 397 | self.profile_name, 398 | "repoinfo", 399 | "--json", 400 | "--log-file", 401 | str(self.log_file), 402 | ) 403 | if rustic.stdout != "": 404 | json_output = json.loads(rustic.stdout) 405 | # print(json_output) 406 | # "config" is not included in repoinfo 407 | repo_files = 1 408 | repo_size = 0 409 | repo_files += sum( 410 | [entry["count"] for entry in json_output["files"]["repo"]] 411 | ) 412 | repo_size += sum( 413 | [entry["size"] for entry in json_output["files"]["repo"]] 414 | ) 415 | clear_line(parallel=self.parallel) 416 | # action.stop("Retrieved repo stats") 417 | print_stats( 418 | "Size of repo:", convert_size(repo_size), parallel=self.parallel 419 | ) 420 | print_stats("Files in repo:", str(repo_files), parallel=self.parallel) 421 | else: 422 | self.result = False # action.abort("Repoinfo output is empty") 423 | 424 | def forget(self) -> None: 425 | """ 426 | Mark snapshots for deletion and evenually prune them. 427 | """ 428 | if self.result: 429 | if self.does_forget: 430 | action = Action("Deprecating old snapshots", self.parallel) 431 | Rustic( 432 | self.profile_name, 433 | "forget", 434 | "--json", 435 | "--fast-repack", 436 | "--no-resize", 437 | "--log-file", 438 | str(self.log_file), 439 | ) 440 | action.stop("Deprecated old snapshots") 441 | 442 | def upload(self, remote_prefix: str) -> None: 443 | """ 444 | Uploads the local repository to a remote destination using rclone. 445 | """ 446 | if self.result: 447 | action = Action("Uploading repo", self.parallel) 448 | rclone_log_file = str(self.log_file) 449 | rclone_origin = str(self.repo).replace("\\", "/").replace("//", "/") 450 | # rclone_destination = remote_prefix + "/" + self.profile_name 451 | repo_name = str(self.repo.name) 452 | rclone_destination = remote_prefix + "/" + repo_name 453 | # print(rclone_destination) 454 | rclone = Rclone( 455 | env=self.env, 456 | log_file=rclone_log_file, 457 | action="sync", 458 | # 1.67+, if added to 1.65.2 complains that log_file is an invalid option 459 | other_flags=[ 460 | "--create-empty-src-dirs=false", 461 | "--no-update-dir-modtime", 462 | ], 463 | origin=rclone_origin, 464 | destination=rclone_destination, 465 | ) 466 | # action.stop(f"Uploaded repo: {rclone_destination}") 467 | if rclone.returncode != 0: 468 | self.result = action.abort("Could not upload repo") 469 | else: 470 | action.stop("Uploaded repo") 471 | 472 | def download(self, remote_prefix: str) -> None: 473 | """ 474 | Uploads the remote repository to a local destination using rclone. 475 | """ 476 | if self.result: 477 | if not str(self.repo).startswith("rclone:"): 478 | action = Action("Downloading repo", self.parallel) 479 | if not self.local_repo_exists: 480 | rclone_log_file = str(self.log_file) 481 | # rclone_origin = remote_prefix + "/" + self.profile_name 482 | repo_name = str(self.repo.name) 483 | rclone_origin = remote_prefix + "/" + repo_name 484 | rclone_destination = str(self.repo) 485 | Rclone( 486 | env=self.env, 487 | log_file=rclone_log_file, 488 | action="sync", 489 | other_flags=[ 490 | "--create-empty-src-dirs=false", 491 | "--no-update-dir-modtime", 492 | ], 493 | origin=rclone_origin, 494 | destination=rclone_destination, 495 | ) 496 | action.stop("Downloaded repo") 497 | # action.stop(f"Downloaded repo: {rclone_destination}") 498 | else: 499 | action.stop("Repo already downloaded") 500 | # print(self.repo) 501 | # else: 502 | # action.stop("Not downloading remote-only repo, extracting it directly") 503 | 504 | def check_latest_snapshot(self) -> None: 505 | """ 506 | Read the latest snapshot from a local repo that has just been downloaded 507 | Require local repo and snapshot to exist 508 | """ 509 | if self.result: 510 | action = Action("Retrieving latest snapshot", self.parallel) 511 | if self.local_repo_exists: 512 | for source in self.sources: 513 | rustic = Rustic( 514 | self.profile_name, 515 | "snapshots", 516 | "latest", 517 | "--json", 518 | "--filter-paths", 519 | f"{source}", 520 | "--log-file", 521 | str(self.log_file), 522 | ) 523 | json_output = json.loads(rustic.stdout) 524 | if json_output is not None and len(json_output) > 0: 525 | # print(f"output: {json_output}" 526 | try: 527 | this_snapshot_timestamp = datetime.fromisoformat( 528 | json_output[-1]["snapshots"][0]["time"] 529 | ) 530 | except ValueError: 531 | self.result = action.abort("Could not parse timestamp") 532 | except KeyError: 533 | print(json.dumps(json_output, indent=1)) 534 | self.result = action.abort("Invalid json output") 535 | else: 536 | if ( 537 | not self.latest_snapshot_timestamp 538 | or this_snapshot_timestamp 539 | > self.latest_snapshot_timestamp 540 | ): 541 | self.latest_snapshot_timestamp = this_snapshot_timestamp 542 | else: 543 | self.result = action.abort("Repo does not have snapshots") 544 | timestamp_pretty = self.latest_snapshot_timestamp.strftime( 545 | "%Y-%m-%d %H:%M" 546 | ) 547 | clear_line(parallel=self.parallel) 548 | # self.snapshot_exists = True 549 | print_stats( 550 | "Restoring from:", 551 | timestamp_pretty, 552 | 19, 553 | 21, 554 | parallel=self.parallel, 555 | ) 556 | else: 557 | self.result = action.abort("Local repo does not exist") 558 | 559 | def check_sources_type(self) -> None: 560 | """ 561 | Check if the source that needs to be restored is a directory or file 562 | Require local repo and snapshot to exist 563 | """ 564 | if self.result: 565 | action = Action("Checking type of sources", self.parallel) 566 | for source in self.sources: 567 | rustic = Rustic( 568 | self.profile_name, 569 | "ls", 570 | "latest", 571 | "--filter-paths", 572 | f"{source}", 573 | "--glob", 574 | f"{source}", 575 | "--long", 576 | "--log-file", 577 | str(self.log_file), 578 | ) 579 | try: 580 | rustic.stdout[0] 581 | except IndexError: 582 | # empty folders do not return results 583 | pass 584 | except AttributeError: 585 | # error in command 586 | self.result = action.abort("Could not determine type of source") 587 | else: 588 | if rustic.stdout[0] == "d" or rustic.stdout.count("\n") > 1: 589 | self.sources_type[source] = "dir" 590 | else: 591 | self.sources_type[source] = "file" 592 | action.stop("Stored source types") 593 | 594 | def restore(self) -> None: 595 | """ 596 | Extract files in the latest snapshot to the source location, after creating it if missing 597 | if source is a directory, it is created if missing 598 | if source is a file, its parent is created if missing 599 | Require local repo and snapshot to exist 600 | """ 601 | if self.result: 602 | action = Action("Extracting snapshot", self.parallel) 603 | for source, source_type in self.sources_type.items(): 604 | if self.result: 605 | if source_type == "dir": 606 | Path(source).mkdir(parents=True, exist_ok=True) 607 | else: 608 | Path(source).parent.mkdir(parents=True, exist_ok=True) 609 | rustic = Rustic( 610 | self.profile_name, 611 | "restore", 612 | f"latest:{source}", 613 | source, 614 | "--filter-paths", 615 | f"{source}", 616 | "--log-file", 617 | str(self.log_file), 618 | ) 619 | if rustic.returncode != 0: 620 | self.result = action.abort(f"Error extracting '{source}'") 621 | action.stop("Snapshot extracted") 622 | 623 | 624 | # ################################################################ FUNCTIONS 625 | 626 | 627 | def parse_repo(config: dict[str, Any]) -> Path: 628 | """ 629 | Extract repository folder from Rustic profile configuration 630 | """ 631 | return Path(config["repository"]["repository"]) 632 | 633 | 634 | def parse_sources(config: dict[str, Any]) -> list[Path]: 635 | """ 636 | Extract list of sources from Rustic profile configuration 637 | """ 638 | sources: list[str] = [] 639 | raw_sources = [snapshot["sources"] for snapshot in config["backup"]["snapshots"]] 640 | # raw_sources can be either lists or strings 641 | for source in raw_sources: 642 | if source and isinstance(source, list): 643 | sources.extend(source) 644 | else: 645 | sources.append(source) 646 | # remove eventual duplicates and convert to Path 647 | unique_sources = list(set({Path(source) for source in sources})) 648 | return unique_sources 649 | 650 | 651 | def parse_log(config: dict[str, Any]) -> Path: 652 | """ 653 | Extract log file location from Rustic profile configuration 654 | """ 655 | return Path(config["global"]["log-file"]) 656 | 657 | 658 | def parse_env(config: dict[str, Any]) -> dict[str, Any]: 659 | """ 660 | Extract environment variables from Rustic profile configuration 661 | """ 662 | return config["global"]["env"] 663 | 664 | 665 | def parse_forget(config: dict[str, Any]) -> bool: 666 | """ 667 | Check if the Rustic config has any "keep-*" keys inside [forget] section 668 | Returns True if any keep-* keys are found, False otherwise 669 | """ 670 | return any(key.startswith("keep-") for key in config["forget"].keys()) 671 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | --------------------------------------------------------------------------------