├── .coveragerc ├── .github └── workflows │ └── build.yaml ├── .gitignore ├── .readthedocs.yml ├── LICENSE ├── README.rst ├── bu ├── clean ├── conftest.py ├── doc ├── Makefile ├── accessories.rst ├── api.rst ├── cache.svg ├── commands.rst ├── conf.py ├── configuring.rst ├── examples.rst ├── index.rst ├── installing.rst ├── monitoring.rst ├── releases.rst └── requirements.txt ├── emborg ├── __init__.py ├── collection.py ├── command.py ├── emborg.py ├── help.py ├── hooks.py ├── main.py ├── overdue.py ├── patterns.py ├── preferences.py ├── python.py ├── shlib.py └── utilities.py ├── od ├── pyproject.toml ├── tests ├── INITIAL_CONFIGS │ ├── home │ ├── root │ └── settings ├── README ├── STARTING_CONFIGS │ ├── README │ ├── overdue.conf │ ├── settings │ ├── subdir │ │ └── file │ ├── test0 │ ├── test1 │ ├── test2 │ ├── test2excludes │ ├── test2passphrase │ ├── test3 │ ├── test4 │ ├── test5 │ ├── test6 │ ├── test6patterns │ ├── test7 │ ├── test7patterns │ ├── test8 │ └── test9 ├── clean ├── test-cases.nt └── test_emborg.py └── tox.ini /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | exclude_lines = 3 | pragma: no cover 4 | raise NotImplementedError 5 | 6 | [run] 7 | source = emborg 8 | omit = */.tox/* 9 | -------------------------------------------------------------------------------- /.github/workflows/build.yaml: -------------------------------------------------------------------------------- 1 | # These tests require that BorgBackup be installed, 2 | # which is not available on GitHub Actions. 3 | 4 | name: Emborg 5 | on: [push, pull_request] 6 | jobs: 7 | check-bats-version: 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | os: [ubuntu-latest] 12 | python-version: ["3.6", "3.x"] 13 | max-parallel: 6 14 | 15 | steps: 16 | - uses: actions/checkout@v2 17 | with: 18 | submodules: true 19 | - name: Set up Python ${{ matrix.python-version }} 20 | uses: actions/setup-node@v2 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | - name: Install Borg 24 | run: | 25 | sudo apt-get -y install python3-pyfuse3 borgbackup 26 | - name: Install packages 27 | run: | 28 | python -m pip install --upgrade pip 29 | pip install . 30 | pip install tox 31 | pip install coveralls 32 | - name: Run tests 33 | run: | 34 | tox 35 | - name: Report test coverage 36 | env: 37 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 38 | COVERALLS_SERVICE_NAME: github 39 | run: coveralls 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | tests/.config 6 | tests/.local 7 | tests/configs 8 | tests/home 9 | .bump.cfg.nt 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | build/ 17 | develop-eggs/ 18 | dist/ 19 | downloads/ 20 | eggs/ 21 | .eggs/ 22 | lib/ 23 | lib64/ 24 | parts/ 25 | sdist/ 26 | var/ 27 | wheels/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | MANIFEST 32 | .*.swp 33 | doc/.build 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *.cover 54 | .hypothesis/ 55 | .pytest_cache/ 56 | tests/configs.symlink 57 | tests/repositories 58 | 59 | 60 | # Translations 61 | *.mo 62 | *.pot 63 | 64 | # Django stuff: 65 | *.log 66 | local_settings.py 67 | db.sqlite3 68 | 69 | # Flask stuff: 70 | instance/ 71 | .webassets-cache 72 | 73 | # Scrapy stuff: 74 | .scrapy 75 | 76 | # Sphinx documentation 77 | docs/_build/ 78 | 79 | # PyBuilder 80 | target/ 81 | 82 | # Jupyter Notebook 83 | .ipynb_checkpoints 84 | 85 | # pyenv 86 | .python-version 87 | 88 | # celery beat schedule file 89 | celerybeat-schedule 90 | 91 | # SageMath parsed files 92 | *.sage.py 93 | 94 | # Environments 95 | .env 96 | .venv 97 | env/ 98 | venv/ 99 | ENV/ 100 | env.bak/ 101 | venv.bak/ 102 | 103 | # Spyder project settings 104 | .spyderproject 105 | .spyproject 106 | 107 | # Rope project settings 108 | .ropeproject 109 | 110 | # mkdocs documentation 111 | /site 112 | 113 | # mypy 114 | .mypy_cache/ 115 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | build: 9 | os: ubuntu-22.04 10 | tools: 11 | python: "3.11" 12 | 13 | # Build documentation in the doc/ directory with Sphinx 14 | sphinx: 15 | configuration: doc/conf.py 16 | 17 | # Optionally build your docs in additional formats such as PDF and ePub 18 | formats: all 19 | 20 | # Optionally set the version of Python and requirements required to build your docs 21 | python: 22 | install: 23 | - requirements: doc/requirements.txt 24 | - method: pip 25 | path: . 26 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Emborg — Front-End to Borg Backup 2 | ================================= 3 | 4 | |downloads| |build status| |coverage| |rtd status| |pypi version| |python version| 5 | 6 | :Author: Ken Kundert 7 | :Version: 1.41 8 | :Released: 2024-11-11 9 | 10 | *Emborg* is a simple command line utility to orchestrate backups. It is built as 11 | a front-end to Borg_, a powerful and fast deduplicating backup program. With 12 | *Emborg*, you specify all the details about your backups once in advance, and 13 | then use a very simple command line interface for your day-to-day activities. 14 | 15 | Use of *Emborg* does not preclude the use of Borg directly on the same 16 | repository. The philosophy of *Emborg* is to provide commands that you would 17 | use often and in an interactive manner with the expectation that you would use 18 | Borg directly for the remaining commands. 19 | 20 | *Emborg* is not intended for use with *Borg* version 2.0 or later. Instead, the 21 | successor to *Emborg* should be used: Assimilate_. 22 | 23 | 24 | Getting Help 25 | ------------ 26 | 27 | You can find the documentation here: Emborg_. 28 | 29 | The *help* command provides information on how to use Emborg's various 30 | features. To get a listing of the topics available, use:: 31 | 32 | emborg help 33 | 34 | Then, for information on a specific topic use:: 35 | 36 | emborg help 37 | 38 | It is worth browsing all of the available topics at least once to get a sense of 39 | all that *Emborg* can do. 40 | 41 | 42 | .. _borg: https://borgbackup.readthedocs.io 43 | .. _assimilate: https://assimilate.readthedocs.io 44 | .. _emborg: https://emborg.readthedocs.io 45 | 46 | .. |downloads| image:: https://pepy.tech/badge/emborg/month 47 | :target: https://pepy.tech/project/emborg 48 | 49 | .. |build status| image:: https://github.com/KenKundert/emborg/actions/workflows/build.yaml/badge.svg 50 | :target: https://github.com/KenKundert/emborg/actions/workflows/build.yaml 51 | 52 | .. |coverage| image:: https://coveralls.io/repos/github/KenKundert/emborg/badge.svg?branch=master 53 | :target: https://coveralls.io/github/KenKundert/emborg?branch=master 54 | 55 | .. |rtd status| image:: https://img.shields.io/readthedocs/emborg.svg 56 | :target: https://emborg.readthedocs.io/en/latest/?badge=latest 57 | 58 | .. |pypi version| image:: https://img.shields.io/pypi/v/emborg.svg 59 | :target: https://pypi.python.org/pypi/emborg 60 | 61 | .. |python version| image:: https://img.shields.io/pypi/pyversions/emborg.svg 62 | :target: https://pypi.python.org/pypi/emborg/ 63 | 64 | -------------------------------------------------------------------------------- /bu: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from emborg.main import main 4 | main() 5 | -------------------------------------------------------------------------------- /clean: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set nonomatch 4 | 5 | (cd tests && ./clean) 6 | rm -rf .mypy_cache .ruff_cache doc/.build .tox 7 | 8 | # the rest is common to all python directories 9 | rm -f *.pyc *.pyo */*.pyc */*.pyo .test*.sum expected result install.out 10 | rm -rf build *.egg-info dist __pycache__ */__pycache__ .coverage 11 | rm -rf tests/home/.python-eggs .eggs tests/.coverage htmlcov tests/htmlcov 12 | rm -rf .cache tests/.cache tests/home/.cache .tox .hypothesis .pytest_cache */.pytest_cache 13 | -------------------------------------------------------------------------------- /conftest.py: -------------------------------------------------------------------------------- 1 | # Add missing dependency command line options to pytest command. 2 | 3 | import os 4 | import pytest 5 | from inform import Error, Info as CmdLineOpts 6 | from shlib import Run, set_prefs 7 | set_prefs(use_inform=True) 8 | 9 | # add command line options used to signal missing dependencies to pytest 10 | def pytest_addoption(parser): 11 | # parser.addoption( 12 | # "--borg-version", action="store", default="99.99.99", help="version number of borg" 13 | # ) 14 | parser.addoption( 15 | "--no-fuse", action="store_true", default=None, help="fuse is not available" 16 | ) 17 | 18 | # process the command line options 19 | @pytest.fixture(scope="session") 20 | def dependency_options(request): 21 | options = CmdLineOpts() 22 | 23 | # run borg and determine its version number 24 | try: 25 | borg = Run(["borg", "--version"], modes="sOEW") 26 | except Error as e: 27 | e.report() 28 | raise SystemExit 29 | borg_version = borg.stdout 30 | borg_version = borg_version.split()[-1] 31 | borg_version = borg_version.partition('a')[0] # strip off alpha version 32 | borg_version = borg_version.partition('b')[0] # strip off beta version 33 | borg_version = borg_version.partition('rc')[0] # strip off release candidate version 34 | borg_version = tuple(int(i) for i in borg_version.split('.')) 35 | options.borg_version = borg_version 36 | 37 | # determine whether FUSE is available 38 | # Can specify the --no-fuse command line option or set the 39 | # MISSING_DEPENDENCIES environment to 'fuse'. 40 | options.fuse_is_missing = request.config.getvalue("--no-fuse") 41 | if 'fuse' in os.environ.get('MISSING_DEPENDENCIES', '').lower().split(): 42 | options.fuse_is_missing = True 43 | 44 | return options 45 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = .build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help clean show html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | default: html 20 | 21 | help: 22 | @echo "Please use \`make ' where is one of" 23 | @echo " html to make standalone HTML files" 24 | @echo " dirhtml to make HTML files named index.html in directories" 25 | @echo " singlehtml to make a single large HTML file" 26 | @echo " pickle to make pickle files" 27 | @echo " json to make JSON files" 28 | @echo " htmlhelp to make HTML files and a HTML help project" 29 | @echo " qthelp to make HTML files and a qthelp project" 30 | @echo " devhelp to make HTML files and a Devhelp project" 31 | @echo " epub to make an epub" 32 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 33 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 34 | @echo " text to make text files" 35 | @echo " man to make manual pages" 36 | @echo " texinfo to make Texinfo files" 37 | @echo " info to make Texinfo files and run them through makeinfo" 38 | @echo " gettext to make PO message catalogs" 39 | @echo " changes to make an overview of all changed/added/deprecated items" 40 | @echo " linkcheck to check all external links for integrity" 41 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 42 | 43 | clean: 44 | -rm -rf $(BUILDDIR)/* 45 | 46 | html: 47 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 48 | @echo 49 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 50 | 51 | show: html 52 | firefox .build/html/index.html 53 | 54 | dirhtml: 55 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 56 | @echo 57 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 58 | 59 | singlehtml: 60 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 61 | @echo 62 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 63 | 64 | pickle: 65 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 66 | @echo 67 | @echo "Build finished; now you can process the pickle files." 68 | 69 | json: 70 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 71 | @echo 72 | @echo "Build finished; now you can process the JSON files." 73 | 74 | htmlhelp: 75 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 76 | @echo 77 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 78 | ".hhp project file in $(BUILDDIR)/htmlhelp." 79 | 80 | qthelp: 81 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 82 | @echo 83 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 84 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 85 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/emborg.qhcp" 86 | @echo "To view the help file:" 87 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/emborg.qhc" 88 | 89 | devhelp: 90 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 91 | @echo 92 | @echo "Build finished." 93 | @echo "To view the help file:" 94 | @echo "# mkdir -p $$HOME/.local/share/devhelp/emborg" 95 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/emborg" 96 | @echo "# devhelp" 97 | 98 | epub: 99 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 100 | @echo 101 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 102 | 103 | latex: 104 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 105 | @echo 106 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 107 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 108 | "(use \`make latexpdf' here to do that automatically)." 109 | 110 | latexpdf: 111 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 112 | @echo "Running LaTeX files through pdflatex..." 113 | $(MAKE) -C $(BUILDDIR)/latex all-pdf 114 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 115 | 116 | text: 117 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 118 | @echo 119 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 120 | 121 | man: 122 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 123 | @echo 124 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 125 | 126 | texinfo: 127 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 128 | @echo 129 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 130 | @echo "Run \`make' in that directory to run these through makeinfo" \ 131 | "(use \`make info' here to do that automatically)." 132 | 133 | info: 134 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 135 | @echo "Running Texinfo files through makeinfo..." 136 | make -C $(BUILDDIR)/texinfo info 137 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 138 | 139 | gettext: 140 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 141 | @echo 142 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 143 | 144 | changes: 145 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 146 | @echo 147 | @echo "The overview file is in $(BUILDDIR)/changes." 148 | 149 | linkcheck: 150 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 151 | @echo 152 | @echo "Link check complete; look for any errors in the above output " \ 153 | "or in $(BUILDDIR)/linkcheck/output.txt." 154 | 155 | doctest: 156 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 157 | @echo "Testing of doctests in the sources finished, look at the " \ 158 | "results in $(BUILDDIR)/doctest/output.txt." 159 | -------------------------------------------------------------------------------- /doc/accessories.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: emborg 2 | 3 | .. _emborg_accessories: 4 | 5 | Accessories 6 | =========== 7 | 8 | .. _borg space: 9 | 10 | Borg-Space 11 | ---------- 12 | 13 | `Borg-Space `_ is a utility that 14 | tracks and reports the space required by your *Borg* repositories. 15 | It also allows you to graph the space used over time. 16 | 17 | The following is an example of a graph generated by *borg-space* that allowed me 18 | to catch a problem that resulted in excessive growth in in the space required to 19 | hold my repository: in the switch from *Borg 1.1* to *Borg 1.2*, I had neglected 20 | to implement a compaction strategy. The problem was resolved on April 5th. 21 | 22 | .. image:: cache.svg 23 | :height: 300px 24 | :align: center 25 | 26 | 27 | .. _ntlog accessory: 28 | 29 | Logging with ntLog 30 | ------------------ 31 | 32 | `ntLog `_ is a log file aggregation 33 | utility. 34 | 35 | When run *Emborg* writes over a previously generated logfile. This becomes 36 | problematic if you have one cron script that runs *create* frequently and 37 | another that runs a command like *prune* less frequently. If there is trouble 38 | with the *prune* command it will be difficult to see and resolve because its 39 | logfile will be overwritten by subsequent *create* commands. 40 | 41 | *ntlog* can be run after each *Emborg* run to aggregate the individual logfile 42 | from each run into a single accumulating log file. To arrange this you can use 43 | :ref:`run_after_borg `:: 44 | 45 | run_after_borg = 'ntlog --keep-for 7d ~/.local/share/emborg/{config_name}.log' 46 | 47 | This accumulates the log files as they are created to 48 | ~/.local/share/emborg/{config_name}.log.nt. 49 | 50 | If your text editor is configured to use fold markers, you can configure *ntlog* 51 | to add headers to the composite logfile that contain fold markers. In doing so 52 | you can collapse large log entries into a single line folds until they are 53 | needed, at which point you can easily open the fold and examine the contents of 54 | the log file. Here is an example that adds headers with Vim fold markers to the 55 | composite log file:: 56 | 57 | run_after_borg = [ 58 | [ 59 | 'ntlog', 60 | '--keep-for=1w', 61 | '--day=D MMMM YYYY {{{{{{1', 62 | '--entry=h:mm A {{{{{{2', 63 | '--description={cmd_name}', 64 | '--fold-marker={{{{{{ ❬❬❬', 65 | '--editor=vim', 66 | '{log_dir}/{config_name}.log', 67 | ], 68 | ] 69 | 70 | If you use Vim, you can figure it to fold the composite log file with ``:set 71 | foldmethod=marker``. You can then open a fold using ``zo`` and close it with 72 | ``zc``. 73 | -------------------------------------------------------------------------------- /doc/api.rst: -------------------------------------------------------------------------------- 1 | .. currentmodule:: emborg 2 | 3 | .. _emborg_api: 4 | 5 | Python API 6 | ========== 7 | 8 | *Emborg* has a simple API that allows you to run borg commands. Here is an 9 | example taken from `sparekeys `_ that 10 | exports the keys from your *Borg* repository so then can be backed up 11 | separately: 12 | 13 | .. code-block:: python 14 | 15 | from emborg import Emborg 16 | from pathlib import Path 17 | 18 | destination = Path('keys') 19 | 20 | with Emborg('home') as emborg: 21 | borg = emborg.run_borg( 22 | cmd = 'key export', 23 | args = [emborg.destination(), destination / '.config/borg.repokey'] 24 | ) 25 | if borg.stdout: 26 | print(borg.stdout.rstrip()) 27 | 28 | *Emborg* takes the config name as an argument, if not given the default config 29 | is used. You can also pass list of *Emborg* options and the path to the 30 | configurations directory. 31 | 32 | *Emborg* provides the following useful methods and attributes: 33 | 34 | 35 | **configs** 36 | 37 | The list of configs associated with the requested config. If a scalar config 38 | was requested, the list be a list with a single member, the requested config. 39 | If the requested config is a composite config, the list consists of all the 40 | member configs of the requested config. 41 | 42 | 43 | **repository** 44 | 45 | The path to the repository. 46 | 47 | 48 | **version** 49 | 50 | The *Emborg* version number as a 3-tuple (major, minor, patch). 51 | 52 | 53 | **destination(archive)** 54 | 55 | Returns the full path to the archive. If Archive is False or None, then the path 56 | to the repository it returned. If Archive is True, then the default archive name 57 | as taken from settings file is used. This is only appropriate when creating new 58 | repositories. 59 | 60 | 61 | **run_borg(cmd, args, borg_opts, emborg_opts)** 62 | 63 | Runs a *Borg* command. 64 | 65 | *cmd* is the desired *Borg* command (ex: 'create', 'prune', etc.). 66 | 67 | *args* contains the command line arguments (such as the repository or 68 | archive). It may also contain any additional command line options not 69 | automatically provided. It may be a list or a string. If it is a string, it 70 | is split at white space. 71 | 72 | *borg_opts* are the command line options needed by *Borg*. If not given, it 73 | is created for you by *Emborg* based upon your configuration settings. 74 | 75 | Finally, *emborg_opts* is a list that may contain any of the following options: 76 | 'verbose', 'narrate', 'dry-run', or 'no-log'. 77 | 78 | This function runs the *Borg* command and returns a process object that 79 | allows you access to stdout via the *stdout* attribute. 80 | 81 | 82 | **run_borg_raw(args)** 83 | 84 | Runs a raw *Borg* command without interpretation except for replacing 85 | a ``@repo`` argument with the path to the repository. 86 | 87 | *args* contains all command line options and arguments except the path to 88 | the executable. 89 | 90 | 91 | **borg_options(cmd, emborg_opts)** 92 | 93 | This function returns the default *Borg* command line options, those that would 94 | be used in *run_borg* if *borg_opts* is not set. It can be used when 95 | constructing a custom *borg_opts*. 96 | 97 | 98 | **value(name, default='')** 99 | 100 | Returns the value of a scalar setting from an *Emborg* configuration. If not 101 | set, *default* is returned. 102 | 103 | 104 | **values(name, default=())** 105 | 106 | Returns the value of a list setting from an *Emborg* configuration. If not set, 107 | *default* is returned. 108 | 109 | 110 | Of these entry points, only *configs* works with composite configurations. 111 | 112 | You can examine the emborg/command.py file for inspiration and examples on how 113 | to use the *Emborg* API. 114 | 115 | 116 | Example 117 | ------- 118 | 119 | A command that queries one or more configs and prints the total size of its 120 | archives. This example is a simplified version of the *Emborg* accessory 121 | available from `Borg-Space `_. 122 | 123 | .. code-block:: python 124 | 125 | #!/usr/bin/env python3 126 | """ 127 | Borg Repository Size 128 | 129 | Reports on the current size of one or more Borg repositories managed by Emborg. 130 | 131 | Usage: 132 | borg-space [options] [...] 133 | 134 | Options: 135 | -m , --message template to use when building output message 136 | 137 | may contain {size}, which is replaced with the measured size, and 138 | {config}, which is replaced by the config name. 139 | If no replacements are made, size is appended to the end of the message. 140 | """ 141 | 142 | import arrow 143 | from docopt import docopt 144 | from emborg import Emborg 145 | from quantiphy import Quantity 146 | from inform import Error, display 147 | import json 148 | 149 | now = str(arrow.now()) 150 | 151 | cmdline = docopt(__doc__) 152 | show_size = not cmdline['--quiet'] 153 | record_size = cmdline['--record'] 154 | message = cmdline['--message'] 155 | 156 | try: 157 | requests = cmdline[''] 158 | if not requests: 159 | requests = [''] # this gets the default config 160 | 161 | for request in requests: 162 | # expand composite configs 163 | with Emborg(request, emborg_opts=['no-log']) as emborg: 164 | configs = emborg.configs 165 | 166 | for config in configs: 167 | with Emborg(config, emborg_opts=['no-log']) as emborg: 168 | 169 | # get name of latest archive 170 | borg = emborg.run_borg( 171 | cmd = 'list', 172 | args = ['--json', emborg.destination()] 173 | ) 174 | response = json.loads(borg.stdout) 175 | try: 176 | archive = response['archives'][-1]['archive'] 177 | except IndexError: 178 | raise Error('no archives available.', culprit=config) 179 | 180 | # get size info for latest archive 181 | borg = emborg.run_borg( 182 | cmd = 'info', 183 | args = ['--json', emborg.destination(archive)] 184 | ) 185 | response = json.loads(borg.stdout) 186 | size = response['cache']['stats']['unique_csize'] 187 | 188 | # report the size 189 | size_in_bytes = Quantity(size, 'B') 190 | if not message: 191 | message = '{config}: {size}' 192 | msg = message.format(config=config, size=size_in_bytes) 193 | if msg == message: 194 | msg = f'{message}: {size_in_bytes}' 195 | display(msg) 196 | 197 | except Error as e: 198 | e.report() 199 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # Emborg documentation build configuration file, created by 4 | # sphinx-quickstart on Mon Jun 12 12:01:56 2017. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | # import sys, os 15 | 16 | # If extensions (or modules to document with autodoc) are in another directory, 17 | # add these directories to sys.path here. If the directory is relative to the 18 | # documentation root, use os.path.abspath to make it absolute, like shown here. 19 | #sys.path.insert(0, os.path.abspath('.')) 20 | 21 | # -- General configuration ----------------------------------------------------- 22 | 23 | # If your documentation needs a minimal Sphinx version, state it here. 24 | #needs_sphinx = '1.0' 25 | 26 | # Add any Sphinx extension module names here, as strings. They can be extensions 27 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 28 | extensions = ''' 29 | sphinx.ext.autodoc 30 | sphinx.ext.coverage 31 | sphinx.ext.doctest 32 | sphinx.ext.napoleon 33 | sphinx.ext.todo 34 | sphinx.ext.viewcode 35 | sphinx_rtd_theme 36 | '''.split() 37 | 38 | # Add any paths that contain templates here, relative to this directory. 39 | templates_path = ['.templates'] 40 | 41 | # The suffix of source filenames. 42 | source_suffix = '.rst' 43 | 44 | # The encoding of source files. 45 | #source_encoding = 'utf-8-sig' 46 | 47 | # The master toctree document. 48 | master_doc = 'index' 49 | 50 | # General information about the project. 51 | project = u'emborg' 52 | copyright = u'2018-2024, Ken Kundert' 53 | 54 | # The version info for the project you're documenting, acts as replacement for 55 | # |version| and |release|, also used in various other places throughout the 56 | # built documents. 57 | # 58 | # The full version, including alpha/beta/rc tags. 59 | release = '1.41' 60 | # The short X.Y version. 61 | version = '.'.join(release.split('.')[0:2]) 62 | 63 | # The language for content autogenerated by Sphinx. Refer to documentation 64 | # for a list of supported languages. 65 | #language = None 66 | 67 | # There are two options for replacing |today|: either, you set today to some 68 | # non-false value, then it is used: 69 | #today = '' 70 | # Else, today_fmt is used as the format for a strftime call. 71 | #today_fmt = '%B %d, %Y' 72 | 73 | # List of patterns, relative to source directory, that match files and 74 | # directories to ignore when looking for source files. 75 | exclude_patterns = ['.build'] 76 | 77 | # The reST default role (used for this markup: `text`) to use for all documents. 78 | #default_role = None 79 | 80 | # If true, '()' will be appended to :func: etc. cross-reference text. 81 | #add_function_parentheses = True 82 | 83 | # If true, the current module name will be prepended to all description 84 | # unit titles (such as .. function::). 85 | #add_module_names = True 86 | 87 | # If true, sectionauthor and moduleauthor directives will be shown in the 88 | # output. They are ignored by default. 89 | #show_authors = False 90 | 91 | # The name of the Pygments (syntax highlighting) style to use. 92 | pygments_style = 'sphinx' 93 | 94 | # A list of ignored prefixes for module index sorting. 95 | #modindex_common_prefix = [] 96 | 97 | 98 | # -- Options for HTML output --------------------------------------------------- 99 | 100 | # The theme to use for HTML and HTML Help pages. See the documentation for 101 | # a list of builtin themes. 102 | html_theme = 'sphinx_rtd_theme' 103 | 104 | # Theme options are theme-specific and customize the look and feel of a theme 105 | # further. For a list of options available for each theme, see the 106 | # documentation. 107 | #html_theme_options = {} 108 | 109 | # Add any paths that contain custom themes here, relative to this directory. 110 | #html_theme_path = [] 111 | 112 | # The name for this set of Sphinx documents. If None, it defaults to 113 | # " v documentation". 114 | #html_title = None 115 | 116 | # A shorter title for the navigation bar. Default is the same as html_title. 117 | #html_short_title = None 118 | 119 | # The name of an image file (relative to this directory) to place at the top 120 | # of the sidebar. 121 | #html_logo = None 122 | 123 | # The name of an image file (within the static path) to use as favicon of the 124 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 125 | # pixels large. 126 | #html_favicon = None 127 | 128 | # Add any paths that contain custom static files (such as style sheets) here, 129 | # relative to this directory. They are copied after the builtin static files, 130 | # so a file named "default.css" will overwrite the builtin "default.css". 131 | #html_static_path = ['.static'] 132 | 133 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 134 | # using the given strftime format. 135 | #html_last_updated_fmt = '%b %d, %Y' 136 | 137 | # If true, SmartyPants will be used to convert quotes and dashes to 138 | # typographically correct entities. 139 | #html_use_smartypants = True 140 | 141 | # Custom sidebar templates, maps document names to template names. 142 | #html_sidebars = {} 143 | 144 | # Additional templates that should be rendered to pages, maps page names to 145 | # template names. 146 | #html_additional_pages = {} 147 | 148 | # If false, no module index is generated. 149 | #html_domain_indices = True 150 | 151 | # If false, no index is generated. 152 | #html_use_index = True 153 | 154 | # If true, the index is split into individual pages for each letter. 155 | #html_split_index = False 156 | 157 | # If true, links to the reST sources are added to the pages. 158 | #html_show_sourcelink = True 159 | 160 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 161 | #html_show_sphinx = True 162 | 163 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 164 | #html_show_copyright = True 165 | 166 | # If true, an OpenSearch description file will be output, and all pages will 167 | # contain a tag referring to it. The value of this option must be the 168 | # base URL from which the finished HTML is served. 169 | #html_use_opensearch = '' 170 | 171 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 172 | #html_file_suffix = None 173 | 174 | # Output file base name for HTML help builder. 175 | htmlhelp_basename = 'emborgdoc' 176 | 177 | 178 | # -- Options for LaTeX output -------------------------------------------------- 179 | 180 | latex_elements = { 181 | # The paper size ('letterpaper' or 'a4paper'). 182 | #'papersize': 'letterpaper', 183 | 184 | # The font size ('10pt', '11pt' or '12pt'). 185 | #'pointsize': '10pt', 186 | 187 | # Additional stuff for the LaTeX preamble. 188 | #'preamble': '', 189 | } 190 | 191 | # Grouping the document tree into LaTeX files. List of tuples 192 | # (source start file, target name, title, author, documentclass [howto/manual]). 193 | latex_documents = [ 194 | ('index', 'emborg.tex', u'Emborg Documentation', 195 | u'Ken Kundert', 'manual'), 196 | ] 197 | 198 | # The name of an image file (relative to this directory) to place at the top of 199 | # the title page. 200 | #latex_logo = None 201 | 202 | # For "manual" documents, if this is true, then toplevel headings are parts, 203 | # not chapters. 204 | #latex_use_parts = False 205 | 206 | # If true, show page references after internal links. 207 | #latex_show_pagerefs = False 208 | 209 | # If true, show URL addresses after external links. 210 | #latex_show_urls = False 211 | 212 | # Documents to append as an appendix to all manuals. 213 | #latex_appendices = [] 214 | 215 | # If false, no module index is generated. 216 | #latex_domain_indices = True 217 | 218 | 219 | # -- Options for manual page output -------------------------------------------- 220 | 221 | # One entry per manual page. List of tuples 222 | # (source start file, name, description, authors, manual section). 223 | man_pages = [ 224 | ('index', 'emborg', u'Emborg Documentation', 225 | [u'Ken Kundert'], 1) 226 | ] 227 | 228 | # If true, show URL addresses after external links. 229 | #man_show_urls = False 230 | 231 | 232 | # -- Options for Texinfo output ------------------------------------------------ 233 | 234 | # Grouping the document tree into Texinfo files. List of tuples 235 | # (source start file, target name, title, author, 236 | # dir menu entry, description, category) 237 | texinfo_documents = [ 238 | ('index', 'Emborg', u'Emborg Documentation', 239 | u'Ken Kundert', 'Emborg', 'Front-end to Borg Backup.', 240 | 'Miscellaneous'), 241 | ] 242 | 243 | # Documents to append as an appendix to all manuals. 244 | #texinfo_appendices = [] 245 | 246 | # If false, no module index is generated. 247 | #texinfo_domain_indices = True 248 | 249 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 250 | #texinfo_show_urls = 'footnote' 251 | 252 | #KSK: add custom css code if present 253 | def setup(app): 254 | import os 255 | if os.path.exists('.static/css/custom.css'): 256 | app.add_stylesheet('css/custom.css') 257 | 258 | -------------------------------------------------------------------------------- /doc/examples.rst: -------------------------------------------------------------------------------- 1 | .. _emborg examples: 2 | 3 | Examples 4 | ======== 5 | 6 | When first run, *Emborg* creates the settings directory and populates it with 7 | two configurations that you can use as starting points. Those two configurations 8 | make up our first two examples. 9 | 10 | 11 | .. _root example: 12 | 13 | Root 14 | ---- 15 | 16 | The *root* configuration is a suitable starting point for someone that wants to 17 | backup an entire machine, including both system and user files. In order to have 18 | permission to access the files, one must run this configuration as the *root* 19 | user. 20 | 21 | This configuration was constructed assuming that the backups would be run 22 | automatically at a fixed time using cron. Since this user only has one 23 | configuration, it is largely arbitrary which file each setting resides in, 24 | however both files must exist, and the *settings* file must contain 25 | *configurations* and *default_configuration*. 26 | 27 | Here is the contents of the settings file: /root/.config/emborg/settings: 28 | 29 | .. code-block:: python 30 | 31 | configurations = 'root' 32 | default_configuration = 'root' 33 | 34 | # basic settings 35 | notify = "root@continuum.com" 36 | upload_ratelimit = 2000 # bandwidth limit in kbps 37 | prune_after_create = True 38 | check_after_create = 'latest' 39 | 40 | # repository settings 41 | repository = 'backups:/mnt/backups/{host_name}-{user_name}-{config_name}' 42 | archive = '{prefix}{{now:%Y%m%d}}' 43 | prefix = '{config_name}-' 44 | compression = 'lz4' 45 | 46 | # shared filter settings 47 | exclude_if_present = '.nobackup' 48 | exclude_caches = True 49 | 50 | # prune settings 51 | keep_within = '1d' # keep all archives created within this interval 52 | keep_hourly = 48 # number of hourly archives to keep 53 | keep_daily = 14 # number of daily archives to keep 54 | keep_weekly = 8 # number of weekly archives to keep 55 | keep_monthly = 24 # number of monthly archives to keep 56 | keep_yearly = 24 # number of yearly archives to keep 57 | 58 | In this case we are assuming that *backups* (used in *repository*) is an entry 59 | in your SSH config file that points to the server that stores your repository. 60 | To be able to run this configuration autonomously from cron, *backups* must be 61 | configured to use a private key that does not have a passphrase. 62 | 63 | And here is the contents of the *root* configuration file: /root/.config/emborg/root: 64 | 65 | .. code-block:: python 66 | 67 | # Settings for root configuration 68 | passphrase = 'carvery overhang vignette platitude pantheon sissy toddler truckle' 69 | encryption = 'repokey' 70 | one_file_system = False 71 | 72 | src_dirs = '/' 73 | excludes = ''' 74 | /dev 75 | /home/*/.cache 76 | /mnt 77 | /proc 78 | /run 79 | /sys 80 | /tmp 81 | /var/cache 82 | /var/lock 83 | /var/run 84 | /var/tmp 85 | ''' # list of files or directories to skip 86 | 87 | This file contains the passphrase, and so you should be careful to set its 88 | permissions so that nobody but root can see its contents. Also, this 89 | configuration uses *repokey* as the encryption method, which is suitable when 90 | you control the server that holds the repository and you know it to be secure. 91 | 92 | Once this configuration is complete and has been tested, you would want to add 93 | a crontab entry so that it runs on a routine schedule. On servers that are 94 | always running, you could use `crontab -e` and add an entry like this: 95 | 96 | .. code-block:: text 97 | 98 | 30 03 * * * emborg --mute --config root create 99 | 100 | For individual workstations or laptops that are likely to be turned off at 101 | night, one would instead create an executable script in /etc/cron.daily that 102 | contains the following: 103 | 104 | .. code-block:: bash 105 | 106 | #/bin/sh 107 | # Run root backups 108 | 109 | emborg --mute --config root create 110 | 111 | Assume that this file is named *emborg*. Then after creating it, you would make 112 | it executable with: 113 | 114 | .. code-block:: bash 115 | 116 | $ chmod a+x /etc/cron.daily/emborg 117 | 118 | Scripts in /etc/cron.daily are one once a day, either at a fixed time generally 119 | early in the morning or, if not powered up at that time, shortly after being 120 | powered up. 121 | 122 | 123 | .. _user examples: 124 | 125 | User 126 | ---- 127 | 128 | The *home* configuration is a suitable starting point for someone that just 129 | wants to backup their home directory on their laptop. In this example, two 130 | configurations are created, one to be run manually that copies all files to 131 | a remote repository, and a second that runs every few minutes and creates 132 | snapshots of key working directories. This second allows you to quickly recover 133 | from mistakes you make during the day without having to go back to yesterday's 134 | copy of a file as a starting point. 135 | 136 | Here is the contents of the shared settings file: ~/.config/emborg/settings. 137 | 138 | .. code-block:: python 139 | 140 | # configurations 141 | configurations = 'home snapshots' 142 | default_configuration = 'home' 143 | 144 | # basic settings 145 | notifier = 'notify-send -u normal {prog_name} "{msg}"' 146 | 147 | # repository settings 148 | compression = 'lz4' 149 | 150 | # shared filter settings 151 | exclude_if_present = '.nobackup' 152 | exclude_caches = True 153 | 154 | 155 | .. _home example: 156 | 157 | Home 158 | ^^^^ 159 | 160 | Here is the contents of the *home* configuration file: ~/.config/emborg/home. 161 | This configuration backs up to a remote untrusted repository and is expected to 162 | be run interactively, perhaps once per day. 163 | 164 | .. code-block:: python 165 | 166 | repository = 'backups:/mnt/borg-backups/repositories/{host_name}-{user_name}-{config_name}' 167 | prefix = '{config_name}-' 168 | encryption = 'keyfile' 169 | avendesora_account = 'laptop-borg' 170 | needs_ssh_agent = True 171 | upload_ratelimit = 2000 172 | prune_after_create = True 173 | check_after_create = 'latest' 174 | 175 | src_dirs = '~' # paths to be backed up 176 | excludes = ''' 177 | ~/.cache 178 | **/.hg 179 | **/.git 180 | **/__pycache__ 181 | **/*.pyc 182 | **/.*.swp 183 | **/.*.swo 184 | **/*~ 185 | ''' 186 | 187 | run_before_backup = '(cd ~/src; ./clean)' 188 | 189 | # prune settings 190 | keep_within = '1d' # keep all archives created within this interval 191 | keep_hourly = 48 # number of hourly archives to keep 192 | keep_daily = 14 # number of daily archives to keep 193 | keep_weekly = 8 # number of weekly archives to keep 194 | keep_monthly = 24 # number of monthly archives to keep 195 | keep_yearly = 24 # number of yearly archives to keep 196 | 197 | In this case we are assuming that *backups* (used in *repository*) is an entry 198 | in your SSH config file that points to the server that stores your repository. 199 | *backups* should be configured to use a private key and that key should be 200 | preloaded into your SSH agent. 201 | 202 | This passphrase for this configuration is kept in `Avendesora 203 | `_, and the encryption method is *keyfile*. 204 | As such, it is critical that you extract the keyfile from *Borg* and copy it and 205 | your *Avendesora* files to a safe place so that both the keyfile and passphrase 206 | are available if you lose your disk. You can use `SpareKeys 207 | `_ to do this for you. Otherwise 208 | extract the keyfile using: 209 | 210 | .. code-block:: bash 211 | 212 | $ emborg borg key export @repo key.borg 213 | 214 | *cron* is not used for this configuration because the machine, being a laptop, 215 | is not guaranteed to be on at any particular time of the day. So instead, you 216 | would simply run *Emborg* on your own at a convenient time using: 217 | 218 | .. code-block:: bash 219 | 220 | $ emborg 221 | 222 | You can use the *Emborg due* command to remind you if a backup is overdue. You 223 | can wire it into status bar programs, such as *i3status* to give you a visual 224 | reminder, or you can configure cron to check every hour and notify you if they 225 | are overdue. This one triggers a notification: 226 | 227 | .. code-block:: text 228 | 229 | 0 * * * * emborg --mute due --days 1 || notify-send 'Backups are overdue' 230 | 231 | And this one sends an email: 232 | 233 | .. code-block:: text 234 | 235 | 0 * * * * emborg --mute due --days 1 --mail me@mydomain.com 236 | 237 | Alternately, you can use :ref:`emborg-overdue `. 238 | 239 | 240 | .. _snapshot example: 241 | 242 | Snap Shots 243 | ^^^^^^^^^^ 244 | 245 | And finally, here is the contents of the *snapshots* configuration file: 246 | ~/.config/emborg/snapshots. 247 | 248 | .. code-block:: python 249 | 250 | repository = '~/.cache/snapshots' 251 | encryption = 'none' 252 | 253 | src_dirs = '~' 254 | excludes = ''' 255 | ~/.cache 256 | ~/media 257 | **/.hg 258 | **/.git 259 | **/__pycache__ 260 | **/*.pyc 261 | **/.*.swp 262 | **/.*.swo 263 | **/.~ 264 | ''' 265 | 266 | # prune settings 267 | keep_hourly = 12 268 | prune_after_create = True 269 | check_after_create = False 270 | 271 | To run this configuration every 10 minutes, add the following entry to your 272 | crontab file using 'crontab -e': 273 | 274 | .. code-block:: text 275 | 276 | 0,10,20,30,40,50 * * * * emborg --mute --config snapshots create 277 | 278 | 279 | .. _rsync.net example: 280 | 281 | Rsync.net 282 | --------- 283 | 284 | *Rsync.net* is a commercial option for off-site storage. In fact, they give you 285 | a discount if you use `Borg Backup `_. 286 | 287 | Once you sign up for *Rsync.net* you can access your storage using *sftp*, 288 | *scp*, *rsync* or *borg* of course. *ssh* access is also available, but only 289 | for a limited set of commands. 290 | 291 | You would configure *Emborg* for *Rsync.net* in much the same way you would for 292 | any remote server. Of course, you should use some form of *keyfile* based 293 | encryption to keep your files secure. The only thing to be aware of is that by 294 | default they provide a old version of borg. To use a newer version, set the 295 | ``remote_path`` to ``borg1``. 296 | 297 | .. code-block:: python 298 | 299 | repository = '78548@ch-s012.rsync.net:repo' 300 | encryption = 'keyfile' 301 | remote_path = 'borg1' 302 | 303 | ... 304 | 305 | In this example, ``78548`` is the user name and ``ch-s012.rsync.net`` is the 306 | server they assign to you. ``repo`` is the name of the directory that is to 307 | contain your *Borg* repository. You are free to name it whatever you like and 308 | you can have as many as you like, with the understanding that you are 309 | constrained in the total amount of storage you consume. 310 | 311 | 312 | .. _borgbase example: 313 | 314 | BorgBase 315 | -------- 316 | 317 | `BorgBase `_ is another commercial alternative for 318 | *Borg Backups*. It allows full *Borg* access, append-only *Borg* access, and 319 | *rsync* access, though each form of access requires its own unique SSH key. 320 | 321 | Again, you should use some form of *keyfile* encryption to keep your files 322 | secure, and *BorgBase* recommends *Blake2* encryption as being the fastest 323 | alternative. 324 | 325 | .. code-block:: python 326 | 327 | repository = 'zMNZCv4B@zMNZCv4B.repo.borgbase.com:repo' 328 | encryption = 'keyfile-blake2' 329 | 330 | ... 331 | 332 | In this example, ``zMNZCv4B`` is the user name and 333 | ``zMNZCv4B.repo.borgbase.com`` is the server they assign to you. You may 334 | request any number of repositories, with each repository getting its own 335 | username and hostname. ``repo`` is the name of the directory that is to contain 336 | your *Borg* repository and cannot be changed. 337 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | .. Emborg documentation master file 2 | 3 | Emborg — Front-End to Borg Backup 4 | ================================= 5 | 6 | | Version: 1.41 7 | | Released: 2024-11-11 8 | | Please report all bugs and suggestions on GitHub_. 9 | 10 | 11 | What is Emborg? 12 | --------------- 13 | 14 | *Emborg* is a simple command line utility to orchestrate backups. It is built as 15 | a front-end to Borg_, a powerful and fast de-duplicating backup program. With 16 | *Emborg*, you specify all the details about your backups once in advance, and 17 | then use a very simple command line interface for your day-to-day activities. 18 | 19 | Use of *Emborg* does not preclude the use of *Borg* directly on the same 20 | repository. The philosophy of *Emborg* is to provide commands that you would 21 | use often and in an interactive manner with the expectation that you would use 22 | *Borg* directly for more unusual or esoteric situations. 23 | 24 | Why Emborg? 25 | ----------- 26 | 27 | There are alternatives to *Emborg* such as BorgMatic_ and Vorta_, both of which 28 | are also front-ends to *BorgBackup*. *BorgMatic* has a command line interface 29 | like *Emborg* while *Vorta* is GUI-based. *Emborg* distinguishes itself by 30 | providing a command line interface that is very efficient for common tasks, such 31 | as creating archives (backups), restoring files or directories, or comparing 32 | existing files to those in an archive. Also, *Emborg* naturally supports 33 | multiple destination repositories. This feature can be used to simultaneously 34 | backup to a local repository, which provides rapid restores, and an off-site 35 | repository, which provides increased safety in case of a local disaster. 36 | 37 | .. important:: 38 | 39 | The *Borg* developers are currently working to release *Borg 2.0*, which is 40 | a substantial change from *Borg 1.4*. *Borg 2.0* is a breaking change. 41 | Existing repositories must be transformed before they can be used with the 42 | new version, and the new version has a somewhat different and more powerful 43 | use model. *Emborg* does not and will not support *Borg 2.0*. Instead it 44 | being replaced by Assimilate_, which is the next generation of *Emborg*. 45 | Like *Borg 2*, *Assimilate* is a breaking change. The format of the 46 | configuration files has changed and some of the settings have been renamed. 47 | Also, the more powerful use model of *Borg 2* is adopted. 48 | 49 | *Emborg* will not support *Borg 2* and *Assimilate* will not support *Borg 50 | 1*. You should use *Emborg* until you are ready to transition to *Borg 2*, 51 | at which point you should also transition to *Assimilate*. 52 | 53 | All future development is moving to *Assimilate* and *Emborg* is 54 | transitioning to maintenance support only. 55 | 56 | 57 | Why Borg? 58 | --------- 59 | 60 | Well, everyone needs to backup their files. So perhaps the questions should be: 61 | why not *Duplicity*? Duplicity_ has been the standard way to do backups on Unix 62 | systems for many years. 63 | 64 | *Duplicity* provides full and incremental backups. A full backup makes complete 65 | copies of each file. With an incremental backup, only the difference between the 66 | current and previous versions of the file are saved. Thus, to retrieve a file 67 | from the backup, *Duplicity* must first get the original version of the file, 68 | and then apply each change. That approach results in the following issues: 69 | 70 | #. The recovery process is slow because the desired file is reconstructed from 71 | possibly a large number of change sets, each of which must be downloaded from 72 | a remote repository before it can be applied. The change sets are large, so 73 | the recovery of even small files can require downloading a large amount of 74 | data. It is common that the recovery of a single small file could require 75 | many hours. 76 | 77 | #. Because the recovery process requires so many steps, it can be fragile. 78 | Apparently it keeps all the change sets open during the recovery process, and 79 | so the recovery process can fail because the operating system limits how many 80 | files you can open at any one time. 81 | 82 | #. Generally, when there are problems, you only find them when you try to 83 | recover a file. At that point it is too late. 84 | 85 | #. Duplicity does not do de-duplication, so if your were to have multiple copies 86 | of the same file, or if you moved a file, then you would keep multiple copies 87 | of it. 88 | 89 | The first two issues can be reduced with frequent full backups, but this greatly 90 | increases the space you need to hold your backups. 91 | 92 | *Borg* works in a very different way. When *Borg* encounters a file, it first 93 | determines whether it is new or not. The file is determined to be new if the 94 | contents of that file do not already exist in the repository, in which case it 95 | copies the contents into the repository. Then, either way, it associates 96 | a pointer to the file's contents with the filepath. This makes it naturally 97 | de-duplicating. When it comes time to recover a file, it simply uses the file 98 | path to find the contents. In this way, it only retrieves the data it needs. 99 | There is no complicated and fragile process needed to reconstruct the file from 100 | a long string of differences. 101 | 102 | After living with *Duplicity* for many years, I now find the *Borg* recovery 103 | process stunningly fast and extremely reliable. I am completely sold on *Borg* 104 | and will never use *Duplicity* again. 105 | 106 | 107 | Terminology 108 | ----------- 109 | 110 | It is helpful to understand two terms that are used used by *Borg* to describe 111 | your backups. 112 | 113 | :repository: 114 | This is the location where all of your files are backed up to. It may be on 115 | a local file system or it may be remote, in which case it is accessed using 116 | *ssh*. 117 | 118 | A repository consists of a collection of disembodied and deduplicated file 119 | contents along with a collection of archives. 120 | 121 | :archive: 122 | This is a snapshot of the files that existed when a particular backup was 123 | run. Basically, it is a collection of file paths along with pointers to the 124 | contents of those files. 125 | 126 | 127 | Quick Tour 128 | ---------- 129 | 130 | You must initially describe your repository or repositories to *Emborg*. You do 131 | so by adding configuration files to ~/.config/emborg. At least two are required. 132 | First is the file that contains settings that are shared between all 133 | configurations. This is a Python file located at ``~/.config/emborg/settings``. 134 | Here is an example: 135 | 136 | .. code-block:: bash 137 | 138 | # configurations 139 | configurations = ['root'] 140 | default_configurations = 'root' 141 | 142 | # things to exclude 143 | exclude_caches = True 144 | exclude_if_present = '.nobackup' 145 | exclude_nodump = True 146 | 147 | There also must be individual settings files for each backup configuration. 148 | They are also a Python files. The above file defines the *root* configuration. 149 | The configuration is described in ``~/.config/emborg/root``, an example of which 150 | is given below. It is designed to back up the whole machine: 151 | 152 | .. code-block:: bash 153 | 154 | # destination repository 155 | repository = 'borgbase:backups' 156 | prefix = '{host_name}-{config_name}-' 157 | 158 | # directories to back up 159 | src_dirs = ['/'] 160 | 161 | # directories to exclude 162 | excludes = [ 163 | "/dev", 164 | "/proc", 165 | "/run", 166 | "/sys", 167 | "/tmp", 168 | "/var/cache", 169 | "/var/tmp", 170 | ] 171 | 172 | Since this configuration needs to back up files that may not be accessible by 173 | normal users, it should be run by the root user. 174 | 175 | Once you have created these files, you can use *Emborg* to perform common tasks 176 | that involve your backups. 177 | 178 | The first step would be to initialize the remote repository. A repository must 179 | be initialized before it can first used. To do so, one uses the :ref:`init 180 | command `: 181 | 182 | .. code-block:: bash 183 | 184 | $ emborg init 185 | 186 | Once the repository is initialized, it is ready for use. The :ref:`create 187 | command ` creates an archive, meaning that it backs up your current 188 | files. 189 | 190 | .. code-block:: bash 191 | 192 | $ emborg create 193 | 194 | Once one or more archives have been created, you can list the available archives 195 | using the :ref:`list command `. 196 | 197 | .. code-block:: bash 198 | 199 | $ emborg list 200 | 201 | The :ref:`manifest or files command ` displays all the files in the 202 | most recent archive. 203 | 204 | .. code-block:: bash 205 | 206 | $ emborg manifest 207 | $ emborg files 208 | 209 | You can restrict the listing to those files contained in the current working 210 | directory using: 211 | 212 | .. code-block:: bash 213 | 214 | $ emborg manifest . 215 | 216 | If you give the name of an archive, it displays all the files in the specified 217 | archive. 218 | 219 | .. code-block:: bash 220 | 221 | $ emborg manifest -a continuum-2019-04-23T18:35:33 222 | 223 | Or, you can give a date, in which case the oldest archive created before that 224 | date is used. 225 | 226 | .. code-block:: bash 227 | 228 | $ emborg manifest -d 2019-04-23 229 | 230 | You can also specify the date and time relative to the current moment: 231 | 232 | .. code-block:: bash 233 | 234 | $ emborg manifest -d 1w 235 | 236 | The :ref:`compare command ` allows you to see and manage the 237 | differences between your local files and those in an archive. You can compare 238 | individual files or entire directories. You can use the date and archive 239 | options to select the particular archive to compare against. You can use the 240 | interactive version of the command to graphically view changes and merge them 241 | back into you local files. 242 | 243 | .. code-block:: bash 244 | 245 | $ emborg compare -i doc/thesis 246 | 247 | The :ref:`restore command ` restores files or directories in place, 248 | meaning it replaces the current version with the one from the archive. 249 | You can also use the date and archive options to select the particular archive 250 | to draw from. 251 | 252 | .. code-block:: bash 253 | 254 | $ cd ~/bin 255 | $ emborg restore accounts.json 256 | 257 | The :ref:`mount command ` creates a directory 'BACKUPS' and then mounts 258 | an archive or the whole repository on this directory. This allows you to move 259 | into the archive or repository, navigating, examining, and retrieving files as 260 | if it were a file system. Again, you can use the date and archive options to 261 | select the particular archive to mount. 262 | 263 | .. code-block:: bash 264 | 265 | $ emborg mount BACKUPS 266 | 267 | The :ref:`umount command ` un-mounts the archive or repository after you 268 | are done with it. 269 | 270 | .. code-block:: bash 271 | 272 | $ emborg umount BACKUPS 273 | 274 | The :ref:`due command ` tells you when the last successful backup was 275 | performed. 276 | 277 | .. code-block:: bash 278 | 279 | $ emborg due 280 | 281 | The :ref:`info command ` shows you information about your repository such 282 | as where it is located and how large it is. 283 | 284 | .. code-block:: bash 285 | 286 | $ emborg info 287 | 288 | The :ref:`help command ` shows you information on how to use 289 | *Emborg*. 290 | 291 | .. code-block:: bash 292 | 293 | $ emborg help 294 | 295 | There are more commands, but the above are the most commonly used. 296 | 297 | 298 | Status 299 | ------ 300 | 301 | *Emborg* includes a utility, *emborg_overdue*, that can be run in a cron script 302 | on either the client or the server machine that notifies you if your back-ups 303 | have not completed successfully in a specified period of time. In addition, 304 | *Emborg* can be configured to update monitoring services such as 305 | HealthChecks.io_ with the status of the backups. 306 | 307 | 308 | Borg 309 | ---- 310 | 311 | *Borg* has more power than what is exposed with *Emborg*. You may 312 | use it directly or through the *Emborg* :ref:`borg command ` when you need 313 | that power. More information can be found at Borg_. 314 | 315 | 316 | Precautions 317 | ----------- 318 | 319 | You should assure you have a backup copy of the encryption key and its 320 | passphrase in a safe place (run 'borg key export' to extract the encryption 321 | keys). This is very important. If the only copy of the encryption credentials 322 | are on the disk being backed up and if that disk were to fail you would not be 323 | able to access your backups. I recommend the use of SpareKeys_ as a way of 324 | assuring that you always have access to the essential information, such as your 325 | *Borg* passphrase and keys, that you would need to get started after 326 | a catastrophic loss of your disk. 327 | 328 | If you keep the passphrase in an *Emborg* configuration file then you should set 329 | the permissions for that file so that it is not readable by others: 330 | 331 | .. code-block:: bash 332 | 333 | chmod 600 ~/.config/emborg/* 334 | 335 | Better is to simply not store the passphrase in *Emborg* configuration files. 336 | You can use the *passcommand* setting for this, or you can use Avendesora_, 337 | which is a flexible password management system. The interface to *Avendesora* is 338 | already built in to *Emborg,* but its use is optional (it need not be 339 | installed). 340 | 341 | It is also best, if it can be arranged, to keep your backups at a remote site so 342 | that your backups do not get destroyed in the same disaster, such as a fire or 343 | flood, that claims your original files. One option is RSync_. Another is 344 | BorgBase_. I have experience with both, and both seem quite good. One I have 345 | not tried is Hetzner_. 346 | 347 | *Borg* supports many different ways of excluding files and directories from your 348 | backup. Thus it is always possible that a small mistake results essential files 349 | from being excluded from your backups. Once you have performed your first 350 | backup you should :ref:`mount ` the most recent archive and then 351 | carefully examine the resulting snapshot and make sure it contains all the 352 | expected files. 353 | 354 | Finally, it is a good idea to practice a recovery. Pretend that you have lost 355 | all your files and then see if you can do a restore from backup. Doing this and 356 | working out the kinks before you lose your files can save you if you ever do 357 | lose your files. 358 | 359 | 360 | Issues 361 | ------ 362 | 363 | Please ask questions or report problems on GitHub_. 364 | 365 | 366 | Contents 367 | -------- 368 | 369 | .. toctree:: 370 | :maxdepth: 1 371 | 372 | installing 373 | commands 374 | configuring 375 | monitoring 376 | accessories 377 | examples 378 | api 379 | releases 380 | 381 | * :ref:`genindex` 382 | 383 | .. _Assimilate: https://assimilate.readthedocs.io 384 | .. _Avendesora: https://avendesora.readthedocs.io 385 | .. _BorgBase: https://www.borgbase.com 386 | .. _Borg: https://borgbackup.readthedocs.io 387 | .. _BorgMatic: https://torsion.org/borgmatic 388 | .. _Duplicity: http://duplicity.nongnu.org 389 | .. _GitHub: https://github.com/KenKundert/emborg/issues 390 | .. _HealthChecks.io: https://healthchecks.io 391 | .. _Hetzner: https://www.hetzner.com/storage/storage-box 392 | .. _RSync: https://www.rsync.net/products/attic.html 393 | .. _SpareKeys: https://github.com/kalekundert/sparekeys 394 | .. _Vorta: https://github.com/borgbase/vorta 395 | -------------------------------------------------------------------------------- /doc/installing.rst: -------------------------------------------------------------------------------- 1 | .. _installing_emborg: 2 | 3 | Getting Started 4 | =============== 5 | 6 | Installing 7 | ---------- 8 | 9 | Many Linux distributions include *Borg* in their package managers. In Fedora it 10 | is referred to as *borgbackup*. In this case you would install *borg* by running 11 | the following: 12 | 13 | .. code-block:: bash 14 | 15 | $ sudo dnf install borgbackup 16 | 17 | Alternately, you can download a precompiled version from `Borg Github Releases 18 | `_, which allows you to install 19 | Borg as an unprivileged user. You can do so with following commands (they may 20 | need to be adjusted to get the latest version): 21 | 22 | .. code-block:: bash 23 | 24 | $ cd ~/bin 25 | $ wget https://github.com/borgbackup/borg/releases/download/1.2.6/borg-linux64 26 | $ wget https://github.com/borgbackup/borg/releases/download/1.2.6/borg-linux64.asc 27 | $ gpg --recv-keys 6D5BEF9ADD2075805747B70F9F88FB52FAF7B393 28 | $ gpg --verify borg-linux64.asc 29 | $ rm borg-linux64.asc 30 | $ chmod 755 borg-linux64 31 | 32 | Finally, you can install it using `pip 33 | `_: 34 | 35 | .. code-block:: bash 36 | 37 | $ pip install borgbackup 38 | 39 | Download and install *Emborg* as follows (requires Python3.6 or better): 40 | 41 | .. code-block:: bash 42 | 43 | $ pip install emborg 44 | 45 | Or, if you want the development version, use: 46 | 47 | .. code-block:: bash 48 | 49 | $ git clone https://github.com/KenKundert/emborg.git 50 | $ pip install ./emborg 51 | 52 | You may also need to install and configure either a notification daemon or 53 | a mail daemon. This allows errors to be reported when you are not running 54 | *Emborg* in a terminal. More information can be found by reading about the 55 | :ref:`notifier` and :ref:`notify` *Emborg* settings. 56 | 57 | 58 | Configuring Emborg to Backup A Home Directory 59 | ---------------------------------------------- 60 | 61 | The basic idea behind *Emborg* is that you place all information relevant to 62 | your backups in two configuration files, which allows you to use *Emborg* to 63 | perform tasks without re-specifying that information. Emborg allows you to have 64 | any number of setups, which you might want if you wanted to backup to multiple 65 | repositories for redundancy or if you want to use different rules for different 66 | sets of files. Regardless, you use a separate configuration for each set up, 67 | plus there is a common configuration file shared by all setups. You are free to 68 | place most settings in either file, whichever is most convenient. All the 69 | configuration files are placed in ~/.config/emborg. If you run *Emborg* without 70 | creating your configuration files, *Emborg* will create some starter files for 71 | you. A configuration is specified using Python, thus the content of these files 72 | is formatted as Python code and is read by a Python interpreter. 73 | 74 | As a demonstration on how to configure *Emborg*, imagine wanting to back up your 75 | home directory in two ways. First, you want to backup the files to an off-site 76 | server. Here the expectation is that you would backup once a day on average and 77 | you would do so interactively so that you can choose an appropriate time. 78 | Second, you have some free space on your machine that you would like to dedicate 79 | to recent snapshots of your files. The idea is that you find that you 80 | occasionally overwrite or delete files that you just spent time creating, and 81 | you want to run local backups every 10-15 minutes so that you can easily recover 82 | these files. To accomplish these two things, you need three configuration 83 | files. 84 | 85 | 86 | Shared Settings 87 | ^^^^^^^^^^^^^^^ 88 | 89 | The first file is the shared configuration file: 90 | 91 | .. code-block:: python 92 | 93 | configurations = 'backups snapshots' 94 | default_configuration = 'backups' 95 | 96 | This is basically the minimum you can give. Your two configurations are listed 97 | in *configurations*. It could be a list of strings, but you can also give 98 | a single string, in which case the string is split on white space. Then you 99 | specify your default configuration. In this example *backups* is to be run 100 | interactively and *snapshots* is to be run on a schedule by *cron*, so the 101 | default is set to *backups* to make it easier to run interactively. 102 | 103 | 104 | Configuration for a Remote Repository: *backups* 105 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 106 | 107 | The second file is the configuration file for *backups*: 108 | 109 | .. code-block:: python 110 | 111 | repository = 'backups:archives' 112 | prefix = '{host_name}-' 113 | encryption = 'keyfile' 114 | passphrase = 'crone excess mandate bedpost' 115 | 116 | src_dirs = '~' 117 | excludes = ''' 118 | ~/.cache 119 | **/*~ 120 | **/.git 121 | **/__pycache__ 122 | **/.*.swp 123 | ''' 124 | exclude_if_present = '.nobackup' 125 | 126 | check_after_create = 'latest' 127 | prune_after_create = True 128 | compact_after_delete = True 129 | keep_daily = 7 130 | keep_weekly = 4 131 | keep_monthly = 12 132 | keep_yearly = 2 133 | 134 | This configuration assumes that you have a *backups* entry in your SSH config 135 | file that contains the appropriate user name, host name, port number, and such 136 | for the server that contains your remote repository. It also assumes that you 137 | have shared an SSH key with this server so you do not need to specify a password 138 | each time you back up, and that that key is pre-loaded into your SSH agent. The 139 | repository is actually in the *archives* directory on that server, and each 140 | back-up archive will be prefixed with your local host name, allowing you to 141 | share this repository with other machines. 142 | 143 | You specify what to backup using *src_dirs* and what not to backup using 144 | *excludes*. Nominally both *src_dirs* and *excludes* take lists of strings, but 145 | you can also specify them using a single string, in which case the strings are 146 | broken into individual lines, any blank lines or lines that begin with ``#`` are 147 | ignored, and then the white space is removed from the front and back of each 148 | line. 149 | 150 | This configuration file ends with settings that tell *Emborg* to run *check* and 151 | *prune* operations after creating a backup, and it gives the desired prune 152 | schedule. 153 | 154 | This is just an example, and a rather minimal one at that. You should not use 155 | it without understanding each of the settings. The *encryption* setting is 156 | a particularly important one for you to understand and set properly. More 157 | comprehensive information about configuring *Emborg* can be found in the section 158 | on :ref:`configuring_emborg`. 159 | 160 | With this configuration, you can now initialize your repository and use it to 161 | perform backups. If the repository does not yet exist, initialize it using: 162 | 163 | .. code-block:: bash 164 | 165 | $ emborg init 166 | 167 | Then perform a back up using: 168 | 169 | .. code-block:: bash 170 | 171 | $ emborg create 172 | 173 | or simply: 174 | 175 | .. code-block:: bash 176 | 177 | $ emborg 178 | 179 | This works because *create* is the default action and *backups* is the default 180 | configuration. 181 | 182 | Then, you can convince yourself it is working as expected by moving a directory 183 | out of the way and using *Emborg* to restore it: 184 | 185 | .. code-block:: bash 186 | 187 | $ mv bin bin-saved 188 | $ emborg restore bin 189 | 190 | 191 | Configuration for a Local Repository: *snapshots* 192 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 193 | 194 | The third file is the configuration file for *snapshots*: 195 | 196 | .. code-block:: python 197 | 198 | repository = '/mnt/snapshots/{user_name}' 199 | prefix = '{config_name}-' 200 | encryption = 'none' 201 | 202 | src_dirs = '~' 203 | excludes = ''' 204 | ~/.cache 205 | **/*~ 206 | **/.git 207 | **/__pycache__ 208 | **/.*.swp 209 | ''' 210 | prune_after_create = True 211 | compact_after_delete = True 212 | keep_within = '1d' 213 | 214 | In this case the repository is on the local machine and it is not encrypted. It 215 | again backs up your home directory, but for this configuration the archives are 216 | only kept for a day. 217 | 218 | The repository must be initialized before it can be used: 219 | 220 | .. code-block:: bash 221 | 222 | $ emborg -c snapshots init 223 | 224 | Here the desired configuration was specified because it is not the default. Now, 225 | a *cron* entry can be created using ``crontab -e`` that creates a snapshot every 226 | 10 minutes: 227 | 228 | .. code-block:: text 229 | 230 | */10 * * * * emborg --config snapshots --mute create 231 | 232 | Once it has run, you can pull a file from the latest snapshot using: 233 | 234 | .. code-block:: bash 235 | 236 | $ emborg -c snapshots restore passwords.gpg 237 | 238 | 239 | Overdue Backups 240 | ^^^^^^^^^^^^^^^ 241 | 242 | *Emborg* allows you to easily determine when your files were last backed up 243 | using: 244 | 245 | .. code-block:: bash 246 | 247 | $ emborg due 248 | 249 | However, you must remember to run this command. *Emborg* also provides 250 | :ref:`emborg-overdue ` to provide automated reminders. You 251 | configure *emborg-overdue* using a configuration file: 252 | ~/.config/emborg/overdue.conf. For example: 253 | 254 | .. code-block:: python 255 | 256 | default_maintainer = 'me@mydomain.com' 257 | dumper = 'me@mydomain.com' 258 | default_max_age = 36 # hours 259 | root = '~/.local/share/emborg' 260 | repositories = [ 261 | dict(host='laptop (snapshots)', path='snapshots.lastbackup', max_age=0.2), 262 | dict(host='laptop (backups)', path='backups.lastbackup'), 263 | ] 264 | 265 | Then you would configure *cron* to run *emborg-overdue* using something like: 266 | 267 | .. code-block:: text 268 | 269 | 00 * * * * ~/.local/bin/emborg-overdue --quiet --mail 270 | 271 | This runs *emborg-overdue* every hour on the hour, and it reports any delinquent 272 | backups by sending mail to the appropriate maintainer (the message is sent from 273 | the *dumper*). You can specify any number of repositories to check, and for 274 | each repository you can specify *host* (a descriptive name), *path* (the path to 275 | the repository from the *root* directory, a *max_age* in hours, and 276 | a *maintainer*. You can also specify defaults for the *maintainer* and 277 | *max_age*. When run, it checks the age of each repository and sends email to 278 | the appropriate maintainer if it exceeds the maximum allowed age. 279 | 280 | In this example the actual repository is not checked directly, rather the 281 | *lastbackup* file is checked. This is a file that is updated by *Emborg* after 282 | every back up. This file is found in the *Emborg* output directory. Every time 283 | *Emborg* runs it creates a log file that can also be found in this directory. 284 | That logfile can be viewed directly, or you can view it using the *log* command: 285 | 286 | .. code-block:: bash 287 | 288 | $ emborg log 289 | 290 | 291 | Configuring Emborg to Backup an Entire Machine 292 | ---------------------------------------------- 293 | 294 | The primary difference between this example and the previous is that *Emborg* 295 | needs to be configured and run by *root*. This allows all the files on the 296 | machine to be backed up regardless of who owns them. Other than being root, the 297 | mechanics are very much the same. 298 | 299 | To start, run *emborg* as root to create the initial configuration files: 300 | 301 | .. code-block:: bash 302 | 303 | # emborg 304 | 305 | This creates the /root/.config/emborg directory in the root account and 306 | populates it with three files: *settings*, *root*, *home*. You can delete *home* 307 | and remove the reference to it in *settings*, leaving only: 308 | 309 | .. code-block:: python 310 | 311 | configurations = 'root' 312 | default_configuration = 'root' 313 | 314 | This assumes that most of the settings will be placed in *root*: 315 | 316 | .. code-block:: python 317 | 318 | repository = 'backups:backups/{host_name}' 319 | prefix = '{config_name}-' 320 | passphrase = 'western teaser landfall spearhead' 321 | encryption = 'repokey' 322 | 323 | src_dirs = '/' 324 | excludes = ''' 325 | /dev 326 | /home/*/.cache 327 | /proc 328 | /root/.cache 329 | /run 330 | /sys 331 | /tmp 332 | /var 333 | ''' 334 | 335 | check_after_create = 'latest' 336 | compact_after_delete = True 337 | prune_after_create = True 338 | keep_daily = 7 339 | keep_weekly = 4 340 | keep_monthly = 12 341 | 342 | Again, this is a rather minimal example. In this case, *repokey* is used as the 343 | encryption method, which is only suitable if the repository is on a server you 344 | control. 345 | 346 | When backing up the root file system it is important to exclude directories that 347 | cannot or should not be backed up. Those include: /dev, /proc, /run, /sys, and 348 | /tmp. 349 | 350 | As before you need to initialize the repository before it can be used: 351 | 352 | .. code-block:: bash 353 | 354 | # emborg init 355 | 356 | To assure that the backups are run daily, the following is added to 357 | /etc/cron.daily/emborg: 358 | 359 | .. code-block:: bash 360 | 361 | #/bin/sh 362 | # Run root backups 363 | 364 | emborg --mute --config root create 365 | 366 | This is preferred for laptops because cron.daily is guaranteed to run each day 367 | as long as machine is turned on for any reasonable length of time. 368 | -------------------------------------------------------------------------------- /doc/monitoring.rst: -------------------------------------------------------------------------------- 1 | .. _utilities: 2 | .. _monitoring: 3 | 4 | Monitoring 5 | ========== 6 | 7 | 8 | Log Files 9 | --------- 10 | 11 | When there are problems, the log file can help you understand what is going 12 | wrong. *Emborg* writes a log file to the *Emborg* data directory. On Linux 13 | systems that would be `~/.local/share/emborg`. Other systems use more awkward 14 | locations, so *Emborg* allows you to specify the data directory using the 15 | `XDG_DATA_HOME` environment variable. If `XDG_DATA_HOME` is set to 16 | `/home/$HOME/.local/share`, then the *Emborg* log files will be written to 17 | `/home/$HOME/.local/share/emborg`, as on Linux systems. 18 | 19 | Besides visiting the *Emborg* data directory and viewing the log file directly, 20 | you can use the :ref:`log command ` to view the log file. 21 | 22 | *Emborg* overwrites its log file every time it runs. You can use :ref:`ntlog 23 | ` to gather log files as they are created into a composite log 24 | file. 25 | 26 | 27 | Due and Info 28 | ------------ 29 | 30 | The :ref:`due ` and :ref:`info ` commands allow you to interactively 31 | check on the current status of your backups. Besides the :ref:`create ` 32 | command, it is good hygiene to run the :ref:`prune `, :ref:`compact 33 | ` and :ref:`check ` on a regular basis. Either the :ref:`due 34 | ` or :ref:`info ` command can be used to determine when each were 35 | last run. 36 | 37 | 38 | .. _emborg_overdue: 39 | 40 | Overdue 41 | ------- 42 | 43 | .. _server_overdue: 44 | 45 | Checking for Overdue Backups from the Server 46 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 47 | 48 | *Emborg* contains an additional executable, *emborg-overdue*, that can be run on 49 | the destination server to determine whether the backups have been performed 50 | recently. It reads its own settings file in `overdue.conf`, contained in the 51 | :ref:`Emborg configuration directory `, that is also 52 | a Python file and may contain the following settings: 53 | 54 | .. code-block:: text 55 | 56 | default_maintainer (email address -- mail is sent to this person upon failure) 57 | default_max_age (hours) 58 | dumper (email address -- mail is sent from this person) 59 | root (default directory for repositories) 60 | status_message (a template for the message to be printed for each repository) 61 | repositories (string or array of dictionaries) 62 | 63 | Here is an example config file: 64 | 65 | .. code-block:: python 66 | 67 | default_maintainer = 'root@continuum.com' 68 | dumper = 'dumper@continuum.com' 69 | default_max_age = 12 # hours 70 | root = '/mnt/borg-backups/repositories' 71 | colorscheme = 'dark' 72 | status_message = "{host}: {age} ago{overdue: — PAST DUE}" 73 | repositories = [ 74 | dict(host='mercury (/)', path='mercury-root-root'), 75 | dict(host='venus (/)', path='venus-root-root'), 76 | dict(host='earth (/)', path='earth-root-root'), 77 | dict(host='mars (/)', path='mars-root-root'), 78 | dict(host='jupiter (/)', path='jupiter-root-root'), 79 | dict(host='saturn (/)', path='saturn-root-root'), 80 | dict(host='uranus (/)', path='uranus-root-root'), 81 | dict(host='neptune (/)', path='neptune-root-root'), 82 | dict(host='pluto (/)', path='pluto-root-root'), 83 | ] 84 | 85 | The dictionaries in *repositories* can contain the following fields: *host*, 86 | *path*, *maintainer*, *max_age*. 87 | 88 | *host*: 89 | An arbitrary string that is used as description of the repository. It is 90 | included in the email that is sent when problems occur to identify the 91 | backup and so should be unique. It is a good idea for it to contain both 92 | the host name and the source directory being backed up. 93 | *path*: 94 | Is either the archive name or a full absolute path to the archive. The 95 | modification time of the target of this path is used as the time of the last 96 | backup. If *path* is an absolute path, it is used, otherwise it is added to 97 | the end of *root*. 98 | 99 | If the path contains a colon (‘:’), then everything before the colon is 100 | taken to be an SSH hostname and everything after the colon is assumed to be 101 | the name of the *emborg-overdue* command on that local machine without 102 | arguments. In most cases the colon will be the last character of the path, 103 | in which case the command name is assumed to be ‘emborg-overdue’. This 104 | command is run on the remote host and the results reported locally. The 105 | version of *emborg* on the remote host must be 1.41 or greater. 106 | *maintainer*: 107 | An email address, an email is sent to this address if there is an issue. 108 | *max_age* is the number of hours that may pass before an archive is 109 | considered overdue. 110 | *max_age*: 111 | The maximum age in hours. If the back-up occurred more than this many hours 112 | in the past it is considered over due. 113 | 114 | *repositories* can also be specified as multi-line string: 115 | 116 | .. code-block:: python 117 | 118 | repositories = """ 119 | # HOST | NAME or PATH | MAINTAINER | MAXIMUM AGE (hours) 120 | mercury (/) | mercury-root-root | | 121 | venus (/) | venus-root-root | | 122 | earth (/) | earth-root-root | | 123 | mars (/) | mars-root-root | | 124 | jupiter (/) | jupiter-root-root | | 125 | saturn (/) | saturn-root-root | | 126 | uranus (/) | uranus-root-root | | 127 | neptune (/) | neptune-root-root | | 128 | pluto (/) | pluto-root-root | | 129 | """ 130 | 131 | If *repositories* is a string, it is first split on newlines, anything beyond 132 | a # is considered a comment and is ignored, and the finally the lines are split 133 | on '|' and the 4 values are expected to be given in order. If the *maintainer* 134 | is not given, the *default_maintainer* is used. If *max_age* is not given, the 135 | *default_max_age* is used. 136 | 137 | There are some additional settings available: 138 | 139 | *default_maintainer*: 140 | Email address of the account running the checks. This will be the sender 141 | address on any email sent as a result of an over due back-up. 142 | *dumper*: 143 | Email address of the account monitoring the checks. This will be the 144 | recipient address on any email sent as a result of an over due back-up. 145 | *root*: 146 | The directory used as the root when converting relative paths given in 147 | *repositories* to absolute paths. By default this will be the *Emborg* log 148 | file directory. 149 | *default_max_age*: 150 | The default maximum age in hours. It is used if a maximum age is not given 151 | for a particular repository. 152 | *colorscheme*: 153 | The color scheme of your terminal. May be "dark" or "light" or None. If 154 | None, the output is not colored. 155 | *status_message*: 156 | The format of the summary for each host. The string may contain keys within 157 | braces that will be replaced before output. The following keys are 158 | supported: 159 | 160 | | *host*: replaced by the host field from the config file, a string. 161 | | *max_age*: replaced by the max_age field from the config file, a float. 162 | | *mtime*: replaced by modification time, a datetime object. 163 | | *hours*: replaced by the number of hours since last update, a float. 164 | | *age*: replaced by time since last update, a string. 165 | | *overdue*: is the back-up overdue, a boolean. 166 | | *locked*: is the back-up currently active, a boolean. 167 | 168 | The message is a Python formatted string, and so the various fields can include 169 | formatting directives. For example: 170 | 171 | - strings than include field width and justification, ex. {host:>20} 172 | - floats can include width, precision and form, ex. {hours:0.1f} 173 | - datetimes can include Arrow formats, ex: {mtime:DD MMM YY @ H:mm A} 174 | - booleans can include true/false strings: ex. {overdue:PAST DUE!/current} 175 | 176 | To run the program interactively, just make sure *emborg-overdue* has been 177 | installed and is on your path. Then type: 178 | 179 | .. code-block:: bash 180 | 181 | $ emborg-overdue 182 | 183 | It is also common to run *emborg-overdue* on a fixed schedule from cron. To do 184 | so, run: 185 | 186 | .. code-block:: bash 187 | 188 | $ crontab -e 189 | 190 | and add something like the following: 191 | 192 | .. code-block:: text 193 | 194 | 34 5 * * * ~/.local/bin/emborg-overdue --quiet --mail 195 | 196 | or: 197 | 198 | .. code-block:: text 199 | 200 | 34 5 * * * ~/.local/bin/emborg-overdue --quiet --notify 201 | 202 | to your crontab. 203 | 204 | The first example runs emborg-overdue at 5:34 AM every day. The use of the 205 | ``--mail`` option causes *emborg-overdue* to send mail to the maintainer when 206 | backups are found to be overdue. 207 | 208 | .. note:: 209 | 210 | By default Linux machines are not configured to send email. If you are 211 | using the ``--mail`` option to *emborg-overdue* be sure that to check that 212 | it is working. You can do so by sending mail to your self using the *mail* 213 | or *mailx* command. If you do not receive your test message you will need 214 | to set up email forwarding on your machine. You can do so by installing and 215 | configuring `PostFix as a null client 216 | `_. 217 | 218 | The second example uses ``--notify``, which sends a notification if a back-up is 219 | overdue and there is not access to the tty (your terminal). 220 | 221 | Alternately you can run *emborg-overdue* from cron.daily (described in the 222 | :ref:`root example `). 223 | 224 | 225 | .. _client_overdue: 226 | 227 | Checking for Overdue Backups from the Client 228 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 229 | 230 | *emborg-overdue* can also be configured to run on the client. This can be used 231 | when you do not control the server and so cannot run *emborg-overdue* there. 232 | The configuration is identical, except you give the path to the *latest.nt* 233 | file. For example: 234 | 235 | .. code-block:: python 236 | 237 | default_maintainer = 'me@continuum.com' 238 | dumper = 'me@continuum.com' 239 | default_max_age = 12 # hours 240 | root = '~/.local/share/emborg' 241 | repositories = [ 242 | dict(host='earth (cache)', path='cache.latest.nt', max_age=0.2), 243 | dict(host='earth (home)', path='home.latest.nt'), 244 | dict(host='sol', path='sol:'), 245 | ] 246 | 247 | Notice the last entry, the one for *sol*. Its path contains a colon, so it is 248 | a remote check. The others are local checks. The remote check splits *path* at 249 | the colon. In this case, the split gives 'sol' and ''. The first component is 250 | taken to be a host name and the second is the name of the emborg-overdue command 251 | on that host. In this case the second component is empty so *emborg-overdue* is 252 | used. On the remote checks, the *emborg-overdue* command is run remotely on the 253 | specified host and the results are included in the output. This generally 254 | requires that you have the SSH keys for the remote host in your SSH agent, which 255 | is generally not the case when *emborg-overdue* is being run from cron. In this 256 | case you should use the ``--local`` option to suppress remote queries. 257 | 258 | 259 | .. _monitoring_services: 260 | 261 | Monitoring Services 262 | ------------------- 263 | 264 | Various monitoring services are available on the web. You can configure 265 | *Emborg* to notify them when back-up jobs have started and finished. These 266 | services allow you to monitor many of your routine tasks and assure they have 267 | completed recently and successfully. 268 | 269 | There are many such services available and they are not difficult to add. If 270 | the service you prefer is not currently available, feel free to request it on 271 | `Github `_ or add it yourself and 272 | issue a pull request. 273 | 274 | .. _cronhub: 275 | 276 | CronHub.io 277 | ~~~~~~~~~~ 278 | 279 | When you sign up with `cronhub.io `_ and configure the 280 | health check for your *Emborg* configuration, you will be given a UUID (a 32 281 | digit hexadecimal number partitioned into 5 parts by dashes). Add that to the 282 | following setting in your configuration file: 283 | 284 | .. code-block:: python 285 | 286 | cronhub_uuid = '51cb35d8-2975-110b-67a7-11b65d432027' 287 | 288 | If given, this setting should be specified on an individual configuration. It 289 | causes a report to be sent to *CronHub* each time an archive is created. 290 | A successful report is given if *Borg* returns with an exit status of 0 or 1, 291 | which implies that the command completed as expected, though there might have 292 | been issues with individual files or directories. If *Borg* returns with an 293 | exit status of 2 or greater, a failure is reported. 294 | 295 | 296 | .. _healthchecks: 297 | 298 | HealthChecks.io 299 | ~~~~~~~~~~~~~~~ 300 | 301 | When you sign up with `healthchecks.io `_ and configure 302 | the health check for your *Emborg* configuration, you will be given a UUID (a 32 303 | digit hexadecimal number partitioned into 5 parts by dashes). Add that to the 304 | following setting in your configuration file: 305 | 306 | .. code-block:: python 307 | 308 | healthchecks_uuid = '51cb35d8-2975-110b-67a7-11b65d432027' 309 | 310 | If given, this setting should be specified on an individual configuration. It 311 | causes a report to be sent to *HealthChecks* each time an archive is created. 312 | A successful report is given if *Borg* returns with an exit status of 0 or 1, 313 | which implies that the command completed as expected, though there might have 314 | been issues with individual files or directories. If *Borg* returns with an 315 | exit status of 2 or greater, a failure is reported. 316 | -------------------------------------------------------------------------------- /doc/releases.rst: -------------------------------------------------------------------------------- 1 | Releases 2 | ======== 3 | 4 | Latest development release 5 | -------------------------- 6 | | Version: 1.41 7 | | Released: 2024-11-11 8 | 9 | 1.41 (2024-11-11) 10 | ----------------- 11 | - When *Emborg* encounters an error when operating on a composite configuration 12 | it terminates the problematic configuration and moves to the next. Previously 13 | it would exit without attempting the remaining configs. 14 | - :ref:`emborg-overdue ` can now run an *emborg-overdue* process 15 | on a remote host and include the results in its report. 16 | - message templates for :ref:`emborg-overdue ` can now contain 17 | the ``locked`` field. 18 | - Allow location of config and data directory to be overridden with 19 | `XDG_CONFIG_HOME` and `XDG_DATA_HOME` environment variables. This replaces an 20 | earlier behavior that simply treated `~/.config/` as the configuration 21 | directory if it exists. No provision was made previously to support an 22 | alternative data directory. 23 | 24 | .. note:: 25 | 26 | If your configuration files are in `~/.config/emborg` and you are not on 27 | a Linux system you will now have to set `XDG_CONFIG_HOME` to 28 | `$HOME/.config`. 29 | 30 | 31 | 1.40 (2024-08-05) 32 | ----------------- 33 | - Enhance :ref:`emborg-overdue ` command. 34 | - Fix bug in :ref:`restore ` when there are multiple roots. 35 | 36 | 37 | 1.39 (2024-04-29) 38 | ----------------- 39 | - Add date of last check to output of :ref:`info ` command. 40 | - Added :ref:`cmd_name` setting. 41 | - Miscellaneous refinements. 42 | 43 | 44 | 1.38 (2023-11-04) 45 | ----------------- 46 | - Added ‘last checked date’ reporting to :ref:`due command`. 47 | - Do not run :ref:`check --repair ` and :ref:`compact ` commands 48 | if `--dry-run` is requested. 49 | - Pass output of *Borg* *create* command to hooks to allow it to be reported to 50 | healthchecks.io_. 51 | 52 | 53 | 1.37 (2023-05-18) 54 | ----------------- 55 | - Add missing dependency. 56 | 57 | 58 | 1.36 (2023-05-15) 59 | ----------------- 60 | This release provides new mechanisms that allow you to monitor your pruning and 61 | compaction operations to help assure that these activities are not neglected. 62 | Both a prune and a compact operation must be performed to release disk space by 63 | eliminating expired archives. The combination of these to operations is 64 | referred to by *Emborg* as a squeeze. 65 | 66 | - specifying an integer for ``--date`` now finds archive by index. 67 | - :ref:`due ` and :ref:`info ` commands now report the latest 68 | :ref:`prune ` and :ref:`compact ` operations as well as the 69 | latest :ref:`create ` operation. 70 | 71 | .. note:: 72 | 73 | If you use :ref:`emborg-overdue ` from the client you will 74 | need to change the paths you specify in *overdue.conf*. They now need to 75 | end in ``.latest.nt`` rather than ``.lastbackup``. 76 | 77 | .. note:: 78 | 79 | If you use :ref:`borg space`, you will need to upgrade to version 2. 80 | 81 | 82 | 1.35 (2023-03-20) 83 | ----------------- 84 | - Improved the time resolution in :ref:`due ` command. 85 | - Added *si* format to :ref:`manifest ` command. 86 | - Allow *config_dir* to be specified through API. 87 | 88 | 89 | 1.34 (2022-11-03) 90 | ----------------- 91 | - Added ability to apply the :ref:`info ` command to a particular archive. 92 | 93 | 94 | 1.33 (2022-10-22) 95 | ----------------- 96 | - Added :ref:`compare ` command. 97 | - Added :ref:`manage_diffs_cmd` and :ref:`report_diffs_cmd` settings. 98 | - Allow ~/.config/emborg to always hold settings files if user prefers. 99 | 100 | 101 | 1.32 (2022-04-01) 102 | ----------------- 103 | - Fixed issues associated with :ref:`compact_after_delete` setting. 104 | 105 | 106 | 1.31 (2022-03-21) 107 | ----------------- 108 | - Enhanced *Emborg* to support new Borg 1.2 features. 109 | 110 | - Added :ref:`compact command ` 111 | - Added :ref:`compact_after_delete`, :ref:`chunker_params`, :ref:`sparse`, 112 | :ref:`threshold`, :ref:`upload_ratelimit`, :ref:`upload_buffer` settings. 113 | 114 | - Added the :ref:`run_before_borg and run_after_borg ` 115 | settings. 116 | - Added the ``--cache-only`` option and the ability to delete multiple archives 117 | at one time to the :ref:`delete command `. 118 | 119 | 120 | 1.30 (2022-01-04) 121 | ----------------- 122 | - Fix some issues with relative paths. 123 | 124 | 125 | 1.29 (2021-12-18) 126 | ----------------- 127 | - Do not signal failure to hooks if Borg completes normally, even if there were 128 | warnings. 129 | - Return an exit status of 1 if *Emborg* runs to completion but with exceptions, 130 | and 2 if it cannot complete normally due to a error or errors. 131 | 132 | 133 | 1.28 (2021-11-06) 134 | ----------------- 135 | - Suppress log file generation for 136 | :ref:`configs `, 137 | :ref:`due `, 138 | :ref:`help `, 139 | :ref:`log `, 140 | :ref:`settings ` and 141 | :ref:`version ` commands. 142 | - Add *version* to the API. 143 | 144 | 145 | 1.27 (2021-09-21) 146 | ----------------- 147 | - Improve the logging for composite configurations. 148 | - Add support for `Borg-Space `_, 149 | a utility that allows you to track and plot disk space usage for your *Borg* 150 | repositories over time. 151 | 152 | 153 | 1.26 (2021-09-03) 154 | ----------------- 155 | - Improve the tests. 156 | - Allow access to names of child configs through API. 157 | 158 | 159 | 1.25 (2021-08-28) 160 | ----------------- 161 | - Added the :ref:`compare command `. 162 | - Added the :ref:`manage_diffs_cmd` and :ref:`report_diffs_cmd` settings. 163 | - Added the 164 | :ref:`run_before_first_backup ` and 165 | :ref:`run_after_last_backup ` settings. 166 | - Allow files listed by :ref:`manifest ` command to be constrained to 167 | those contained within a path. 168 | - Allow relative dates to be specified on the :ref:`extract `, 169 | :ref:`manifest `, :ref:`mount ` and :ref:`restore ` 170 | commands. 171 | - Allow *BORG_PASSPHRASE*, *BORG_PASSPHRASE_FD*, or *BORG_PASSCOMMAND* to 172 | dominate over *Emborg* passphrase settings. 173 | 174 | 175 | 1.24 (2021-07-05) 176 | ----------------- 177 | - Added *healthchecks_url* and *cronhub_url* settings. 178 | 179 | 180 | 1.23 (2021-07-01) 181 | ----------------- 182 | - Fix missing dependency. 183 | 184 | 185 | 1.22 (2021-06-21) 186 | ----------------- 187 | - Added support for `healthchecks.io `_ monitoring 188 | service. 189 | - Added support for `cronhub.io `_ monitoring service. 190 | 191 | 192 | 1.21 (2021-03-11) 193 | ----------------- 194 | - Made extensive changes to :ref:`manifest ` command to make it more 195 | flexible 196 | 197 | - colorized the output based on file health (green implies healthy, red 198 | implies unhealthy) 199 | - added ``--no-color`` option to :ref:`manifest ` to suppress 200 | colorization 201 | - added :ref:`colorscheme` setting. 202 | - added :ref:`manifest_default_format` setting. 203 | - added support for *Borg* *list* command field names for both reporting 204 | and sorting. 205 | - added *Emborg* variants to some of the *Borg* field names. 206 | - added ``--show-formats`` command line option. 207 | - added ``--format`` command line option. 208 | - added ``--sort-by-field`` command line option. 209 | - change predefined formats to use fields that render faster 210 | 211 | .. warning:: 212 | These changes are not backward compatible. If you have 213 | a :ref:`manifest_formats` setting from a previous version, it may 214 | need to be updated. 215 | 216 | - It is now an error for :ref:`prefix` setting to contain ``{{now}}``. 217 | - :ref:`Settings ` command will now print a single setting value 218 | if its name is given. 219 | 220 | 221 | 1.20 (2021-02-13) 222 | ----------------- 223 | 224 | - Add ``--progress`` command-line option and :ref:`show_progress` option to 225 | the :ref:`create ` command. 226 | 227 | 228 | 1.19 (2021-01-02) 229 | ----------------- 230 | - Added ``--list`` command-line option to the :ref:`prune ` command. 231 | 232 | 233 | 1.18 (2020-07-19) 234 | ----------------- 235 | - Added ``--repo`` option to :ref:`delete ` command. 236 | - Added ``--relocated`` global command-line option. 237 | - *Emborg* now automatically confirms to *Borg* that you know what you are doing 238 | when you delete a repository or repair an archive. 239 | 240 | 241 | 1.17 (2020-04-15) 242 | ----------------- 243 | - :ref:`Borg ` command allows archive to be added to ``@repo``. 244 | - Added :ref:`encoding` setting. 245 | 246 | 247 | 1.16 (2020-03-17) 248 | ----------------- 249 | - Refinements and bug fixes. 250 | 251 | 252 | 1.15 (2020-03-06) 253 | ----------------- 254 | - Improve messaging from *emborg-overdue* 255 | - :ref:`Configs ` command now outputs default configuration too. 256 | - Some commands now use first subconfig when run with a composite configuration 257 | rather than terminating with an error. 258 | - Added :ref:`show_stats` setting. 259 | - Added ``--stats`` option to :ref:`create `, :ref:`delete ` and 260 | :ref:`prune ` commands. 261 | - Added ``--list`` option to :ref:`create `, :ref:`extract ` 262 | and :ref:`restore ` commands. 263 | - Added sorting and formatting options to :ref:`manifest ` command. 264 | - Added :ref:`manifest_formats` setting. 265 | - Renamed ``--trial-run`` option to ``--dry-run`` to be more consistent with 266 | *Borg*. 267 | - Add *files* and *f* aliases to :ref:`manifest ` command. 268 | - Added :ref:`working_dir` setting. 269 | - Added :ref:`do_not_expand` setting. 270 | - Added :ref:`exclude_nodump` setting 271 | - Added :ref:`patterns` and :ref:`patterns_from` settings. 272 | - *Emborg* lock file is now ignored if the process it references is no longer 273 | running 274 | - Support ``--repair`` option on :ref:`check command `. 275 | 276 | 277 | 1.14 (2019-12-31) 278 | ----------------- 279 | - Remove debug message accidentally left in *emborg-overdue* 280 | 281 | 282 | 1.13 (2019-12-31) 283 | ----------------- 284 | - Enhance *emborg-overdue* to work on clients as well as servers 285 | 286 | 287 | 1.12 (2019-12-25) 288 | ----------------- 289 | - Added :ref:`default_mount_point` setting. 290 | - Fixed some issues with :ref:`borg ` command. 291 | - Added ``--oldest`` option to :ref:`due ` command. 292 | 293 | 294 | 1.11 (2019-11-27) 295 | ----------------- 296 | - Bug fix release. 297 | 298 | 299 | 1.10 (2019-11-11) 300 | ----------------- 301 | - Bug fix release. 302 | 303 | 304 | 1.9 (2019-11-08) 305 | ---------------- 306 | - Added ability to check individual archives to the :ref:`check ` 307 | command. 308 | - Made latest archive the default for :ref:`check ` command. 309 | - Allow :ref:`exclude_from ` setting to be a list of file names. 310 | 311 | 312 | 1.8 (2019-10-12) 313 | ---------------- 314 | - Remove duplicated commands. 315 | 316 | 317 | 1.7 (2019-10-07) 318 | ---------------- 319 | - Fixed bug that involved the Boolean Borg settings 320 | (:ref:`one_file_system `, :ref:`exclude_caches 321 | `, ...) 322 | 323 | 324 | 1.6 (2019-10-04) 325 | ---------------- 326 | - Added :ref:`restore ` command. 327 | - Added :ref:`verbose ` setting. 328 | 329 | 330 | 1.5 (2019-09-30) 331 | ---------------- 332 | - Added composite configurations. 333 | - Added support for multiple backup configurations in a single repository. 334 | - Added :ref:`prefix ` and :ref:`exclude_from ` settings. 335 | - Provide default value for :ref:`archive ` setting. 336 | - Add ``--all`` command line option to :ref:`mount ` command. 337 | - Add ``--include-external`` command line option to :ref:`check `, 338 | :ref:`list `, :ref:`mount `, and :ref:`prune ` commands. 339 | - Add ``--sort`` command line option to :ref:`manifest ` command. 340 | - Add ``--latest`` command line option to :ref:`delete ` command. 341 | - Added ``--quiet`` command line option 342 | - :ref:`umount ` command now deletes directory used as mount point. 343 | - Moved log files to ~/.local/share/emborg 344 | (run 'mv ~/.config/emborg/\*.{log,lastbackup}\* ~/.local/share/emborg' before 345 | using this version). 346 | 347 | 348 | 1.4 (2019-04-24) 349 | ---------------- 350 | - Added *ssh_command* setting 351 | - Added ``--fast`` option to :ref:`info ` command 352 | - Added *emborg-overdue* executable 353 | - Allow :ref:`run_before_backup ` and :ref:`run_after_backup 354 | ` to be simple strings 355 | 356 | 357 | 1.3 (2019-01-16) 358 | ---------------- 359 | - Added the raw :ref:`borg ` command. 360 | 361 | 362 | 1.2 (2019-01-16) 363 | ---------------- 364 | - Added the :ref:`borg_executable ` and :ref:`passcommand 365 | ` settings. 366 | 367 | 368 | 1.1 (2019-01-13) 369 | ---------------- 370 | - Improved and documented API. 371 | - Creates the settings directory if it is missing and add example files. 372 | - Added ``--mute`` command line option. 373 | - Support multiple email addresses in :ref:`notify `. 374 | - Added warning if settings file is world readable and contains a passphrase. 375 | 376 | 377 | 1.0 (2019-01-09) 378 | ---------------- 379 | - Added :ref:`remote_path ` setting. 380 | - Formal public release. 381 | 382 | 383 | 0.3 (2018-12-25) 384 | ---------------- 385 | - Initial public release (beta). 386 | 387 | 388 | 0.0 (2018-12-05) 389 | ---------------- 390 | - Initial release (alpha). 391 | -------------------------------------------------------------------------------- /doc/requirements.txt: -------------------------------------------------------------------------------- 1 | docutils!=0.18 2 | # There is an incompatibility between Sphinx and docutils version 0.18. 3 | sphinx_rtd_theme 4 | -------------------------------------------------------------------------------- /emborg/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "1.41" 2 | __released__ = "2024-11-11" 3 | 4 | from .emborg import Emborg 5 | -------------------------------------------------------------------------------- /emborg/collection.py: -------------------------------------------------------------------------------- 1 | # Collections 2 | # 3 | # Provides common interface for dictionaries and lists. If a string is passed 4 | # in, it is split and then treated as a list. Optimized for convenience rather 5 | # than for large collections. 6 | 7 | # License {{{1 8 | # Copyright (C) 2016-2024 Kenneth S. Kundert 9 | # 10 | # This program is free software: you can redistribute it and/or modify 11 | # it under the terms of the GNU General Public License as published by 12 | # the Free Software Foundation, either version 3 of the License, or 13 | # (at your option) any later version. 14 | # 15 | # This program is distributed in the hope that it will be useful, 16 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 17 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 | # GNU General Public License for more details. 19 | # 20 | # You should have received a copy of the GNU General Public License 21 | # along with this program. If not, see http://www.gnu.org/licenses. 22 | 23 | # Imports {{{1 24 | from inform import is_collection, is_str 25 | 26 | # Globals {{{1 27 | __version__ = "0.5.0" 28 | __released__ = "2021-01-27" 29 | 30 | 31 | # Utilities {{{1 32 | # split_lines() {{{2 33 | def split_lines(text, comment=None, strip=False, cull=False, sep=None): 34 | """Split lines 35 | 36 | Can be passed as a splitter to Collection. Takes a multi-line string, 37 | converts it to individual lines where comments are removed (if comment 38 | string is provided), each line is stripped (if strip is True), and empty 39 | lines are culled (if cull is True). If sep is specified, the line is 40 | partitioned into a key and value (everything before sep is the key, 41 | everything after is value). In this case a dictionary is returned. Otherwise 42 | a list is returned. 43 | 44 | """ 45 | lines = text.splitlines() 46 | if comment: 47 | lines = list(l.partition(comment)[0] for l in lines) 48 | if strip: 49 | lines = list(l.strip() for l in lines) 50 | if cull: 51 | lines = list(l for l in lines if l) 52 | if sep: 53 | pairs = dict(l.partition(sep)[::2] for l in lines) 54 | if strip: 55 | lines = {k.strip(): v.strip() for k, v in pairs.items()} 56 | return lines 57 | 58 | 59 | # Unspecified {{{2 60 | # class that is used as a default in functions to signal nothing was given 61 | class Unspecified: 62 | def __bool__(self): 63 | return False 64 | 65 | 66 | # Collection {{{1 67 | class Collection(object): 68 | fmt = '{v}' # default value format 69 | sep = " " # default separator 70 | splitter = "|" # default format splitter (goes between fmt and sep in template) 71 | 72 | """Collection 73 | 74 | Takes a list or dictionary and provides both a list like and dictionary like 75 | API, meaning that it provides the keys, values, and items methods like 76 | dictionary and you can iterate through the value like a list. When applying 77 | keys to a list, the indices are returned as the keys. You can also use an 78 | index or a key to get a value, you test to see if a value is in the 79 | collection using the *in* operator, and you can get the length of the 80 | collection. 81 | 82 | If splitter is a string or None, then you can pass in a string and it will 83 | be split into a list to form the collection. You can also pass in a 84 | splitting function. 85 | """ 86 | 87 | def __init__(self, collection, splitter=None, **kwargs): 88 | if is_str(collection): 89 | if callable(splitter): 90 | self.collection = splitter(collection, **kwargs) 91 | return 92 | elif splitter is not False: 93 | self.collection = collection.split(splitter) 94 | return 95 | if is_collection(collection): 96 | self.collection = collection 97 | return 98 | if collection is None: 99 | self.collection = [] 100 | return 101 | # is scalar 102 | self.collection = {None: collection} 103 | 104 | def keys(self): 105 | try: 106 | return list(self.collection.keys()) 107 | except AttributeError: 108 | return list(range(len(self.collection))) 109 | 110 | def values(self): 111 | try: 112 | return [self.collection[k] for k in self.collection.keys()] 113 | except AttributeError: 114 | return list(self.collection) 115 | 116 | def items(self): 117 | try: 118 | return [(k, self.collection[k]) for k in self.collection.keys()] 119 | except AttributeError: 120 | return list(enumerate(self.collection)) 121 | 122 | def get(self, key, default=Unspecified): 123 | try: 124 | return self.collection[key] 125 | except (KeyError, IndexError): 126 | if default == Unspecified: 127 | raise 128 | return default 129 | 130 | def render(self, fmt=None, sep=None): 131 | """Convert the collection into a string 132 | 133 | fmt (str): 134 | fmt is a format string applied to each of the items in the 135 | collection where {k} is replaced with the key and {v} replaced with 136 | the value. 137 | sep (str): 138 | The string used to join the formatted items. 139 | 140 | Example: 141 | 142 | >>> from collection import Collection 143 | 144 | >>> dogs = Collection({'collie': 3, 'beagle':1, 'shepherd': 2}) 145 | >>> print('dogs: {}.'.format(dogs.render('{k} ({v})', ', '))) 146 | dogs: collie (3), beagle (1), shepherd (2). 147 | 148 | >>> print('dogs: {}.'.format(dogs.render(sep=', '))) 149 | dogs: 3, 1, 2. 150 | 151 | """ 152 | if not fmt: 153 | fmt = self.fmt 154 | if not sep: 155 | sep = self.sep 156 | 157 | if callable(fmt): 158 | return sep.join(fmt(k, v) for k, v in self.items()) 159 | return sep.join(fmt.format(v, k=k, v=v) for k, v in self.items()) 160 | 161 | def __format__(self, template): 162 | """Convert the collection into a string 163 | 164 | The template consists of two components separated by a vertical bar. The 165 | first component specifies the formatting from each item. The key and 166 | value are interpolated using {{k}} to represent the key and {{v}} to 167 | represent the value. The second component is the separator. Thus: 168 | 169 | >>> dogs = Collection({'collie': 3, 'beagle':1, 'shepherd': 2}) 170 | >>> print('dogs: {:{{k}} ({{v}})|, }.'.format(dogs)) 171 | dogs: collie (3), beagle (1), shepherd (2). 172 | 173 | """ 174 | if template: 175 | components = template.split(self.splitter) 176 | if len(components) == 2: 177 | fmt, sep = components 178 | else: 179 | fmt, sep = components[0], " " 180 | else: 181 | fmt, sep = None, None 182 | if not fmt: 183 | fmt = self.fmt 184 | if not sep: 185 | sep = self.sep 186 | 187 | if callable(fmt): 188 | return sep.join(fmt(k, v) for k, v in self.items()) 189 | return sep.join(fmt.format(v, k=k, v=v) for k, v in self.items()) 190 | 191 | def __contains__(self, item): 192 | return item in self.values() 193 | 194 | def __iter__(self): 195 | return iter(self.values()) 196 | 197 | def __len__(self): 198 | return len(self.collection) 199 | 200 | def __getitem__(self, key): 201 | return self.collection[key] 202 | 203 | def __repr__(self): 204 | return f'{self.__class__.__name__}({self.collection!r})' 205 | 206 | def __str__(self): 207 | return str(self.collection) 208 | -------------------------------------------------------------------------------- /emborg/help.py: -------------------------------------------------------------------------------- 1 | # Help 2 | # Output a help topic. 3 | 4 | # License {{{1 5 | # Copyright (C) 2018-2024 Kenneth S. Kundert 6 | # 7 | # This program is free software: you can redistribute it and/or modify it under 8 | # the terms of the GNU General Public License as published by the Free Software 9 | # Foundation, either version 3 of the License, or (at your option) any later 10 | # version. 11 | # 12 | # This program is distributed in the hope that it will be useful, but WITHOUT 13 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 14 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 15 | # details. 16 | # 17 | # You should have received a copy of the GNU General Public License along with 18 | # this program. If not, see http://www.gnu.org/licenses. 19 | 20 | 21 | # Imports {{{1 22 | from textwrap import dedent 23 | 24 | from inform import Error, error, output 25 | 26 | from .command import Command 27 | from .utilities import pager, two_columns 28 | 29 | 30 | # HelpMessage base class {{{1 31 | class HelpMessage(object): 32 | # get_name() {{{2 33 | @classmethod 34 | def get_name(cls): 35 | try: 36 | return cls.name.lower() 37 | except AttributeError: 38 | # consider converting lower to upper case transitions in __name__ to 39 | # dashes. 40 | return cls.__name__.lower() 41 | 42 | # topics {{{2 43 | @classmethod 44 | def topics(cls): 45 | for sub in cls.__subclasses__(): 46 | yield sub 47 | 48 | # show {{{2 49 | @classmethod 50 | def show(cls, name=None): 51 | if name: 52 | # search commands 53 | try: 54 | command, _ = Command.find(name) 55 | if command: 56 | return pager(command.help()) 57 | except Error: 58 | pass 59 | 60 | # search topics 61 | for topic in cls.topics(): 62 | if name == topic.get_name(): 63 | return pager(topic.help()) 64 | 65 | error("topic not found.", culprit=name) 66 | else: 67 | from .main import synopsis 68 | 69 | cls.help(synopsis) 70 | 71 | # summarize {{{2 72 | @classmethod 73 | def summarize(cls, width=16): 74 | summaries = [] 75 | for topic in sorted(cls.topics(), key=lambda topic: topic.get_name()): 76 | summaries.append(two_columns(topic.get_name(), topic.DESCRIPTION)) 77 | return "\n".join(summaries) 78 | 79 | # help {{{2 80 | @classmethod 81 | def help(cls, desc): 82 | if desc: 83 | output(desc.strip() + "\n") 84 | 85 | output("Available commands:") 86 | output(Command.summarize()) 87 | 88 | output("\nAvailable topics:") 89 | output(cls.summarize()) 90 | 91 | 92 | # Overview class {{{1 93 | class Overview(HelpMessage): 94 | DESCRIPTION = "overview of emborg" 95 | 96 | @staticmethod 97 | def help(): 98 | text = dedent( 99 | """ 100 | Emborg is a simple command line utility to orchestrate backups. It 101 | is built on Borg, which is a powerful and fast de-duplicating backup 102 | utility for managing encrypted backups, however it can be rather 103 | clumsy to use directly. With Emborg, you specify all the details 104 | about your backups once in advance, and then use a very simple 105 | command line interface for your day-to-day activities. The details 106 | are contained in ~/.config/emborg. That directory will contain a 107 | file (settings) that contains shared settings, and then another file 108 | for each of your backup configurations. 109 | 110 | Use of Emborg does not preclude the use of Borg directly on the same 111 | repository. The philosophy of Emborg is to provide commands that 112 | you would use often and in an interactive manner with the 113 | expectation that you would use Borg directly for the remaining 114 | commands. In fact, Emborg provides the borg command to make that 115 | easier. 116 | """ 117 | ).strip() 118 | return text 119 | 120 | 121 | # Precautions class {{{1 122 | class Precautions(HelpMessage): 123 | DESCRIPTION = "what everybody should know before using emborg" 124 | 125 | @staticmethod 126 | def help(): 127 | text = dedent( 128 | """ 129 | You should assure you have a backup copy of the encryption 130 | passphrase in a safe place. This is very important. If the only 131 | copy of the passphrase is on the disk being backed up and that disk 132 | were to fail you would not be able to access your backups. 133 | I recommend the use of sparekeys (https://github.com/kalekundert/sparekeys) 134 | as a way of assuring that you always have access to the essential 135 | information, such as your Borg passphrase and keys, that you would 136 | need to get started after a catastrophic loss of your disk. 137 | 138 | In addition it is important to understand that your backup data is 139 | not encrypted with your passphrase, rather your passphrase encrypts 140 | a key, and your backup data is encrypted with the key. Thus, your 141 | backup files cannot be decrypted without both the passphrase and the 142 | key. If you choose encryption=repokey then Borg copies your key to 143 | the remote repository, so you do not need to keep a copy yourself. 144 | Using repokey is appropriate if you control the server that holds 145 | the remote repository. If you choose encryption=keyfile, which is 146 | appropriate if you do not control the resository's server, then it 147 | does not copy the key to the repository, so it is essential 148 | that you export and keep a copy of the key in a safe place. 149 | Sparekeys can do this for you. 150 | 151 | If you keep the passphrase in a settings file, you should set 152 | its permissions so that it is not readable by others: 153 | 154 | chmod 700 ~/.config/emborg/settings 155 | 156 | Better yet is to simply not store the passphrase. This can be 157 | arranged if you are using Avendesora 158 | (https://avendesora.readthedocs.io/en/stable), which is a flexible 159 | password management system. The interface to Avendesora is already 160 | built in to Emborg, but its use is optional (it need not be 161 | installed). 162 | 163 | It is also best, if it can be arranged, to keep your backups at a 164 | remote site so that your backups do not get destroyed in the same 165 | disaster, such as a fire or flood, that claims your original files. 166 | Some commercial options are: 167 | rsync.net (https://www.rsync.net/products/attic.html) 168 | BorgBase (https://www.borgbase.com) 169 | Hetzner (https://www.hetzner.com/storage/storage-box) 170 | 171 | Finally, it is a good idea to practice a recovery. Pretend that you 172 | have lost all your files and then see if you can do a restore from 173 | backup. Doing this and working out the kinks before you lose your 174 | files can save you if you ever do lose your files. 175 | """ 176 | ).strip() 177 | return text 178 | -------------------------------------------------------------------------------- /emborg/hooks.py: -------------------------------------------------------------------------------- 1 | # Hooks 2 | 3 | # License {{{1 4 | # Copyright (C) 2018-2024 Kenneth S. Kundert 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see http://www.gnu.org/licenses. 18 | 19 | 20 | # Imports {{{1 21 | from inform import Error, full_stop, log, os_error 22 | from .preferences import EMBORG_SETTINGS 23 | import requests 24 | 25 | # Hooks base class {{{1 26 | class Hooks: 27 | @classmethod 28 | def provision_hooks(cls): 29 | for subclass in cls.__subclasses__(): 30 | for k, v in subclass.EMBORG_SETTINGS.items(): 31 | assert k not in EMBORG_SETTINGS 32 | EMBORG_SETTINGS[k] = v 33 | 34 | def __init__(self, settings): 35 | self.active_hooks = [] 36 | for subclass in self.__class__.__subclasses__(): 37 | c = subclass(settings) 38 | if c.is_active(): 39 | self.active_hooks.append(c) 40 | 41 | def report_results(self, borg): 42 | for hook in self.active_hooks: 43 | hook.borg = borg 44 | 45 | def __enter__(self): 46 | for hook in self.active_hooks: 47 | hook.signal_start() 48 | return self 49 | 50 | def __exit__(self, exc_type, exc_value, exc_traceback): 51 | for hook in self.active_hooks: 52 | hook.signal_end(exc_value) 53 | 54 | def is_active(self): 55 | return bool(self.uuid) 56 | 57 | def signal_start(self): 58 | url = self.START_URL.format(url=self.url, uuid=self.uuid) 59 | log(f'signaling start of backups to {self.NAME}: {url}.') 60 | try: 61 | requests.get(url) 62 | except requests.exceptions.RequestException as e: 63 | raise Error(f'{self.NAME} connection error.', codicil=full_stop(e)) 64 | 65 | def signal_end(self, exception): 66 | if exception: 67 | url = self.FAIL_URL.format(url=self.url, uuid=self.uuid) 68 | result = 'failure' 69 | else: 70 | url = self.SUCCESS_URL.format(url=self.url, uuid=self.uuid) 71 | result = 'success' 72 | log(f'signaling {result} of backups to {self.NAME}: {url}.') 73 | try: 74 | requests.get(url) 75 | except requests.exceptions.RequestException as e: 76 | raise Error('{self.NAME} connection error.', codicil=full_stop(e)) 77 | 78 | 79 | # HealthChecks class {{{1 80 | class HealthChecks(Hooks): 81 | NAME = 'healthchecks.io' 82 | EMBORG_SETTINGS = dict( 83 | healthchecks_url = 'the healthchecks.io URL for back-ups monitor', 84 | healthchecks_uuid = 'the healthchecks.io UUID for back-ups monitor', 85 | ) 86 | URL = 'https://hc-ping.com' 87 | 88 | def __init__(self, settings): 89 | self.uuid = settings.healthchecks_uuid 90 | self.url = settings.healthchecks_url 91 | if not self.url: 92 | self.url = self.URL 93 | self.borg = None 94 | 95 | def signal_start(self): 96 | url = f'{self.url}/{self.uuid}/start' 97 | log(f'signaling start of backups to {self.NAME}: {url}.') 98 | try: 99 | requests.post(url) 100 | except requests.exceptions.RequestException as e: 101 | raise Error('{self.NAME} connection error.', codicil=full_stop(e)) 102 | 103 | def signal_end(self, exception): 104 | if exception: 105 | result = 'failure' 106 | if isinstance(exception, OSError): 107 | status = 1 108 | payload = os_error(exception) 109 | else: 110 | try: 111 | status = exception.status 112 | payload = exception.stderr 113 | except AttributeError: 114 | status = 1 115 | payload = str(exception) 116 | else: 117 | result = 'success' 118 | if self.borg: 119 | status = self.borg.status 120 | payload = self.borg.stderr 121 | else: 122 | status = 0 123 | payload = '' 124 | 125 | url = f'{self.url}/{self.uuid}/{status}' 126 | log(f'signaling {result} of backups to {self.NAME}: {url}.') 127 | try: 128 | if payload: 129 | requests.post(url, data=payload.encode('utf-8')) 130 | else: 131 | requests.post(url) 132 | except requests.exceptions.RequestException as e: 133 | raise Error('{self.NAME} connection error.', codicil=full_stop(e)) 134 | 135 | 136 | # CronHub class {{{1 137 | class CronHub(Hooks): 138 | NAME = 'cronhub.io' 139 | EMBORG_SETTINGS = dict( 140 | cronhub_uuid = 'the cronhub.io UUID for back-ups monitor', 141 | cronhub_url = 'the cronhub.io URL for back-ups monitor', 142 | ) 143 | START_URL = '{url}/start/{uuid}' 144 | SUCCESS_URL = '{url}/finish/{uuid}' 145 | FAIL_URL = '{url}/fail/{uuid}' 146 | URL = 'https://cronhub.io' 147 | 148 | def __init__(self, settings): 149 | self.uuid = settings.cronhub_uuid 150 | self.url = settings.cronhub_url 151 | if not self.url: 152 | self.url = self.URL 153 | -------------------------------------------------------------------------------- /emborg/main.py: -------------------------------------------------------------------------------- 1 | # Usage {{{1 2 | """ 3 | Emborg Backups 4 | 5 | Backs up the contents of a file hierarchy. A front end for Borg's 6 | encrypted incremental backup utility. 7 | 8 | Usage: 9 | emborg [options] [ [...]] 10 | 11 | Options: 12 | -c , --config Specifies the configuration to use. 13 | -d, --dry-run Run Borg in dry run mode. 14 | -h, --help Output basic usage information. 15 | -m, --mute Suppress all output. 16 | -n, --narrate Send emborg and Borg narration to stdout. 17 | -q, --quiet Suppress optional output. 18 | -r, --relocated Acknowledge that repository was relocated. 19 | -v, --verbose Make Borg more verbose. 20 | --no-log Do not create log file. 21 | """ 22 | 23 | # License {{{1 24 | # Copyright (C) 2018-2024 Kenneth S. Kundert 25 | # 26 | # This program is free software: you can redistribute it and/or modify 27 | # it under the terms of the GNU General Public License as published by 28 | # the Free Software Foundation, either version 3 of the License, or 29 | # (at your option) any later version. 30 | # 31 | # This program is distributed in the hope that it will be useful, 32 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 33 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 34 | # GNU General Public License for more details. 35 | # 36 | # You should have received a copy of the GNU General Public License 37 | # along with this program. If not, see http://www.gnu.org/licenses. 38 | 39 | 40 | # Imports {{{1 41 | import os 42 | import sys 43 | from docopt import docopt 44 | from inform import ( 45 | Error, Inform, LoggingCache, cull, display, error, os_error, terminate 46 | ) 47 | from . import __released__, __version__ 48 | from .command import Command 49 | from .hooks import Hooks 50 | from .emborg import ConfigQueue, Emborg 51 | 52 | # Globals {{{1 53 | version = f"{__version__} ({__released__})" 54 | commands = """ 55 | Commands: 56 | {commands} 57 | 58 | Use 'emborg help ' for information on a specific command. 59 | Use 'emborg help' for list of available help topics. 60 | """ 61 | synopsis = __doc__ 62 | expanded_synopsis = synopsis + commands.format(commands=Command.summarize()) 63 | 64 | 65 | # Main {{{1 66 | def main(): 67 | with Inform( 68 | error_status = 2, 69 | flush = True, 70 | logfile = LoggingCache(), 71 | prog_name = 'emborg', 72 | version = version, 73 | ) as inform: 74 | 75 | try: 76 | worst_exit_status = 0 77 | 78 | # emborg fails if the current working directory does not exist and 79 | # the message returned by OSError does not make the problem obvious. 80 | try: 81 | os.getcwd() 82 | except OSError as e: 83 | raise Error(os_error(e), codicil="Does the current working directory exist?") 84 | 85 | # read command line 86 | cmdline = docopt(expanded_synopsis, options_first=True, version=version) 87 | config = cmdline["--config"] 88 | command = cmdline[""] 89 | args = cmdline[""] 90 | if cmdline["--mute"]: 91 | inform.mute = True 92 | if cmdline["--quiet"]: 93 | inform.quiet = True 94 | if cmdline["--relocated"]: 95 | os.environ['BORG_RELOCATED_REPO_ACCESS_IS_OK'] = 'YES' 96 | emborg_opts = cull( 97 | [ 98 | "verbose" if cmdline["--verbose"] else None, 99 | "narrate" if cmdline["--narrate"] else None, 100 | "dry-run" if cmdline["--dry-run"] else None, 101 | "no-log" if cmdline["--no-log"] else None, 102 | ] 103 | ) 104 | if cmdline["--narrate"]: 105 | inform.narrate = True 106 | 107 | Hooks.provision_hooks() 108 | 109 | # find the command 110 | cmd, cmd_name = Command.find(command) 111 | 112 | # execute the command initialization 113 | exit_status = cmd.execute_early(cmd_name, args, None, emborg_opts) 114 | if exit_status is not None: 115 | terminate(exit_status) 116 | 117 | queue = ConfigQueue(cmd) 118 | while queue: 119 | with Emborg(config, emborg_opts, queue=queue, cmd_name=cmd_name) as settings: 120 | try: 121 | exit_status = cmd.execute( 122 | cmd_name, args, settings, emborg_opts 123 | ) 124 | except Error as e: 125 | exit_status = 2 126 | settings.fail(e, cmd=' '.join(sys.argv)) 127 | e.report() 128 | 129 | if exit_status and exit_status > worst_exit_status: 130 | worst_exit_status = exit_status 131 | inform.errors_accrued(reset=True) 132 | 133 | # execute the command termination 134 | exit_status = cmd.execute_late(cmd_name, args, None, emborg_opts) 135 | if exit_status and exit_status > worst_exit_status: 136 | worst_exit_status = exit_status 137 | 138 | except Error as e: 139 | exit_status = 2 140 | e.report() 141 | except OSError as e: 142 | exit_status = 2 143 | error(os_error(e)) 144 | except KeyboardInterrupt: 145 | exit_status = 0 146 | display("Terminated by user.") 147 | if exit_status and exit_status > worst_exit_status: 148 | worst_exit_status = exit_status 149 | terminate(worst_exit_status) 150 | -------------------------------------------------------------------------------- /emborg/overdue.py: -------------------------------------------------------------------------------- 1 | # Usage {{{1 2 | """ 3 | Overdue Emborg Backups 4 | 5 | This program notifies you if backups have not been run recently. It can be run 6 | either from the server (the destination) or from the client (the source). It 7 | simply lists those archives that are out-of-date. If you specify --mail, email 8 | is sent that describes the situation if a backup is overdue. 9 | 10 | Usage: 11 | emborg-overdue [options] 12 | 13 | Options: 14 | -c, --no-color Do not color the output 15 | -h, --help Output basic usage information 16 | -l, --local Only report on local repositories 17 | -m, --mail Send mail message if backup is overdue 18 | -n, --notify Send notification if backup is overdue 19 | -N, --nt Output summary in NestedText format 20 | -p, --no-passes Do not show hosts that are not overdue 21 | -q, --quiet Suppress output to stdout 22 | -v, --verbose Give more information about each repository 23 | -M, --message Status message template for each repository 24 | --version Show software version 25 | 26 | The program requires a configuration file, overdue.conf, which should be placed 27 | in the Emborg configuration directory, typically ~/.config/emborg. The contents 28 | are described here: 29 | 30 | https://emborg.readthedocs.io/en/stable/monitoring.html#overdue 31 | 32 | The message given by --message may contain the following keys in braces: 33 | host: replaced by the host field from the config file, a string. 34 | max_age: replaced by the max_age field from the config file, a float in hours. 35 | mtime: replaced by modification time, a datetime object. 36 | hours: replaced by the number of hours since last update, a float. 37 | age: replaced by time since last update, a string. 38 | overdue: is the back-up overdue, a boolean. 39 | locked: is the back-up currently active, a boolean. 40 | 41 | The status message is a Python formatted string, and so the various fields can include 42 | formatting directives. For example: 43 | - strings than include field width and justification, ex. {host:>20} 44 | - floats can include width, precision and form, ex. {hours:0.1f} 45 | - datetimes can include Arrow formats, ex: {mtime:DD MMM YY @ H:mm A} 46 | - boolean can include true/false strings, ex: {overdue:PAST DUE!/current} 47 | """ 48 | 49 | # License {{{1 50 | # Copyright (C) 2018-2024 Kenneth S. Kundert 51 | # 52 | # This program is free software: you can redistribute it and/or modify 53 | # it under the terms of the GNU General Public License as published by 54 | # the Free Software Foundation, either version 3 of the License, or 55 | # (at your option) any later version. 56 | # 57 | # This program is distributed in the hope that it will be useful, 58 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 59 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 60 | # GNU General Public License for more details. 61 | # 62 | # You should have received a copy of the GNU General Public License 63 | # along with this program. If not, see http://www.gnu.org/licenses. 64 | 65 | 66 | # Imports {{{1 67 | import os 68 | import sys 69 | import pwd 70 | import socket 71 | import arrow 72 | from docopt import docopt, DocoptExit 73 | from inform import ( 74 | Color, 75 | Error, 76 | Inform, 77 | InformantFactory, 78 | conjoin, 79 | dedent, 80 | display, 81 | error, 82 | fatal, 83 | get_prog_name, 84 | is_str, 85 | os_error, 86 | output, 87 | terminate, 88 | truth, 89 | warn, 90 | ) 91 | import nestedtext as nt 92 | 93 | from . import __released__, __version__ 94 | from .preferences import CONFIG_DIR, DATA_DIR, OVERDUE_FILE, OVERDUE_LOG_FILE 95 | from .python import PythonFile 96 | from .shlib import Run, to_path, set_prefs as set_shlib_prefs 97 | from .utilities import read_latest, when 98 | 99 | # Globals {{{1 100 | set_shlib_prefs(use_inform=True, log_cmd=True) 101 | username = pwd.getpwuid(os.getuid()).pw_name 102 | hostname = socket.gethostname() 103 | now = arrow.now() 104 | 105 | # colors {{{2 106 | default_colorscheme = "dark" 107 | current_color = "green" 108 | overdue_color = "red" 109 | 110 | # message templates {{{2 111 | verbose_status_message = dedent("""\ 112 | HOST: {host} 113 | sentinel file: {path!s} 114 | last modified: {mtime} 115 | since last change: {hours:0.1f} hours 116 | maximum age: {max_age} hours 117 | overdue: {overdue} 118 | locked: {locked} 119 | """, strip_nl='l') 120 | 121 | terse_status_message = "{host}: {age} ago{locked: (currently active)}{overdue: — PAST DUE}" 122 | 123 | mail_status_message = dedent(""" 124 | Backup of {host} is overdue: 125 | the backup sentinel file has not changed in {hours:0.1f} hours. 126 | """, strip_nl='b') 127 | 128 | error_message = dedent(f""" 129 | {get_prog_name()} generated the following error: 130 | from: {username}@{hostname} at {now} 131 | message: {{}} 132 | """, strip_nl='b') 133 | 134 | # Utilities {{{1 135 | # get_local_data {{{2 136 | def get_local_data(path, host, max_age): 137 | if path.is_dir(): 138 | paths = list(path.glob("index.*")) 139 | if not paths: 140 | raise Error("no sentinel file found.", culprit=path) 141 | if len(paths) > 1: 142 | raise Error("too many sentinel files.", *paths, sep="\n ") 143 | path = paths[0] 144 | locked = list(path.glob('lock.*')) 145 | mtime = arrow.get(path.stat().st_mtime) 146 | if path.suffix == '.nt': 147 | latest = read_latest(path) 148 | locked = (path.parent / path.name.replace('.latest.nt', '.lock')).exists() 149 | mtime = latest.get('create last run') 150 | if not mtime: 151 | raise Error('backup time is not available.', culprit=path) 152 | delta = now - mtime 153 | hours = 24 * delta.days + delta.seconds / 3600 154 | overdue = truth(hours > max_age) 155 | locked = truth(locked) 156 | yield dict( 157 | host=host, path=path, mtime=mtime, hours=hours, max_age=max_age, 158 | overdue=overdue, locked=locked 159 | ) 160 | 161 | # get_remote_data {{{2 162 | def get_remote_data(name, path): 163 | host, _, cmd = path.partition(':') 164 | cmd = cmd or "emborg-overdue" 165 | display(f"\n{name}:") 166 | try: 167 | ssh = Run(['ssh', host, cmd, '--nt'], 'sOEW2') 168 | for repo_data in nt.loads(ssh.stdout, top=list): 169 | if 'mtime' in repo_data: 170 | repo_data['mtime'] = arrow.get(repo_data['mtime']) 171 | if 'overdue' in repo_data: 172 | repo_data['overdue'] = truth(repo_data['overdue'] == 'yes') 173 | if 'hours' in repo_data: 174 | repo_data['hours'] = float(repo_data.get('hours', 0)) 175 | if 'max_age' in repo_data: 176 | repo_data['max_age'] = float(repo_data.get('max_age', 0)) 177 | if 'locked' in repo_data: 178 | repo_data['locked'] = truth(repo_data['locked'] == 'yes') 179 | else: 180 | repo_data['locked'] = truth(False) 181 | yield repo_data 182 | except Error as e: 183 | e.report(culprit=host) 184 | 185 | if ssh.status > 1: 186 | raise Error( 187 | "error found by remote overdue process.", 188 | culprit=host, codicil=ssh.stderr.strip() 189 | ) 190 | 191 | # fixed() {{{2 192 | # formats float using fixed point notation while removing trailing zeros 193 | def fixed(num, prec=2): 194 | return format(num, f".{prec}f").strip('0').strip('.') 195 | 196 | # Main {{{1 197 | def main(): 198 | # read the settings file 199 | try: 200 | settings_file = PythonFile(CONFIG_DIR, OVERDUE_FILE) 201 | settings = settings_file.run() 202 | except Error as e: 203 | e.terminate() 204 | 205 | # gather needed settings 206 | default_maintainer = settings.get("default_maintainer") 207 | default_max_age = settings.get("default_max_age", 28) 208 | dumper = settings.get("dumper", f"{username}@{hostname}") 209 | repositories = settings.get("repositories") 210 | root = settings.get("root") 211 | colorscheme = settings.get("colorscheme", default_colorscheme) 212 | status_message = settings.get("status_message", terse_status_message) 213 | 214 | version = f"{__version__} ({__released__})" 215 | try: 216 | cmdline = docopt(__doc__, version=version) 217 | except DocoptExit as e: 218 | sys.stderr.write(str(e) + '\n') 219 | terminate(3) 220 | quiet = cmdline["--quiet"] 221 | exit_status = 0 222 | report_as_current = InformantFactory( 223 | clone=display, message_color=current_color 224 | ) 225 | report_as_overdue = InformantFactory( 226 | clone=display, message_color=overdue_color, 227 | notify=cmdline['--notify'] and not Color.isTTY() 228 | ) 229 | if cmdline["--message"]: 230 | status_message = cmdline["--message"] 231 | if cmdline["--no-color"]: 232 | colorscheme = None 233 | if cmdline["--verbose"]: 234 | status_message = verbose_status_message 235 | 236 | # prepare to create logfile 237 | log = to_path(DATA_DIR, OVERDUE_LOG_FILE) if OVERDUE_LOG_FILE else False 238 | if log: 239 | data_dir = to_path(DATA_DIR) 240 | if not data_dir.exists(): 241 | try: 242 | # data dir does not exist, create it 243 | data_dir.mkdir(mode=0o700, parents=True, exist_ok=True) 244 | except OSError as e: 245 | warn(os_error(e)) 246 | log = False 247 | 248 | with Inform( 249 | flush=True, quiet=quiet or cmdline["--nt"], logfile=log, 250 | error_status=2, colorscheme=colorscheme, version=version, 251 | stream_policy = 'header' if cmdline['--nt'] else 'termination' 252 | ): 253 | overdue_hosts = {} 254 | 255 | # process repositories table 256 | backups = [] 257 | if is_str(repositories): 258 | for line in repositories.split("\n"): 259 | line = line.split("#")[0].strip() # discard comments 260 | if not line: 261 | continue 262 | backups.append([c.strip() for c in line.split("|")]) 263 | else: 264 | for each in repositories: 265 | backups.append( 266 | [ 267 | each.get("host"), 268 | each.get("path"), 269 | each.get("maintainer"), 270 | each.get("max_age"), 271 | ] 272 | ) 273 | 274 | def send_mail(recipient, subject, message): 275 | if cmdline["--mail"]: 276 | if cmdline['--verbose']: 277 | display(f"Reporting to {recipient}.\n") 278 | mail_cmd = ["mailx", "-r", dumper, "-s", subject, recipient] 279 | Run(mail_cmd, stdin=message, modes="soeW0") 280 | 281 | # check age of repositories 282 | for host, path, maintainer, max_age in backups: 283 | maintainer = default_maintainer if not maintainer else maintainer 284 | max_age = float(max_age if max_age else default_max_age) 285 | try: 286 | if ':' in str(path): 287 | if cmdline['--local']: 288 | repos_data = [] 289 | else: 290 | repos_data = get_remote_data(host, str(path)) 291 | else: 292 | repos_data = get_local_data(to_path(root, path), host, max_age) 293 | for repo_data in repos_data: 294 | repo_data['age'] = when(repo_data['mtime']) 295 | overdue = repo_data['overdue'] 296 | report = report_as_overdue if overdue else report_as_current 297 | 298 | if overdue or not cmdline["--no-passes"]: 299 | if cmdline["--nt"]: 300 | output(nt.dumps([repo_data], converters={float:fixed}, default=str)) 301 | else: 302 | try: 303 | report(status_message.format(**repo_data)) 304 | except ValueError as e: 305 | fatal(e, culprit=(host, '--message')) 306 | except KeyError as e: 307 | fatal( 308 | f"‘{e.args[0]}’ is an unknown key.", 309 | culprit=(host, '--message'), 310 | codicil=f"Choose from: {conjoin(repo_data.keys())}.", 311 | ) 312 | 313 | if overdue: 314 | exit_status = max(exit_status, 1) 315 | overdue_hosts[host] = mail_status_message.format(**repo_data) 316 | 317 | except OSError as e: 318 | exit_status = max(exit_status, 2) 319 | msg = os_error(e) 320 | error(msg) 321 | if maintainer: 322 | send_mail( 323 | maintainer, 324 | f"{get_prog_name()} error", 325 | error_message.format(msg), 326 | ) 327 | except Error as e: 328 | exit_status = max(exit_status, 2) 329 | e.report() 330 | if maintainer: 331 | send_mail( 332 | maintainer, 333 | f"{get_prog_name()} error", 334 | error_message.format(str(e)), 335 | ) 336 | 337 | if overdue_hosts: 338 | if len(overdue_hosts) > 1: 339 | subject = "backups are overdue" 340 | else: 341 | subject = "backup is overdue" 342 | messages = '\n\n'.join(overdue_hosts.values()) 343 | send_mail(maintainer, subject, messages) 344 | 345 | terminate(exit_status) 346 | -------------------------------------------------------------------------------- /emborg/patterns.py: -------------------------------------------------------------------------------- 1 | # Patterns and Excludes 2 | 3 | # License {{{1 4 | # Copyright (C) 2018-2024 Kenneth S. Kundert 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see http://www.gnu.org/licenses. 18 | 19 | # Imports {{{1 20 | from os.path import expanduser as expand_user 21 | from inform import Error, error 22 | from .shlib import to_path 23 | 24 | 25 | # Globals {{{1 26 | known_kinds = "PR+-!" 27 | known_styles = "fm sh re pp pf".split() 28 | 29 | 30 | # check_root() {{{1 31 | def check_root(root, working_dir): 32 | # if str(root) == '.': 33 | # log( 34 | # "Unable to determine whether paths are contained in a root,", 35 | # "because '.' is a root." 36 | # ) 37 | abs_root = working_dir / root 38 | if not abs_root.exists(): 39 | raise Error("not found.", culprit=root) 40 | # the following is a bit cleaner, but not available until python 3.9 41 | # if not abs_root.is_relative_to(working_dir): 42 | # raise Error("not in working directory:", working_dir, culprit=root) 43 | try: 44 | abs_root.relative_to(working_dir) 45 | except ValueError: 46 | raise Error("not in working directory:", working_dir, culprit=root) 47 | return to_path(root).is_absolute() 48 | 49 | 50 | # check_roots() {{{1 51 | def check_roots(roots, working_dir): 52 | if not roots: 53 | raise Error("no roots or source directories given.") 54 | is_absolute = [check_root(root, working_dir) for root in roots] 55 | all_are_absolute = all(is_absolute) 56 | any_are_absolute = any(is_absolute) 57 | if all_are_absolute != any_are_absolute: 58 | raise Error( 59 | "mix of relative and absolute paths used for recursive roots:", 60 | *roots, 61 | sep="\n " 62 | ) 63 | if any_are_absolute and str(working_dir) != "/": 64 | raise Error("working directory must be '/' if root paths are absolute.") 65 | 66 | 67 | # check_pattern() {{{1 68 | def check_pattern(pattern, default_style, roots, expand_tilde): 69 | if ":" in pattern: 70 | style, colon, path = pattern.partition(":") 71 | prefix = style + colon 72 | else: 73 | style, path = default_style, pattern 74 | prefix = "" 75 | 76 | if style not in known_styles: 77 | raise Error("unknown pattern style.") 78 | 79 | # accept regular expression without further checking 80 | if style == "re" or path[0] == "*": 81 | return pattern 82 | 83 | # process leading ~ in path 84 | if path[0] == "~": 85 | if expand_tilde: 86 | path = expand_user(path) 87 | pattern = prefix + path 88 | else: 89 | raise Error( 90 | "do not use leading ~,", 91 | "borg does not treat it as a user home directory.", 92 | ) 93 | 94 | # check to see that path corresponds to a root 95 | path = str(to_path(path)) 96 | if any(path.startswith(str(root)) for root in roots): 97 | return pattern 98 | if style in ["fm", "sh"]: 99 | if any(str(root).startswith(path.partition("*")[0]) for root in roots): 100 | return pattern 101 | if any(str(root) == '.' for root in roots): 102 | # cannot check paths if root is '.' 103 | return pattern 104 | raise Error("path is not in a known root.") 105 | 106 | 107 | # check_patterns() {{{1 108 | def check_patterns( 109 | patterns, roots, working_dir, src, expand_tilde=True, skip_checks=False 110 | ): 111 | paths = [] 112 | default_style = "sh" 113 | for pattern in patterns: 114 | pattern = pattern.strip() 115 | culprit = src 116 | codicil = repr(pattern) 117 | kind = pattern[0:1] 118 | arg = pattern[1:].lstrip() 119 | if not kind or not arg: 120 | raise Error(f"invalid pattern: ‘{pattern}’") 121 | if kind in ["", "#"]: 122 | continue # is comment 123 | if kind not in known_kinds: 124 | error("unknown type", culprit=culprit) 125 | continue 126 | if kind == "R": 127 | if arg[0] == "~" and not expand_tilde: 128 | error( 129 | "borg does not interpret leading tilde as user.", 130 | culprit=culprit, codicil=codicil 131 | ) 132 | root = expand_user(arg) 133 | if not skip_checks: 134 | check_root(root, working_dir) 135 | roots.append(to_path(root)) 136 | paths.append(kind + " " + root) 137 | elif kind == "P": 138 | if arg in known_styles: 139 | default_style = arg 140 | else: 141 | error("unknown pattern style.", culprit=culprit, codicil=codicil) 142 | paths.append(kind + " " + arg) 143 | elif not roots: 144 | error("no roots available.", culprit=culprit, codicil=codicil) 145 | return [] 146 | else: 147 | try: 148 | paths.append( 149 | kind + " " + check_pattern(arg, default_style, roots, expand_tilde) 150 | ) 151 | except Error as e: 152 | e.report(culprit=culprit, codicil=codicil) 153 | return paths 154 | 155 | 156 | # check_excludes() {{{1 157 | def check_excludes(patterns, roots, src, expand_tilde=True): 158 | if not roots: 159 | error("no roots available.", culprit=src) 160 | return 161 | paths = [] 162 | 163 | for pattern in patterns: 164 | pattern = str(pattern).strip() 165 | if not pattern or pattern[0] == "#": 166 | continue # is comment 167 | try: 168 | paths.append(check_pattern(pattern, "fm", roots, expand_tilde)) 169 | except Error as e: 170 | e.report(culprit=src, codicil=repr(pattern)) 171 | return paths 172 | 173 | 174 | # check_patterns_files() {{{1 175 | def check_patterns_files(filenames, roots, working_dir, skip_checks=False): 176 | for filename in filenames: 177 | patterns = to_path(filename).read_text().splitlines() 178 | check_patterns( 179 | patterns, roots, working_dir, filename, 180 | expand_tilde=False, skip_checks=skip_checks 181 | ) 182 | 183 | 184 | # check_excludes_files() {{{1 185 | def check_excludes_files(filenames, roots): 186 | for filename in filenames: 187 | excludes = to_path(filename).read_text().splitlines() 188 | check_excludes(excludes, roots, filename, expand_tilde=False) 189 | -------------------------------------------------------------------------------- /emborg/preferences.py: -------------------------------------------------------------------------------- 1 | # Emborg Preferences 2 | # 3 | 4 | # License {{{1 5 | # Copyright (C) 2018-2024 Kenneth S. Kundert 6 | # 7 | # This program is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # This program is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with this program. If not, see http://www.gnu.org/licenses. 19 | 20 | # Imports {{{1 21 | from appdirs import user_config_dir, user_data_dir 22 | from inform import dedent 23 | import os 24 | 25 | # Preferences {{{1 26 | # Constants {{{2 27 | PROGRAM_NAME = "emborg" 28 | DEFAULT_COMMAND = "create" 29 | INDENT = " " 30 | BORG = "borg" 31 | 32 | # The default configuration directories for both Microsoft and Apple are not 33 | # always preferred by users, whereas the directory used by Linux 34 | # (~/.config/emborg) seems to have universal appeal. So, use the Linux path if 35 | # it exists and backstop with the system preferred location. 36 | 37 | if 'XDG_CONFIG_HOME' in os.environ: 38 | CONFIG_DIR = os.sep.join([os.environ['XDG_CONFIG_HOME'], PROGRAM_NAME]) 39 | else: 40 | CONFIG_DIR = user_config_dir(PROGRAM_NAME) 41 | if 'XDG_DATA_HOME' in os.environ: 42 | DATA_DIR = os.sep.join([os.environ['XDG_DATA_HOME'], PROGRAM_NAME]) 43 | else: 44 | DATA_DIR = user_data_dir(PROGRAM_NAME) 45 | 46 | SETTINGS_FILE = "settings" 47 | OVERDUE_FILE = "overdue.conf" 48 | LOG_FILE = "{config_name}.log" 49 | OVERDUE_LOG_FILE = "overdue.log" 50 | PREV_LOG_FILE = "{config_name}.log.prev" 51 | LOCK_FILE = "{config_name}.lock" 52 | DATE_FILE = "{config_name}.latest.nt" 53 | 54 | CONFIGS_SETTING = "configurations" 55 | DEFAULT_CONFIG_SETTING = "default_configuration" 56 | INCLUDE_SETTING = "include" 57 | DEFAULT_ENCODING = "utf-8" 58 | 59 | # Emborg settings {{{2 60 | EMBORG_SETTINGS = dict( 61 | archive="template Borg should use when creating archive names", 62 | avendesora_account="account name that holds passphrase for encryption key in Avendesora", 63 | avendesora_field="name of field in Avendesora that holds the passphrase", 64 | borg_executable="path to borg", 65 | check_after_create="run check as the last step of an archive creation", 66 | cmd_name="name of Emborg command being run (read only)", 67 | colorscheme="the color scheme", 68 | config_dir="absolute path to configuration directory (read-only)", 69 | config_name="name of active configuration (read only)", 70 | configurations="available Emborg configurations", 71 | default_configuration="default Emborg configuration", 72 | default_mount_point="directory to use as mount point if one is not specified", 73 | do_not_expand="names of settings that must not undergo setting evaluation", 74 | encoding="encoding when talking to borg", 75 | encryption="encryption method (see Borg documentation)", 76 | excludes="list of glob strings of files or directories to skip", 77 | exclude_from="file that contains exclude patterns", 78 | home_dir="users home directory (read only)", 79 | include="include the contents of another file", 80 | log_dir="emborg log directory (read only)", 81 | manage_diffs_cmd="command to use to manage differences in files and directories", 82 | manifest_formats="format strings used by manifest", 83 | manifest_default_format="the format that manifest should use if none is specified", 84 | must_exist="if set, each of these files or directories must exist or create will quit with an error", 85 | needs_ssh_agent="if set, Emborg will complain if ssh_agent is not available", 86 | notifier="notification program", 87 | notify="email address to notify when things go wrong", 88 | passcommand="command used by Borg to acquire the passphrase", 89 | passphrase="passphrase for encryption key (if specified, Avendesora is not used)", 90 | patterns="patterns that indicate whether a path should be included or excluded", 91 | patterns_from="file that contains patterns", 92 | prune_after_create="run prune after creating an archive", 93 | compact_after_delete="run compact after deleting an archive or pruning a repository", 94 | report_diffs_cmd="shell command to use to report differences in files and directories", 95 | repository="path to remote directory that contains repository", 96 | run_after_backup="commands to run after archive has been created", 97 | run_before_backup="commands to run before archive is created", 98 | run_after_last_backup="commands to run after last archive has been created", 99 | run_before_first_backup="commands to run before first archive is created", 100 | run_after_borg="commands to run after last Borg command has run", 101 | run_before_borg="commands to run before first Borg command is run", 102 | show_progress="show borg progress when running create command", 103 | show_stats="show borg statistics when running create, delete, and prune commands", 104 | src_dirs="the directories to archive", 105 | ssh_command="command to use for SSH, can be used to specify SSH options", 106 | verbose="make Borg more verbose", 107 | working_dir="working directory", 108 | ) 109 | # Any setting found in the users settings files that is not found in 110 | # EMBORG_SETTINGS or BORG_SETTINGS is highlighted as a unknown setting by 111 | # the settings command. 112 | 113 | # Borg settings {{{2 114 | BORG_SETTINGS = dict( 115 | append_only = dict( 116 | cmds = ["init"], 117 | desc = "create an append-only mode repository" 118 | ), 119 | chunker_params = dict( 120 | cmds = ["create"], 121 | arg = "PARAMS", 122 | desc = "specify the chunker parameters" 123 | ), 124 | compression = dict( 125 | cmds = ["create"], 126 | arg = "COMPRESSION", 127 | desc = "compression algorithm" 128 | ), 129 | exclude_caches = dict( 130 | cmds = ["create"], 131 | desc = "exclude directories that contain a CACHEDIR.TAG file" 132 | ), 133 | exclude_nodump = dict( 134 | cmds = ["create"], 135 | desc = "exclude files flagged NODUMP" 136 | ), 137 | exclude_if_present = dict( 138 | cmds = ["create"], 139 | arg = "NAME", 140 | desc = "exclude directories that are tagged by containing a filesystem object with the given NAME", 141 | ), 142 | lock_wait = dict( 143 | cmds = ["all"], 144 | arg = "SECONDS", 145 | desc = "wait at most SECONDS for acquiring a repository/cache lock (default: 1)", 146 | ), 147 | keep_within = dict( 148 | cmds = ["prune"], 149 | arg = "INTERVAL", 150 | desc = "keep all archives within this time interval" 151 | ), 152 | keep_last = dict( 153 | cmds = ["prune"], 154 | arg = "NUM", 155 | desc = "number of the most recent archives to keep" 156 | ), 157 | keep_minutely = dict( 158 | cmds = ["prune"], 159 | arg = "NUM", 160 | desc = "number of minutely archives to keep" 161 | ), 162 | keep_hourly = dict( 163 | cmds = ["prune"], 164 | arg = "NUM", 165 | desc = "number of hourly archives to keep" 166 | ), 167 | keep_daily = dict( 168 | cmds = ["prune"], 169 | arg = "NUM", 170 | desc = "number of daily archives to keep" 171 | ), 172 | keep_weekly = dict( 173 | cmds = ["prune"], 174 | arg = "NUM", 175 | desc = "number of weekly archives to keep" 176 | ), 177 | keep_monthly = dict( 178 | cmds = ["prune"], 179 | arg = "NUM", 180 | desc = "number of monthly archives to keep" 181 | ), 182 | keep_yearly = dict( 183 | cmds = ["prune"], 184 | arg = "NUM", 185 | desc = "number of yearly archives to keep" 186 | ), 187 | one_file_system = dict( 188 | cmds = ["create"], 189 | desc = "stay in the same file system and do not store mount points of other file systems", 190 | ), 191 | remote_path = dict( 192 | cmds = ["all"], 193 | arg = "CMD", 194 | desc = "name of borg executable on remote platform", 195 | ), 196 | remote_ratelimit = dict( 197 | cmds = ["all"], 198 | arg = "RATE", 199 | desc = "set remote network upload rate limit in kiB/s (default: 0=unlimited)", 200 | ), 201 | sparse = dict( 202 | cmds = ["create"], 203 | desc = "detect sparse holes in input (supported only by fixed chunker)" 204 | ), 205 | threshold = dict( 206 | cmds = ["compact"], 207 | arg = "PERCENT", 208 | desc = "set minimum threshold in percent for saved space when compacting (default: 10)", 209 | ), 210 | umask = dict( 211 | cmds = ["all"], 212 | arg = "M", 213 | desc = "set umask to M (local and remote, default: 0077)" 214 | ), 215 | upload_buffer = dict( 216 | cmds = ["all"], 217 | arg = "UPLOAD_BUFFER", 218 | desc = "set network upload buffer size in MiB (default: 0=no buffer)", 219 | ), 220 | upload_ratelimit = dict( 221 | cmds = ["all"], 222 | arg = "RATE", 223 | desc = "set rate limit in kiB/s, used when writing to a remote network (default: 0=unlimited)", 224 | ), 225 | prefix = dict( 226 | cmds = ["check", "delete", "info", "list", "mount", "prune"], 227 | arg = "PREFIX", 228 | desc = "only consider archive names starting with this prefix", 229 | ), 230 | glob_archives = dict( 231 | cmds = ["check", "delete", "info", "list", "mount", "prune"], 232 | arg = "GLOB", 233 | desc = "only consider archive names matching the glob", 234 | ), 235 | ) 236 | 237 | # Utilities {{{2 238 | # utility function that converts setting names to borg option names 239 | def convert_name_to_option(name): 240 | return "--" + name.replace("_", "-") 241 | 242 | 243 | # Initial contents of files {{{2 244 | INITIAL_SETTINGS_FILE_CONTENTS = dedent( 245 | """ 246 | # These settings are common to all configurations 247 | 248 | # configurations 249 | configurations = '⟪list your configurations here⟫' 250 | default_configuration = '⟪default-config⟫' 251 | 252 | # encryption 253 | encryption = '⟪encryption⟫' # borg encryption method 254 | # Common choices are 'repokey' and 'keyfile'. 255 | # With 'repokey' the encryption key is copied into repository, use this 256 | # only if the remote repository is owned by you and is secure. 257 | # With 'keyfile' the encryption key is only stored locally. Be sure to 258 | # export it and save a copy in a safe place, otherwise you may not be 259 | # able to access your backups if you lose your disk. 260 | # specify either passphrase or avendesora_account 261 | passphrase = '⟪passcode⟫' # passphrase for encryption key 262 | avendesora_account = '⟪account-name⟫' # avendesora account holding passphrase 263 | 264 | # basic settings 265 | # specify notify if batch and notifier if interactive 266 | notify = '⟪your-email-address⟫' # who to notify when things go wrong 267 | notifier = 'notify-send -u normal {prog_name} "{msg}"' 268 | # interactive notifier program 269 | prune_after_create = True # automatically run prune after a backup 270 | check_after_create = 'latest' # automatically run check after a backup 271 | compact_after_delete = True # automatically run compact after a delete or prune 272 | 273 | # repository settings 274 | repository = '⟪host⟫:⟪path⟫/{host_name}-{user_name}-{config_name}' 275 | archive = '{host_name}-{{now}}' 276 | # These may contain {} where is any of host_name, user_name, 277 | # prog_name config_name, or any of the user specified settings. 278 | # Double up the braces to specify parameters that should be interpreted 279 | # directly by borg, such as {{now}}. 280 | compression = 'lz4' 281 | 282 | # filter settings 283 | exclude_if_present = '.nobackup' 284 | exclude_caches = True 285 | 286 | # prune settings 287 | keep_within = '1d' # keep all archives created in interval 288 | keep_hourly = 48 # number of hourly archives to keep 289 | keep_daily = 14 # number of daily archives to keep 290 | keep_weekly = 8 # number of weekly archives to keep 291 | keep_monthly = 24 # number of monthly archives to keep 292 | keep_yearly = 2 # number of yearly archives to keep 293 | """ 294 | ).lstrip() 295 | 296 | INITIAL_ROOT_CONFIG_FILE_CONTENTS = dedent( 297 | """ 298 | # Settings for root configuration 299 | # use of absolute paths is recommended 300 | src_dirs = '/' # paths to directories to be backed up 301 | excludes = ''' 302 | /dev 303 | /home/*/.cache 304 | /mnt 305 | /proc 306 | /root/.cache 307 | /run 308 | /sys 309 | /tmp 310 | /var/cache 311 | /var/run 312 | /var/tmp 313 | ''' # list of files or directories to skip 314 | """ 315 | ).lstrip() 316 | 317 | INITIAL_HOME_CONFIG_FILE_CONTENTS = dedent( 318 | """ 319 | # Settings for home configuration 320 | # use of absolute paths is recommended 321 | src_dirs = '~' # absolute path to directory to be backed up 322 | excludes = ''' 323 | ~/.cache 324 | **/*~ 325 | **/__pycache__ 326 | **/*.pyc 327 | **/.*.swp 328 | **/.*.swo 329 | ''' # list of files or directories to skip 330 | """ 331 | ).lstrip() 332 | -------------------------------------------------------------------------------- /emborg/python.py: -------------------------------------------------------------------------------- 1 | # 2 | # Read and Write Python files 3 | # 4 | # Package for reading and writing Python files. 5 | 6 | # License {{{1 7 | # Copyright (C) 2018-2024 Kenneth S. Kundert 8 | # 9 | # This program is free software: you can redistribute it and/or modify it under 10 | # the terms of the GNU General Public License as published by the Free Software 11 | # Foundation, either version 3 of the License, or (at your option) any later 12 | # version. 13 | # 14 | # This program is distributed in the hope that it will be useful, but WITHOUT 15 | # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 | # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 | # details. 18 | # 19 | # You should have received a copy of the GNU General Public License along with 20 | # this program. If not, see http://www.gnu.org/licenses. 21 | 22 | 23 | # Imports {{{1 24 | from inform import Error, display, full_stop, narrate, os_error 25 | from .shlib import cp, to_path 26 | 27 | 28 | # PythonFile class {{{1 29 | class PythonFile: 30 | ActivePythonFile = None 31 | 32 | @classmethod 33 | def get_active_python_file(cls): 34 | return cls.ActivePythonFile 35 | 36 | def __init__(self, *path_components): 37 | self.path = to_path(*path_components) 38 | 39 | def save(self, contents): 40 | path = self.path 41 | path.write_text(contents, encoding="utf-8") 42 | 43 | def read(self): 44 | path = self.path 45 | return path.read_text(encoding="utf-8") 46 | 47 | def remove(self): 48 | self.path.unlink() 49 | 50 | def backup(self, extension): 51 | """Creates a backup copy of the file. 52 | 53 | The name of the new file has the specified extension prepended to the 54 | existing suffixes. 55 | """ 56 | # prepend extension to list of suffixes 57 | suffixes = self.path.suffixes 58 | stem = self.path.stem.partition(".")[0] # remove all suffixes 59 | new = to_path(self.path.parent, "".join([stem, extension] + suffixes)) 60 | self.backup_path = new 61 | 62 | cp(self.path, new) 63 | return new 64 | 65 | def restore(self): 66 | "Restores the backup copy of the file." 67 | cp(self.backup_path, self.path) 68 | 69 | def run(self): 70 | self.ActivePythonFile = self.path 71 | path = self.path 72 | narrate("reading:", path) 73 | try: 74 | self.code = self.read() 75 | # need to save the code for the new command 76 | except OSError as err: 77 | raise Error(os_error(err)) 78 | 79 | try: 80 | compiled = compile(self.code, str(path), "exec") 81 | except SyntaxError as err: 82 | culprit = (err.filename, err.lineno) 83 | if err.text is None or err.offset is None: 84 | raise Error(full_stop(err.msg), culprit=culprit) 85 | else: 86 | raise Error( 87 | err.msg + ":", 88 | err.text.rstrip(), 89 | (err.offset - 1) * " " + "^", 90 | culprit=culprit, 91 | sep="\n", 92 | ) 93 | 94 | contents = {} 95 | try: 96 | exec(compiled, contents) 97 | except Exception as err: 98 | from .utilities import error_source 99 | 100 | raise Error(full_stop(err), culprit=error_source()) 101 | self.ActivePythonFile = None 102 | # strip out keys that start with '__' and return them 103 | return {k: v for k, v in contents.items() if not k.startswith("__")} 104 | 105 | def create(self, contents): 106 | path = self.path 107 | try: 108 | if path.exists(): 109 | # file creation (init) requested, but file already exists 110 | # don't overwrite the file, instead read it so the information 111 | # can be used to create any remaining files. 112 | display("%s: already exists." % path) 113 | return 114 | # create the file 115 | display("%s: creating." % path) 116 | # file is not encrypted 117 | with path.open("wb") as f: 118 | f.write(contents.encode("utf-8")) 119 | except OSError as err: 120 | raise Error(os_error(err)) 121 | 122 | def exists(self): 123 | return self.path.exists() 124 | 125 | def __str__(self): 126 | return str(self.path) 127 | -------------------------------------------------------------------------------- /emborg/utilities.py: -------------------------------------------------------------------------------- 1 | # Utilities 2 | 3 | # License {{{1 4 | # Copyright (C) 2018-2024 Kenneth S. Kundert 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see http://www.gnu.org/licenses/. 18 | 19 | # Imports {{{1 20 | import arrow 21 | import pwd 22 | import os 23 | import socket 24 | import nestedtext as nt 25 | from inform import Error, narrate, os_error, warn 26 | from .shlib import Run, set_prefs as set_shlib_prefs 27 | set_shlib_prefs(use_inform=True, log_cmd=True) 28 | 29 | 30 | # gethostname {{{1 31 | # returns short version of the hostname (the hostname without any domain name) 32 | def gethostname(): 33 | return socket.gethostname().split(".")[0] 34 | 35 | 36 | def getfullhostname(): 37 | return socket.gethostname() 38 | 39 | 40 | # getusername {{{1 41 | def getusername(): 42 | return pwd.getpwuid(os.getuid()).pw_name 43 | 44 | 45 | # pager {{{1 46 | def pager(text): 47 | program = os.environ.get("PAGER", "less") 48 | Run([program], stdin=text, modes="Woes") 49 | 50 | 51 | # two_columns {{{1 52 | def two_columns(col1, col2, width=16, indent=True): 53 | indent = " " 54 | if len(col1) > width: 55 | return "%s%s\n%s%s%s" % (indent, col1, indent, " " + width * " ", col2) 56 | else: 57 | return "%s%-*s %s" % (indent, width, col1, col2) 58 | 59 | 60 | # error_source {{{1 61 | def error_source(): 62 | """Source of error 63 | Reads stack trace to determine filename and line number of error. 64 | """ 65 | import traceback 66 | 67 | try: 68 | # return filename and lineno 69 | # context and content are also available 70 | import sys 71 | 72 | exc_cls, exc, tb = sys.exc_info() 73 | trace = traceback.extract_tb(tb) 74 | filename, line, context, text = trace[-1] 75 | except SyntaxError: 76 | # extract_stack() does not work on binary encrypted files. It generates 77 | # a syntax error that indicates that the file encoding is missing 78 | # because the function tries to read the file and sees binary data. This 79 | # is not a problem with ascii encrypted files as we don't actually show 80 | # code, which is gibberish, but does not require an encoding. In this 81 | # case, extract the line number from the trace. 82 | from .gpg import get_active_python_file 83 | 84 | filename = get_active_python_file() 85 | line = tb.tb_next.tb_lineno 86 | return filename, "line %s" % line 87 | 88 | # when {{{1 89 | def when(time, relative_to=None, as_past=None, as_future=None): 90 | """Converts time into a human friendly description of a time difference 91 | 92 | Takes a time and returns a string that is intended for people. It is a 93 | short description of the time difference between the given time and the 94 | current time or a reference time. It is like arrow.humanize(), but provides 95 | more resolution. It is suitable for use with time differences that exceed 96 | 1 second. Any smaller than that will round to 0. 97 | 98 | Arguments: 99 | time (datetime): 100 | The time of the event. May either be in the future or the past. 101 | relative_to (datetime): 102 | Time to compare against to form the time difference. If not given, 103 | the current time is used. 104 | as_past (bool or str): 105 | If true, the word “ago” will be added to the end of the returned 106 | time difference if it is negative, indicating it occurred in the 107 | past. It it a string, it should contain ‘{}’, which is replaced 108 | with the time difference. 109 | as_future (bool or str): 110 | If true, the word “in” will be added to the front of the returned 111 | time difference if it is positive, indicating it occurs in the 112 | past. It it a string, it should contain ‘{}’, which is replaced 113 | with the time difference. 114 | Returns: 115 | A user friendly string that describes the time difference. 116 | 117 | Examples: 118 | 119 | >>> import arrow 120 | >>> now = arrow.now() 121 | >>> print(when(now.shift(seconds=60.1))) 122 | 1 minute 123 | 124 | >>> print(when(now.shift(seconds=2*60), as_future=True)) 125 | in 2 minutes 126 | 127 | >>> print(when(now.shift(seconds=-60*60), as_past=True)) 128 | 60 minutes ago 129 | 130 | >>> print(when(now.shift(seconds=3.5*60), as_future="{} from now")) 131 | 3.5 minutes from now 132 | 133 | >>> print(when(now.shift(days=-2*365), as_past="last run {} ago")) 134 | last run 2 years ago 135 | """ 136 | 137 | if relative_to is None: 138 | relative_to = arrow.now() 139 | difference = time - relative_to 140 | seconds = 60*60*24*difference.days + difference.seconds 141 | 142 | def fmt(dt, prec, unit): 143 | if prec: 144 | num = f'{dt:0.1f}' 145 | if num.endswith('.0'): 146 | num = num[:-2] 147 | else: 148 | num = f'{dt:0.0f}' 149 | if num == '1': 150 | offset = f'{num} {unit}' 151 | else: 152 | offset = f'{num} {unit}s' 153 | return offset 154 | 155 | if seconds < 0 and as_past: 156 | if as_past is True: 157 | as_past = "{} ago" 158 | 159 | def annotate(dt, prec, unit): 160 | return as_past.format(fmt(dt, prec, unit)) 161 | 162 | elif seconds >= 0 and as_future: 163 | if as_future is True: 164 | as_future = "in {}" 165 | 166 | def annotate(dt, prec, unit): 167 | return as_future.format(fmt(dt, prec, unit)) 168 | 169 | else: 170 | annotate = fmt 171 | 172 | seconds = abs(seconds) 173 | if seconds < 60: 174 | return annotate(seconds, 0, "second") 175 | minutes = seconds / 60 176 | if minutes < 10: 177 | return annotate(minutes, 1, "minute") 178 | if minutes < 120: 179 | return annotate(minutes, 0, "minute") 180 | hours = minutes / 60 181 | if hours < 10: 182 | return annotate(hours, 1, "hour") 183 | if hours < 36: 184 | return annotate(hours, 0, "hour") 185 | days = hours / 24 186 | if days < 14: 187 | return annotate(days, 1, "day") 188 | weeks = days / 7 189 | if weeks < 8: 190 | return annotate(weeks, 0, "week") 191 | months = days / 30 192 | if months < 18: 193 | return annotate(months, 0, "month") 194 | years = days / 365 195 | if years < 10: 196 | return annotate(years, 1, "year") 197 | return annotate(years, 0, "year") 198 | 199 | 200 | # update_latest {{{1 201 | def update_latest(command, path, repo_size=None): 202 | narrate(f"updating date file for {command}: {str(path)}") 203 | latest = {} 204 | try: 205 | latest = nt.load(path, dict) 206 | except nt.NestedTextError as e: 207 | warn(e) 208 | except FileNotFoundError: 209 | pass 210 | except OSError as e: 211 | warn(os_error(e)) 212 | latest[f"{command} last run"] = str(arrow.now()) 213 | if repo_size: 214 | latest['repository size'] = repo_size 215 | elif 'repository size' in latest: 216 | if repo_size is False: 217 | del latest['repository size'] 218 | 219 | try: 220 | nt.dump(latest, path, sort_keys=True) 221 | except nt.NestedTextError as e: 222 | warn(e) 223 | except OSError as e: 224 | warn(os_error(e)) 225 | 226 | # read_latest {{{1 227 | def read_latest(path): 228 | try: 229 | latest = nt.load(path, dict) 230 | for k, v in latest.items(): 231 | if "last run" in k: 232 | try: 233 | latest[k] = arrow.get(v) 234 | except arrow.parser.ParserError: 235 | warn(f"{k}: date not given in iso format.", culprit=path) 236 | return latest 237 | except nt.NestedTextError as e: 238 | raise Error(e) 239 | -------------------------------------------------------------------------------- /od: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from emborg.overdue import main 4 | main() 5 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "emborg" 3 | version = "1.41" 4 | description = "Borg front end." 5 | readme = "README.rst" 6 | keywords = ["emborg", "borg", "borgbackup", "borgmatic", "vorta", "backups"] 7 | authors = [ 8 | {name = "Ken Kundert"}, 9 | {email = "emborg@nurdletech.com"} 10 | ] 11 | classifiers = [ 12 | "Development Status :: 5 - Production/Stable", 13 | "Environment :: Console", 14 | "Intended Audience :: End Users/Desktop", 15 | "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", 16 | "Natural Language :: English", 17 | "Operating System :: POSIX :: Linux", 18 | "Programming Language :: Python :: 3", 19 | "Topic :: Utilities", 20 | ] 21 | requires-python = ">=3.6" 22 | dependencies = [ 23 | "appdirs", 24 | "arrow", 25 | # "avendesora", # optional 26 | "docopt", 27 | "inform>=1.31", 28 | "nestedtext", 29 | "quantiphy", 30 | "requests", 31 | ] 32 | 33 | [project.optional-dependencies] 34 | avendesora = [ 35 | 'avendesora', 36 | ] 37 | 38 | [project.scripts] 39 | emborg = "emborg.main:main" 40 | emborg-overdue = "emborg.overdue:main" 41 | 42 | [project.urls] 43 | homepage = "https://emborg.readthedocs.io" 44 | documentation = "https://emborg.readthedocs.io" 45 | repository = "https://github.com/kenkundert/emborg" 46 | changelog = "https://emborg.readthedocs.io/en/latest/releases.html" 47 | 48 | [build-system] 49 | requires = ["flit_core >=2,<4"] 50 | build-backend = "flit_core.buildapi" 51 | 52 | [tool.ruff] 53 | exclude = [".tox", "doc"] 54 | 55 | [tool.ruff.lint] 56 | select = ["F"] 57 | ignore = [] 58 | 59 | [tool.ruff.lint.per-file-ignores] 60 | "__init__.py" = ["F401"] 61 | "emborg/overdue.py" = ["F841"] 62 | 63 | -------------------------------------------------------------------------------- /tests/INITIAL_CONFIGS/home: -------------------------------------------------------------------------------- 1 | # Settings for home configuration 2 | # use of absolute paths is recommended 3 | src_dirs = '~' # absolute path to directory to be backed up 4 | excludes = ''' 5 | ~/.cache 6 | **/*~ 7 | **/__pycache__ 8 | **/*.pyc 9 | **/.*.swp 10 | **/.*.swo 11 | ''' # list of files or directories to skip 12 | -------------------------------------------------------------------------------- /tests/INITIAL_CONFIGS/root: -------------------------------------------------------------------------------- 1 | # Settings for root configuration 2 | # use of absolute paths is recommended 3 | src_dirs = '/' # paths to directories to be backed up 4 | excludes = ''' 5 | /dev 6 | /home/*/.cache 7 | /mnt 8 | /proc 9 | /root/.cache 10 | /run 11 | /sys 12 | /tmp 13 | /var/cache 14 | /var/lib/dnf 15 | /var/lock 16 | /var/run 17 | /var/tmp 18 | ''' # list of files or directories to skip 19 | -------------------------------------------------------------------------------- /tests/INITIAL_CONFIGS/settings: -------------------------------------------------------------------------------- 1 | # These settings are common to all configurations 2 | 3 | # configurations 4 | configurations = '⟪list your configurations here⟫' 5 | default_configuration = '⟪default-config⟫' 6 | 7 | # encryption 8 | encryption = '⟪encryption⟫' # borg encryption method 9 | # Common choices are 'repokey' and 'keyfile'. 10 | # With 'repokey' the encryption key is copied into repository, use this 11 | # only if the remote repository is owned by you and is secure. 12 | # With 'keyfile' the encryption key is only stored locally. Be sure to 13 | # export it and save a copy in a safe place, otherwise you may not be 14 | # able to access your backups if you lose your disk. 15 | # specify either passphrase or avendesora_account 16 | passphrase = '⟪passcode⟫' # passphrase for encryption key 17 | avendesora_account = '⟪account-name⟫' # avendesora account holding passphrase 18 | 19 | # basic settings 20 | # specify notify if batch and notifier if interactive 21 | notify = '⟪your-email-address⟫' # who to notify when things go wrong 22 | notifier = 'notify-send -u normal {prog_name} "{msg}"' 23 | # interactive notifier program 24 | prune_after_create = True # automatically run prune after a backup 25 | check_after_create = 'latest' # automatically run check after a backup 26 | compact_after_delete = True # automatically run compact after a delete or prune 27 | 28 | # repository settings 29 | repository = '⟪host⟫:⟪path⟫/{host_name}-{user_name}-{config_name}' 30 | archive = '{host_name}-{{now}}' 31 | # These may contain {} where is any of host_name, user_name, 32 | # prog_name config_name, or any of the user specified settings. 33 | # Double up the braces to specify parameters that should be interpreted 34 | # directly by borg, such as {{now}}. 35 | compression = 'lz4' 36 | 37 | # filter settings 38 | exclude_if_present = '.nobackup' 39 | exclude_caches = True 40 | 41 | # prune settings 42 | keep_within = '1d' # keep all archives created in interval 43 | keep_hourly = 48 # number of hourly archives to keep 44 | keep_daily = 14 # number of daily archives to keep 45 | keep_weekly = 8 # number of weekly archives to keep 46 | keep_monthly = 24 # number of monthly archives to keep 47 | keep_yearly = 2 # number of yearly archives to keep 48 | -------------------------------------------------------------------------------- /tests/README: -------------------------------------------------------------------------------- 1 | Certain CI environments make it difficult or impossible to install all of 2 | the dependencies needed to test Emborg. This is common with two important 3 | dependencies: Borg and Fuse. 4 | 5 | Borg is needed for virtually all tests, so no accommodations are made for any 6 | testing if Borg is missing. 7 | 8 | Fuse is needed by the mount and compare commands; it is used by Borg when 9 | mounting remote archives. 10 | 11 | You can skip tests dependent on Fuse by defining the MISSING_DEPENDENCIES 12 | environment variable. To do so, add the following lines to the tox.ini file: 13 | 14 | [testenv:pytest] 15 | setenv = ← add 16 | MISSING_DEPENDENCIES = fuse ← add 17 | commands = py.test -vv --cov {posargs} 18 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/README: -------------------------------------------------------------------------------- 1 | There are 9 base configurations and one composite configurations. 2 | Before the start of testing, 'STARTING_CONFIGS' is copied to 'configs' 3 | 4 | The tests back up only this directory or the 'configs' subdirectory 5 | - need a controlled set of files to test manifest 6 | 7 | test0: 8 | - backs up the ./configs directory 9 | - uses excludes to select which files to backup 10 | - uses default mount point 11 | 12 | test1: 13 | - backs up the . directory 14 | - uses excludes to select which files to backup 15 | - does not use default mount point 16 | 17 | test2: 18 | - backs up the . directory 19 | - uses encryption and passcommand 20 | - uses exclude_from to select which files to backup 21 | - does not use default mount point 22 | 23 | test3: 24 | - backs up the . directory 25 | - uses encryption and passphrase 26 | - uses patterns with simple excludes to select which files to backup 27 | - does not use default mount point 28 | 29 | test4: 30 | - backs up the ./configs directory 31 | - uses patterns with root and excludes to select which files to backup 32 | - sets working_dir to .. (relative paths) 33 | - uses default mount point 34 | 35 | test5: 36 | - backs up the . directory 37 | - uses patterns with root, includes and excludes to select which files to 38 | backup 39 | - uses default mount point 40 | 41 | test6: 42 | - backs up the . directory 43 | - uses patterns_from with root and excludes to select which files to backup 44 | - sets working_dir to .. (relative paths) 45 | - uses default mount point 46 | 47 | test7: 48 | - backs up the . directory 49 | - uses patterns_from with root, includes and excludes to select which files 50 | to backup 51 | - uses default mount point 52 | 53 | test8: 54 | - backs up the ./configs.symlink/subdir directory 55 | - uses patterns with root and excludes to select which files to backup 56 | - sets working_dir to . (relative paths) 57 | - uses default mount point 58 | 59 | tests contains test{0,1,2,3} 60 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/overdue.conf: -------------------------------------------------------------------------------- 1 | default_maintainer = 'nobody@nowhere.com' 2 | dumper = 'nobody@nowhere.com' 3 | default_max_age = 36 # hours 4 | root = '~/.local/share/emborg' 5 | status_message = "{host}: {hours:0.1f} hours" 6 | repositories = [ 7 | dict(host='test0', path='test0.latest.nt', max_age=0.2), 8 | dict(host='test1', path='test1.latest.nt'), 9 | dict(host='test2', path='test2.latest.nt'), 10 | dict(host='test3', path='test3.latest.nt'), 11 | dict(host='test4', path='test4.latest.nt'), 12 | dict(host='test5', path='test5.latest.nt'), 13 | dict(host='test6', path='test6.latest.nt'), 14 | dict(host='test7', path='test7.latest.nt'), 15 | ] 16 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/settings: -------------------------------------------------------------------------------- 1 | # These settings are common to all configurations 2 | # configurations 3 | configurations = ''' 4 | test0 test1 test2 test3 test4 test5 test6 test7 test8 test9 5 | tests=test0,test1,test2,test3 6 | ''' 7 | 8 | default_configuration = 'tests' 9 | 10 | # repository/archive settings 11 | repository = '~/repositories' 12 | archive = '{config_name}-{{now}}' 13 | glob_archives = '{config_name}-*' 14 | encryption = 'none' 15 | 16 | # shared filter settings 17 | exclude_if_present = '.nobackup' 18 | exclude_caches = True 19 | 20 | # prune settings 21 | keep_within = '1d' # keep all archives created within this interval 22 | prune_after_create = True 23 | check_after_create = True 24 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/subdir/file: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KenKundert/emborg/542f58626eb9965a5ee3952f432589507e6e80fc/tests/STARTING_CONFIGS/subdir/file -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/test0: -------------------------------------------------------------------------------- 1 | src_dirs = '~/configs'.split() 2 | 3 | excludes = [ 4 | 'fm:*/.viminfo', 5 | 'sh:**/.*.swp', 6 | 'sh:**/.*.swo', 7 | ] 8 | 9 | run_before_backup = """ 10 | # this line is ignored 11 | echo run_before_backup on test0 12 | """ 13 | run_before_first_backup = "echo run_before_first_backup on test0" 14 | run_before_borg = ["echo run_before_borg on test0"] 15 | run_after_backup = """ 16 | # this line is ignored 17 | echo run_after_backup on test0 18 | """ 19 | run_after_last_backup = [ 20 | "echo run_after_last_backup on test0", 21 | ] 22 | run_after_borg = "echo run_after_borg on test0" 23 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/test1: -------------------------------------------------------------------------------- 1 | src_dirs = '~'.split() 2 | 3 | excludes = ''' 4 | # this is a comment 5 | ~/repositories 6 | ~/.local 7 | ~/tests 8 | 9 | # here is another comment 10 | fm:*/.viminfo 11 | sh:**/.*.swp 12 | sh:**/.*.swo 13 | ''' 14 | 15 | run_before_backup = """ 16 | # this line is ignored 17 | echo run_before_backup on test1 18 | """ 19 | run_before_first_backup = "echo run_before_first_backup on test1" 20 | run_before_borg = ["echo run_before_borg on test1"] 21 | run_after_backup = """ 22 | # this line is ignored 23 | echo run_after_backup on test1 24 | """ 25 | run_after_last_backup = [ 26 | "echo run_after_last_backup on test1", 27 | ] 28 | run_after_borg = "echo run_after_borg on test1" 29 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/test2: -------------------------------------------------------------------------------- 1 | encryption = 'repokey' 2 | passcommand = 'cat {config_dir}/test2passphrase' 3 | 4 | src_dirs = '~'.split() 5 | 6 | exclude_from = '{config_dir}/test2excludes' 7 | 8 | run_before_backup = """ 9 | # this line is ignored 10 | echo run_before_backup on test2 11 | """ 12 | run_before_first_backup = "echo run_before_first_backup on test2" 13 | run_before_borg = ["echo run_before_borg on test2"] 14 | run_after_backup = """ 15 | # this line is ignored 16 | echo run_after_backup on test2 17 | """ 18 | run_after_last_backup = [ 19 | "echo run_after_last_backup on test2", 20 | ] 21 | run_after_borg = "echo run_after_borg on test2" 22 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/test2excludes: -------------------------------------------------------------------------------- 1 | # here is another comment 2 | fm:*/.viminfo 3 | sh:**/.*.swp 4 | sh:**/.*.swo 5 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/test2passphrase: -------------------------------------------------------------------------------- 1 | pretty-kitty 2 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/test3: -------------------------------------------------------------------------------- 1 | src_dirs = '~'.split() 2 | encryption = 'repokey' 3 | passphrase = 'pretty kitty' 4 | 5 | patterns = '''- ~/repositories 6 | - ~/.local 7 | - ~/tests 8 | - **/.viminfo 9 | - **/.*.swp 10 | - **/.*.swo''' 11 | 12 | run_before_backup = """ 13 | # this line is ignored 14 | echo run_before_backup on test3 15 | """ 16 | run_before_first_backup = "echo run_before_first_backup on test3" 17 | run_before_borg = ["echo run_before_borg on test3"] 18 | run_after_backup = """ 19 | # this line is ignored 20 | echo run_after_backup on test3 21 | """ 22 | run_after_last_backup = [ 23 | "echo run_after_last_backup on test3", 24 | ] 25 | run_after_borg = "echo run_after_borg on test3" 26 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/test4: -------------------------------------------------------------------------------- 1 | from shlib import cwd 2 | current_dir = cwd() 3 | parent_dir = str(current_dir.parent) 4 | this_dir = current_dir.name 5 | working_dir = parent_dir 6 | patterns = f""" 7 | R {this_dir}/configs 8 | - fm:{this_dir}/*/.viminfo 9 | - {this_dir}/**/.*.swp 10 | - {this_dir}/**/.*.swo 11 | """ 12 | default_mount_point = '~/EMBORG' 13 | report_diffs_cmd = "diff -r" 14 | 15 | run_before_backup = """ 16 | # this line is ignored 17 | echo run_before_backup 0 18 | echo run_before_backup 1 19 | """ 20 | run_before_first_backup = [ 21 | "echo run_before_first_backup 0", 22 | "echo run_before_first_backup 1", 23 | ] 24 | run_before_borg = """ 25 | # this line is ignored 26 | echo run_before_borg 0 27 | echo run_before_borg 1 28 | """ 29 | run_after_backup = """ 30 | # this line is ignored 31 | echo run_after_backup 0 32 | echo run_after_backup 1 33 | """ 34 | run_after_last_backup = [ 35 | "echo run_after_last_backup 0", 36 | "echo run_after_last_backup 1", 37 | ] 38 | run_after_borg = """ 39 | # this line is ignored 40 | echo run_after_borg 0 41 | echo run_after_borg 1 42 | """ 43 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/test5: -------------------------------------------------------------------------------- 1 | patterns = ''' 2 | R ~ 3 | + ~/configs 4 | - ~ 5 | - fm:*/.viminfo 6 | - ~/**/.*.swp 7 | - **/.*.swo 8 | ''' 9 | default_mount_point = '{home_dir}/EMBORG' 10 | manage_diffs_cmd = "diff -r" 11 | verbose = True 12 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/test6: -------------------------------------------------------------------------------- 1 | from shlib import cwd 2 | working_dir = str(cwd().parent) 3 | patterns_from = '{config_dir}/test6patterns' 4 | default_mount_point = '~/EMBORG' 5 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/test6patterns: -------------------------------------------------------------------------------- 1 | R ./tests/configs 2 | - fm:./tests/*/.viminfo 3 | - ./tests/**/.*.swp 4 | - ./tests/**/.*.swo 5 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/test7: -------------------------------------------------------------------------------- 1 | patterns_from = '{config_dir}/test7patterns' 2 | default_mount_point = '{home_dir}/EMBORG' 3 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/test7patterns: -------------------------------------------------------------------------------- 1 | R ⟪EMBORG⟫/tests 2 | + ⟪EMBORG⟫/tests/configs 3 | - ⟪EMBORG⟫/tests 4 | - fm:*/.viminfo 5 | - **/.*.swp 6 | - **/.*.swo 7 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/test8: -------------------------------------------------------------------------------- 1 | from shlib import cwd 2 | working_dir = str(cwd()) 3 | patterns = ''' 4 | R ./configs.symlink/subdir 5 | - fm:*/.viminfo 6 | - **/.*.swp 7 | - **/.*.swo 8 | ''' 9 | default_mount_point = '~/EMBORG' 10 | -------------------------------------------------------------------------------- /tests/STARTING_CONFIGS/test9: -------------------------------------------------------------------------------- 1 | patterns = ''' 2 | R ~ 3 | + ~/configs 4 | - ~ 5 | - fm:*/.viminfo 6 | - ~/**/.*.swp 7 | - **/.*.swo 8 | ''' 9 | compact_after_delete = True 10 | -------------------------------------------------------------------------------- /tests/clean: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ -d EMBORG/home ] || [ -d EMBORG/configs.symlink ]; then 4 | fusermount -u EMBORG 5 | fi 6 | 7 | rm -f configs.symlink 8 | rm -rf configs .config .local repositories home __pycache__ EMBORG 9 | rm -rf .cache .hypothesis .test.sum .coverage .test.emborg.sum 10 | -------------------------------------------------------------------------------- /tests/test_emborg.py: -------------------------------------------------------------------------------- 1 | # Test Emborg 2 | # 3 | # These tests require Borg Backup (borg) to be available. 4 | # 5 | # Some tests require FUSE (Filesystem in User Space). Pass “--no-fuse” as 6 | # command line option to pytest to skip those tests if FUSE is not available. 7 | # 8 | # Some tests require a specific version borg version or newer to be available. 9 | # Add “--borg-version=❬version❭” to indicate your version. Typically this is 10 | # done using “--borg-version=$(borg --version)” (bash) or “--borg-version="`borg 11 | # --version`"”. If you do not specify “--borg-version”, the latest version is 12 | # assumed. 13 | # 14 | # When using tox, you can pass these options through tox using: 15 | # tox -- --borg-version=❬version❭ --no-fuse 16 | # Anything after the first -- is passed to pytest. 17 | 18 | # Imports {{{1 19 | import arrow 20 | import json 21 | import os 22 | from parametrize_from_file import parametrize 23 | from functools import partial 24 | import pytest 25 | import re 26 | from inform import is_str, Color, Error, dedent, indent 27 | from shlib import Run, cd, cp, cwd, ln, lsf, mkdir, rm, set_prefs, to_path, touch 28 | from voluptuous import Schema, Optional, Required, Any 29 | 30 | 31 | # Adapt parametrize_for_file to read dictionary rather than list 32 | def name_from_dict_keys(cases): 33 | return [{**v, 'name': k} for k,v in cases.items()] 34 | 35 | parametrize = partial(parametrize, preprocess=name_from_dict_keys) 36 | 37 | 38 | # Globals {{{1 39 | # uses local versions of emborg (../bu) and emborg_overdue (../od) 40 | emborg_exe = "../bu".split() 41 | emborg_overdue_exe = "../od".split() 42 | # uses installed code 43 | #emborg_exe = "emborg".split() 44 | #emborg_overdue_exe = "emborg-overdue".split() 45 | set_prefs(use_inform=True) 46 | tests_dir = to_path(__file__).parent 47 | emborg_dir = str(tests_dir.parent) 48 | emborg_dir_wo_slash = emborg_dir.strip('/') 49 | # remove the leading slashes, it will be added back in tests if needed 50 | 51 | # schema for test cases {{{2 52 | emborg_schema = Schema({ 53 | Required('name'): str, # this field is promoted to key by above code 54 | Optional('args', default='❬PASS❭'): Any(str, list), 55 | Optional('expected', default=""): str, 56 | Optional('expected_type', default=""): str, 57 | Optional('cmp_dirs', default=""): str, 58 | Optional('remove', default=""): str, 59 | Optional('dependencies', default=""): str, 60 | }, required=True) 61 | emborg_overdue_schema = Schema({ 62 | Required('name'): str, 63 | Optional('conf', default=""): str, 64 | Optional('args', default=[]): Any(str, list), 65 | Required('expected', default=""): str, 66 | Required('expected_type', default=""): str, 67 | Optional('dependencies', default=""): str, 68 | }, required=True) 69 | 70 | # EmborgTester class {{{1 71 | class EmborgTester(object): 72 | # constructor {{{2 73 | def __init__(self, args, expected, expected_type, cmp_dirs, remove): 74 | # args are the arguments to the emborg command 75 | # If expected, stdout/stderr should match the given value 76 | # expected_type may contain keywords, 'regex', 'diff', 'error', and/or 77 | # 'ignore' 78 | # - if regex is given then expected is taken to be a regular expression 79 | # otherwise the result much match expected verbatim 80 | # - if diff is given then emborg is expected to exit with an exit status of 1 81 | # - if error is given then emborg is expected to exit with an exit status of 2 82 | # - if ignore is given, stdout/stderr is not checked 83 | # cmp_dirs is a pair of directories that, if given, should match exactly 84 | # remove contains files or directories to be deleted before the test runs 85 | # 86 | # args, expected, and cmp_dirs may contain the literal text fragment: ⟪EMBORG⟫ 87 | # that is replaced by the absolute path to the emborg directory: .../emborg 88 | 89 | # replace ⟪EMBORG⟫ and ⟪DATE⟫ macros 90 | date = arrow.now().format("YYYY-MM-DD") 91 | args = args.split() if is_str(args) else args 92 | args = [a.replace("⟪EMBORG⟫", emborg_dir_wo_slash) for a in args] 93 | args = [a.replace("⟪DATE⟫", date) for a in args] 94 | if expected is not None: 95 | expected = expected.replace("⟪EMBORG⟫", emborg_dir_wo_slash) 96 | expected = expected.replace("⟪DATE⟫", date) 97 | if cmp_dirs: 98 | cmp_dirs = cmp_dirs.replace("⟪EMBORG⟫", emborg_dir_wo_slash) 99 | cmp_dirs = cmp_dirs.replace("⟪DATE⟫", date) 100 | 101 | self.args = args 102 | self.expected = expected 103 | self.expected_type = expected_type.split() 104 | self.cmp_dirs = cmp_dirs.split() if is_str(cmp_dirs) else cmp_dirs 105 | self.cmd = emborg_exe + self.args 106 | self.command = " ".join(self.cmd) 107 | self.remove = remove 108 | self.diffout = None 109 | 110 | # run() {{{2 111 | def run(self): 112 | # remove requested files and directories 113 | if self.remove: 114 | rm(self.remove.split()) 115 | if '❬PASS❭' in self.args: 116 | return True 117 | 118 | # run command 119 | emborg = Run(self.cmd, "sOMW*") 120 | self.result = dedent(Color.strip_colors(emborg.stdout)).strip("\n") 121 | if emborg.stdout: 122 | print(emborg.stdout) 123 | 124 | # check stdout 125 | matches = True 126 | if 'ignore' not in self.expected_type: 127 | expected, result = self.expected, self.result 128 | if 'sort-lines' in self.expected_type: 129 | expected = '\n'.join(sorted(expected.splitlines())) 130 | result = '\n'.join(sorted(result.splitlines())) 131 | if 'regex' in self.expected_type: 132 | matches = bool(re.fullmatch(expected, result)) 133 | else: 134 | if 'error' in self.expected_type: 135 | # because error messages use wrapping the white space can 136 | # vary, so simply eliminate the white space 137 | matches = expected.split() == result.split() 138 | else: 139 | matches = expected == result 140 | 141 | if matches and self.cmp_dirs: 142 | gen_dir, ref_dir = self.cmp_dirs 143 | #self.expected = '' 144 | try: 145 | diff = Run(["diff", "--recursive", gen_dir, ref_dir], "sOMW") 146 | self.result = diff.stdout 147 | except Error as e: 148 | self.result = e.stdout 149 | matches = False 150 | 151 | # check exit status 152 | self.exit_status = emborg.status 153 | self.expected_exit_status = 0 154 | if 'diff' in self.expected_type: 155 | self.expected_exit_status = 1 156 | if 'error' in self.expected_type: 157 | self.expected_exit_status = 2 158 | if self.exit_status != self.expected_exit_status: 159 | matches = False 160 | 161 | return matches 162 | 163 | # get_expected() {{{2 164 | def get_expected(self): 165 | if "\n" in self.expected: 166 | expected = "\n" + indent(self.expected, stops=2) 167 | else: 168 | expected = self.expected 169 | return dict(exit_status=self.expected_exit_status, output=expected) 170 | 171 | # get_result() {{{2 172 | def get_result(self): 173 | if "\n" in self.result: 174 | result = "\n" + indent(self.result, stops=2) 175 | else: 176 | result = self.result 177 | return dict(exit_status=self.exit_status, output=result) 178 | 179 | 180 | # Setup {{{1 181 | # dependency_options fixture {{{3 182 | # This fixture is defined at the top-level in ../conftest.py. 183 | 184 | # initialize fixture {{{3 185 | @pytest.fixture(scope="session") 186 | def initialize(dependency_options): 187 | with cd(tests_dir): 188 | rm("configs .config .local repositories configs.symlink".split()) 189 | cp("STARTING_CONFIGS", "configs") 190 | mkdir(".config repositories .local".split()) 191 | ln("~/.local/bin", ".local/bin") 192 | ln("~/.local/lib", ".local/lib") 193 | ln("configs", "configs.symlink") 194 | os.environ["XDG_CONFIG_HOME"] = str(f"{cwd()}/.config") 195 | os.environ["XDG_DATA_HOME"] = str(f"{cwd()}/.local/share") 196 | os.environ["HOME"] = str(cwd()) 197 | return dependency_options 198 | 199 | # initialize_configs fixture {{{3 200 | @pytest.fixture(scope="session") 201 | def initialize_configs(initialize, dependency_options): 202 | with cd(tests_dir): 203 | cp("STARTING_CONFIGS", ".config/emborg") 204 | rm(".config/emborg/subdir") 205 | for p in lsf(".config/emborg"): 206 | contents = p.read_text() 207 | contents = contents.replace('⟪EMBORG⟫', emborg_dir) 208 | p.write_text(contents) 209 | touch(".config/.nobackup") 210 | return dependency_options 211 | 212 | # missing dependencies {{{2 213 | def skip_test_if_missing_dependencies(cmdline_options, dependencies): 214 | if cmdline_options.borg_version < (1, 2, 0): 215 | if 'borg1.2' in dependencies: 216 | pytest.skip("test requires borg 1.2 or higher") 217 | if cmdline_options.fuse_is_missing and 'fuse' in dependencies: 218 | pytest.skip("test requires fuse") 219 | 220 | # Tests {{{1 221 | # test_emborg_without_configs{{{2 222 | #@pytest.fixture(scope='session') 223 | @parametrize( 224 | path = f'{tests_dir}/test-cases.nt', 225 | key = 'emborg without configs', 226 | schema = emborg_schema 227 | ) 228 | def test_emborg_without_configs( 229 | initialize, 230 | name, args, expected, expected_type, cmp_dirs, remove, dependencies 231 | ): 232 | skip_test_if_missing_dependencies(initialize, dependencies) 233 | with cd(tests_dir): 234 | tester = EmborgTester(args, expected, expected_type, cmp_dirs, remove) 235 | passes = tester.run() 236 | if not passes: 237 | result = tester.get_result() 238 | expected = tester.get_expected() 239 | assert result == expected, f"Test ‘{name}’ fails." 240 | raise AssertionError('test code failure') 241 | 242 | # test_emborg_with_configs{{{2 243 | @parametrize( 244 | path = f'{tests_dir}/test-cases.nt', 245 | key = 'emborg with configs', 246 | schema = emborg_schema 247 | ) 248 | def test_emborg_with_configs( 249 | initialize_configs, 250 | name, args, expected, expected_type, cmp_dirs, remove, dependencies 251 | ): 252 | skip_test_if_missing_dependencies(initialize_configs, dependencies) 253 | with cd(tests_dir): 254 | tester = EmborgTester(args, expected, expected_type, cmp_dirs, remove) 255 | passes = tester.run() 256 | if not passes: 257 | result = tester.get_result() 258 | expected = tester.get_expected() 259 | assert result == expected, f"Test ‘{name}’ fails." 260 | raise AssertionError('test code failure') 261 | 262 | # test_emborg_overdue {{{2 263 | @parametrize( 264 | path = f'{tests_dir}/test-cases.nt', 265 | key = 'emborg-overdue', 266 | schema = emborg_overdue_schema 267 | ) 268 | def test_emborg_overdue( 269 | initialize, 270 | name, conf, args, expected, expected_type, dependencies 271 | ): 272 | skip_test_if_missing_dependencies(initialize, dependencies) 273 | with cd(tests_dir): 274 | if conf: 275 | with open('.config/overdue.conf', 'w') as f: 276 | f.write(conf) 277 | try: 278 | args = args.split() if is_str(args) else args 279 | overdue = Run(emborg_overdue_exe + args, "sOEW") 280 | if 'regex' in expected_type.split(): 281 | assert bool(re.fullmatch(expected, overdue.stdout)), f"Test ‘{name}’ fails." 282 | else: 283 | assert expected == overdue.stdout, f"Test ‘{name}’ fails." 284 | except Error as e: 285 | assert str(e) == expected, f"Test ‘{name}’ fails." 286 | 287 | 288 | # test_emborg_api {{{2 289 | def test_emborg_api(initialize): 290 | from emborg import Emborg 291 | 292 | # get available configs 293 | # do this before changing the working directory so as to test the ability 294 | # to explicitly set the config_dir 295 | config_dir = tests_dir / '.config/emborg' 296 | try: 297 | with Emborg('tests', config_dir=config_dir) as emborg: 298 | configs = emborg.configs 299 | except Error as e: 300 | e.report() 301 | assert not e, str(e) 302 | assert configs == 'test0 test1 test2 test3'.split() 303 | 304 | with cd(tests_dir): 305 | 306 | # list each config and assure than it contains expected paths 307 | for config in configs: 308 | try: 309 | # get the name of latest archive 310 | with Emborg(config) as emborg: 311 | borg = emborg.run_borg( 312 | cmd = 'list', 313 | args = ['--json', emborg.destination()] 314 | ) 315 | response = json.loads(borg.stdout) 316 | archive = response['archives'][-1]['archive'] 317 | 318 | # list files in latest archive 319 | borg = emborg.run_borg( 320 | cmd = 'list', 321 | args = ['--json-lines', emborg.destination(archive)] 322 | ) 323 | json_data = '[' + ','.join(borg.stdout.splitlines()) + ']' 324 | response = json.loads(json_data) 325 | paths = sorted([entry['path'] for entry in response]) 326 | for each in [ 327 | '⟪EMBORG⟫/tests/configs', 328 | '⟪EMBORG⟫/tests/configs/README', 329 | '⟪EMBORG⟫/tests/configs/overdue.conf', 330 | '⟪EMBORG⟫/tests/configs/settings', 331 | '⟪EMBORG⟫/tests/configs/subdir', 332 | '⟪EMBORG⟫/tests/configs/subdir/file', 333 | '⟪EMBORG⟫/tests/configs/test0', 334 | '⟪EMBORG⟫/tests/configs/test1', 335 | '⟪EMBORG⟫/tests/configs/test2', 336 | '⟪EMBORG⟫/tests/configs/test2excludes', 337 | '⟪EMBORG⟫/tests/configs/test2passphrase', 338 | '⟪EMBORG⟫/tests/configs/test3', 339 | '⟪EMBORG⟫/tests/configs/test4', 340 | '⟪EMBORG⟫/tests/configs/test5', 341 | '⟪EMBORG⟫/tests/configs/test6', 342 | '⟪EMBORG⟫/tests/configs/test6patterns', 343 | '⟪EMBORG⟫/tests/configs/test7', 344 | '⟪EMBORG⟫/tests/configs/test7patterns', 345 | '⟪EMBORG⟫/tests/configs/test8', 346 | ]: 347 | each = each.replace("⟪EMBORG⟫", emborg_dir_wo_slash) 348 | assert each in paths 349 | except Error as e: 350 | e.report() 351 | assert not e, str(e) 352 | 353 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = test, types, lint 3 | isolated_build = True 4 | 5 | [testenv:test] 6 | deps = 7 | # borgbackup[pyfuse3]>=1.4.0,<2.0.0 8 | # above breaks github workflow 9 | nestedtext 10 | parametrize-from-file 11 | pytest 12 | pytest-cov 13 | setuptools 14 | shlib 15 | voluptuous 16 | commands = py.test -vv --cov {posargs} --cov-branch --cov-report term 17 | 18 | [testenv:types] 19 | deps = 20 | setuptools 21 | mypy 22 | quantiphy 23 | commands = 24 | mypy --install-types --non-interactive --disable-error-code import {toxinidir}/emborg 25 | 26 | [testenv:lint] 27 | deps = 28 | setuptools 29 | ruff 30 | commands = ruff check 31 | 32 | --------------------------------------------------------------------------------