├── .codespellrc ├── .gitattributes ├── .github └── workflows │ ├── codespell.yaml │ ├── linux.yaml │ ├── pylint-linux.yaml │ ├── pylint-windows.yaml │ └── windows.yaml ├── .gitignore ├── ANTIVIRUS.md ├── CHANGELOG ├── COMPATIBILITY.md ├── LICENSE ├── PRIVATE ├── README.md └── __no_init__.py ├── README.md ├── RESTIC_CMDLINE_CONSISTENCY.md ├── RESTIC_SOURCE_FILES ├── README.txt ├── legacy_restic_builder.cmd ├── restic_0.18.0_windows_legacy_386.exe ├── restic_0.18.0_windows_legacy_amd64.exe ├── update_restic.py └── update_restic.sh ├── ROADMAP.md ├── SECURITY.md ├── UPGRADE_SERVER.md ├── bin ├── COMPILE.cmd ├── COMPILE.sh ├── compile.py ├── npbackup-cli ├── npbackup-gui └── npbackup-viewer ├── examples ├── NPBackup v2 Grafana Dashboard.json ├── NPBackup v3 Grafana Dashboard.json ├── README.md ├── kvm-qemu │ ├── cube-backup.sh │ └── npbackup-cube.conf.template ├── npbackup.linux.conf.dist ├── npbackup.windows.conf.dist ├── systemd │ └── npbackup_upgrade_server.service └── upgrade_server │ ├── npbackup_upgrade_server.conf.dist │ ├── upgrade_script.cmd │ └── upgrade_script.sh ├── excludes ├── generic_excluded_extensions ├── generic_excludes ├── linux_excludes ├── synology_excludes ├── windows_excludes └── windows_program ├── img ├── backup_window_v2.2.0.png ├── configuration_v2.1.0.png ├── configuration_v2.2.0.png ├── configuration_v2.2.0rc2.png ├── configuration_v3.0.0.png ├── grafana_dashboard.png ├── grafana_dashboard_2.2.0.png ├── grafana_dashboard_20250211.png ├── grafana_dashboard_20250226.png ├── interface_v2.1.0.png ├── interface_v2.2.0.png ├── interface_v3.0.0.png ├── orchestrator_v3.0.0.png ├── restore_window_v2.1.0.png ├── restore_window_v2.2.0.png ├── restore_window_v3.0.0.png └── viewer_v3.0.0.png ├── misc └── npbackup-cli.cmd ├── npbackup ├── __debug__.py ├── __env__.py ├── __init__.py ├── __main__.py ├── __version__.py ├── common.py ├── configuration.py ├── core │ ├── __init__.py │ ├── i18n_helper.py │ ├── jobs.py │ ├── nuitka_helper.py │ ├── restic_source_binary.py │ ├── runner.py │ └── upgrade_runner.py ├── gui │ ├── __init__.py │ ├── __main__.py │ ├── config.py │ ├── handle_window.py │ ├── helpers.py │ ├── operations.py │ └── viewer.py ├── key_management.py ├── obfuscation.py ├── path_helper.py ├── requirements-compilation.txt ├── requirements-win32.txt ├── requirements.txt ├── restic_metrics │ ├── __init__.py │ └── requirements.txt ├── restic_wrapper │ ├── __init__.py │ └── schema.py ├── runner_interface.py ├── secret_keys.py ├── task.py ├── translations │ ├── config_gui.en.yml │ ├── config_gui.fr.yml │ ├── generic.en.yml │ ├── generic.fr.yml │ ├── main_gui.en.yml │ ├── main_gui.fr.yml │ ├── operations_gui.en.yml │ └── operations_gui.fr.yml ├── upgrade_client │ ├── __init__.py │ └── upgrader.py └── windows │ └── sign_windows.py ├── resources ├── __init__.py ├── customization.py ├── file_icon.png ├── folder_icon.png ├── inherited_file_icon.png ├── inherited_folder_icon.png ├── inherited_icon.png ├── inherited_irregular_file_icon.png ├── inherited_missing_file_icon.png ├── inherited_neutral_icon.png ├── inherited_symlink_icon.png ├── inherited_tree_icon.png ├── irregular_file_icon.png ├── missing_file_icon.png ├── non_inherited_icon.png ├── npbackup_icon.ico ├── symlink_icon.png ├── tree_icon.png └── update_custom_resources.py ├── setup.py ├── tests ├── npbackup-cli-test-linux.yaml ├── npbackup-cli-test-windows.yaml ├── test_npbackup-cli.py └── test_restic_metrics.py └── upgrade_server ├── requirements.txt ├── upgrade_server.py └── upgrade_server ├── __debug__.py ├── __init__.py ├── api.py ├── configuration.py ├── crud.py └── models ├── files.py └── oper.py /.codespellrc: -------------------------------------------------------------------------------- 1 | [codespell] 2 | # Ref: https://github.com/codespell-project/codespell#using-a-config-file 3 | skip = .git*,.codespellrc,*.fr.yml,requirements.txt,customization.py,RESTIC_CMDLINE_CONSISTENCY.md 4 | check-hidden = true 5 | # ignore-regex = 6 | ignore-words-list = requestor -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.github/workflows/codespell.yaml: -------------------------------------------------------------------------------- 1 | # Codespell configuration is within .codespellrc 2 | --- 3 | name: Codespell 4 | 5 | on: 6 | push: 7 | branches: [main] 8 | pull_request: 9 | branches: [main] 10 | 11 | permissions: 12 | contents: read 13 | 14 | jobs: 15 | codespell: 16 | name: Check for spelling errors 17 | runs-on: ubuntu-latest 18 | 19 | steps: 20 | - name: Checkout 21 | uses: actions/checkout@v4 22 | - name: Annotate locations with typos 23 | uses: codespell-project/codespell-problem-matcher@v1 24 | - name: Codespell 25 | uses: codespell-project/actions-codespell@v2 -------------------------------------------------------------------------------- /.github/workflows/linux.yaml: -------------------------------------------------------------------------------- 1 | name: linux-tests 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | os: [ubuntu-latest] 12 | # Python 3.3 and 3.4 have been removed since github won't provide these anymore 13 | # As of 2023/01/09, we have removed python 3.5 and 3.6 as they don't work anymore with linux on github actions 14 | # As of 2023/08/30, we have removed python 2.7 since github actions won't provide it anymore 15 | # As of 2024/09/15, we have (temporarily) removed 'pypy-3.10' and 'pypy-3.8' since msgspec won't compile properly 16 | # As of 2024/12/24, we have remove python 3.7 as they don't work anymore with linux on github actions 17 | python-version: [3.8, 3.9, "3.10", "3.11", "3.12", "3.13"] 18 | 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v3 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Install dependencies 26 | run: | 27 | python -m pip install --upgrade pip 28 | python -m pip install --upgrade setuptools 29 | if [ -f npbackup/requirements.txt ]; then pip install -r npbackup/requirements.txt; fi 30 | - name: Generate Report 31 | env: 32 | RUNNING_ON_GITHUB_ACTIONS: true 33 | run: | 34 | pip install pytest coverage 35 | python -m coverage run -m pytest -vvs tests 36 | - name: Upload Coverage to Codecov 37 | uses: codecov/codecov-action@v3 38 | -------------------------------------------------------------------------------- /.github/workflows/pylint-linux.yaml: -------------------------------------------------------------------------------- 1 | name: pylint-linux-tests 2 | 3 | # Quick and dirty pylint 4 | 5 | # pylint --disable=C,W1201,W1202,W1203,W0718,W0621,W0603,R0801,R0912,R0913,R0915,R0911,R0914,R0911,R1702,R0902,R0903,R0904 npbackup 6 | 7 | on: [push, pull_request] 8 | 9 | jobs: 10 | build: 11 | 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | matrix: 15 | os: [ubuntu-latest] 16 | # python-version: [3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, "3.10", 'pypy-3.6', 'pypy-3.7'] 17 | python-version: ["3.13"] 18 | 19 | steps: 20 | - uses: actions/checkout@v2 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v2 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | - name: Install dependencies 26 | run: | 27 | python -m pip install --upgrade pip 28 | python -m pip install --upgrade setuptools 29 | if [ -f npbackup/requirements.txt ]; then python -m pip install -r npbackup/requirements.txt; fi 30 | if [ -f upgrade_server/requirements.txt ]; then python -m pip install -r upgrade_server/requirements.txt; fi 31 | - name: Lint with Pylint 32 | #if: ${{ matrix.python-version == '3.11' }} 33 | run: | 34 | python -m pip install pylint 35 | # Do not run pylint on python 3.3 because isort is not available for python 3.3, don't run on python 3.4 because pylint: disable=xxxx does not exist 36 | # Disable E0401 import error since we lint on linux and pywin32 is obviously missing 37 | python -m pylint --disable=C,W,R --max-line-length=127 npbackup 38 | python -m pylint --disable=C,W,R --max-line-length=127 upgrade_server/upgrade_server 39 | - name: Lint with flake8 40 | #if: ${{ matrix.python-version == '3.11' }} 41 | run: | 42 | python -m pip install flake8 43 | # stop the build if there are Python syntax errors or undefined names 44 | python -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics npbackup 45 | python -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics upgrade_server/upgrade_server 46 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 47 | python -m flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics npbackup 48 | python -m flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics upgrade_server 49 | - name: Lint with Black 50 | # Don't run on python < 3.6 since black does not exist there, run only once 51 | #if: ${{ matrix.python-version == '3.11' }} 52 | run: | 53 | pip install black 54 | python -m black --check npbackup 55 | python -m black --check upgrade_server/upgrade_server -------------------------------------------------------------------------------- /.github/workflows/pylint-windows.yaml: -------------------------------------------------------------------------------- 1 | name: pylint-windows-tests 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | matrix: 11 | os: [windows-latest] 12 | # Don't use pypy on windows since it does not have pywin32 module 13 | # python-version: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10"] 14 | python-version: ["3.13"] 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Set up Python ${{ matrix.python-version }} 19 | uses: actions/setup-python@v2 20 | with: 21 | python-version: ${{ matrix.python-version }} 22 | - name: Install dependencies 23 | run: | 24 | python -m pip install --upgrade pip 25 | python -m pip install --upgrade setuptools 26 | if (Test-Path "npbackup/requirements.txt") { pip install -r npbackup/requirements.txt } 27 | if (Test-Path "upgrade_server/requirements.txt") { pip install -r upgrade_server/requirements.txt } 28 | - name: Lint with Pylint 29 | #if: ${{ matrix.python-version == '3.12' }} 30 | run: | 31 | python -m pip install pylint 32 | # Do not run pylint on python 3.3 because isort is not available for python 3.3, don't run on python 3.4 because pylint: disable=xxxx does not exist 33 | python -m pylint --disable=C,W,R --max-line-length=127 npbackup 34 | python -m pylint --disable=C,W,R --max-line-length=127 upgrade_server/upgrade_server 35 | - name: Lint with flake8 36 | #if: ${{ matrix.python-version == '3.12' }} 37 | run: | 38 | python -m pip install flake8 39 | # stop the build if there are Python syntax errors or undefined names 40 | python -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics npbackup 41 | python -m flake8 --count --select=E9,F63,F7,F82 --show-source --statistics upgrade_server/upgrade_server 42 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 43 | python -m flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics npbackup 44 | python -m flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics upgrade_server/upgrade_server 45 | - name: Lint with Black 46 | # Don't run on python < 3.6 since black does not exist there, run only once 47 | #if: ${{ matrix.python-version == '3.12' }} 48 | run: | 49 | pip install black 50 | python -m black --check npbackup 51 | python -m black --check upgrade_server/upgrade_server -------------------------------------------------------------------------------- /.github/workflows/windows.yaml: -------------------------------------------------------------------------------- 1 | name: windows-tests 2 | 3 | # The default shell here is Powershell 4 | # Don't run with python 3.3 as using python -m to run flake8 or pytest will fail. 5 | # Hence, without python -m, pytest will not have it's PYTHONPATH set to current dir and imports will fail 6 | # Don't run with python 3.4 as github cannot install it (pip install --upgrade pip fails) 7 | 8 | on: [push, pull_request] 9 | 10 | jobs: 11 | build: 12 | 13 | runs-on: ${{ matrix.os }} 14 | strategy: 15 | matrix: 16 | os: [windows-latest] 17 | # As of 2023/08/30, we have removed python 2.7 since github actions won't provide it anymore 18 | # Don't test on pypy since we don't have pywin32 19 | # Drop python 3.7 support since 3.8 requires Win7+ and even restic needs a legacy build for Win7 20 | python-version: [3.8, 3.9, "3.10", "3.11", "3.12", "3.13"] 21 | 22 | steps: 23 | - uses: actions/checkout@v3 24 | - name: Set up Python ${{ matrix.python-version }} 25 | uses: actions/setup-python@v3 26 | with: 27 | python-version: ${{ matrix.python-version }} 28 | - name: Install dependencies 29 | run: | 30 | python -m pip install --upgrade pip 31 | python -m pip install --upgrade setuptools 32 | if (Test-Path "npbackup/requirements.txt") { pip install -r npbackup/requirements.txt } 33 | - name: Generate Report 34 | env: 35 | RUNNING_ON_GITHUB_ACTIONS: true 36 | run: | 37 | pip install pytest coverage 38 | python -m coverage run -m pytest -vvs tests 39 | - name: Upload Coverage to Codecov 40 | uses: codecov/codecov-action@v3 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 105 | __pypackages__/ 106 | 107 | # Celery stuff 108 | celerybeat-schedule 109 | celerybeat.pid 110 | 111 | # SageMath parsed files 112 | *.sage.py 113 | 114 | # Environments 115 | .env 116 | .venv 117 | env/ 118 | venv/ 119 | ENV/ 120 | env.bak/ 121 | venv.bak/ 122 | 123 | # Spyder project settings 124 | .spyderproject 125 | .spyproject 126 | 127 | # Rope project settings 128 | .ropeproject 129 | 130 | # mkdocs documentation 131 | /site 132 | 133 | # mypy 134 | .mypy_cache/ 135 | .dmypy.json 136 | dmypy.json 137 | 138 | # Pyre type checker 139 | .pyre/ 140 | 141 | # pytype static type analyzer 142 | .pytype/ 143 | 144 | # Cython debug symbols 145 | cython_debug/ 146 | 147 | # PyCharm 148 | # JetBrains specific template is maintainted in a separate JetBrains.gitignore that can 149 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 150 | # and can be added to the global gitignore or merged into this file. For a more nuclear 151 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 152 | #.idea/ 153 | 154 | RESTIC_SOURCE_FILES/restic* 155 | !RESTIC_SOURCE_FILES/restic*legacy*.exe 156 | RESTIC_SOURCE_FILES/ARCHIVES 157 | PRIVATE/_ev_data.py 158 | PRIVATE/_obfuscation.py 159 | _private* 160 | _NOUSE_private* 161 | NOUSE_private* 162 | BUILDS 163 | BUILDS-PRIVATE 164 | *.conf 165 | *.log? 166 | -------------------------------------------------------------------------------- /ANTIVIRUS.md: -------------------------------------------------------------------------------- 1 | ## Antivirus reports from various Nuitka builds for Windows 2 | 3 | ### 2024/03/10 4 | #### Viewer compilation 5 | Build type: Onefile 6 | Compiler: Nuitka 2.1 Commercial 7 | Backend: gcc 13.2.0 8 | Signed: No 9 | Build target: npbackup-viewer-x64.exe 10 | Result: 9/73 security vendors and no sandboxes flagged this file as malicious 11 | Link: https://www.virustotal.com/gui/file/efb327149ae84878fee9a2ffa5c8c24dbdad2ba1ebb6d383b6a2b23c4d5b723f 12 | 13 | Build type: Standalone 14 | Compiler: Nuitka 2.1 Commercial 15 | Backend: gcc 13.2.0 16 | Signed: No 17 | Build target: npbackup-viewer-x64.exe 18 | Result: 3/73 security vendors and no sandboxes flagged this file as malicious 19 | Link: https://www.virustotal.com/gui/file/019ca063a250f79f38582b980f983c1e1c5ad67f36cfe4376751e0473f1fbb29 20 | 21 | Build type: Onefile 22 | Compiler: Nuitka 2.1 Commercial 23 | Backend: gcc 13.2.0 24 | Signed: Yes (EV Code signing certificate) 25 | Build target: npbackup-viewer-x64.exe 26 | Result: 4/73 security vendors and no sandboxes flagged this file as malicious 27 | Link: https://www.virustotal.com/gui/file/48211bf5d53fd8c010e60d46aacd04cf4dc889180f9abc6159fde9c6fad65f61 28 | 29 | Build type: Standalone 30 | Compiler: Nuitka 2.1 Commercial 31 | Backend: gcc 13.2.0 32 | Signed: Yes (EV Code signing certificate) 33 | Build target: npbackup-viewer-x64.exe 34 | Result: 2/73 security vendor and no sandboxes flagged this file as malicious 35 | Link: https://www.virustotal.com/gui/file/31fbd01a763b25111c879b61c79b5045c1a95d32f02bf2c26aa9a45a9a8583ea 36 | 37 | 38 | ### 2024/04/27 39 | #### CLI compilation for 3.0.0-beta2 40 | Build type: Onefile 41 | Compiler: Nuitka 2.1.6 Commercial 42 | Backend: gcc 13.2.0 43 | Signed: No 44 | Build target: npbackup-cli-x64.exe 45 | Result: 11/72 security vendors and no sandboxes flagged this file as malicious 46 | Link: https://www.virustotal.com/gui/file/51e1c4ffc38609276a34ac67e87d0dd3fb489448f270d4c7f195e1c3926885c5 47 | 48 | Build type: Standalone 49 | Compiler: Nuitka 2.1.6 Commercial 50 | Backend: gcc 13.2.0 51 | Signed: No 52 | Build target: npbackup-cli-x64.exe 53 | Result: 4/72 security vendors and no sandboxes flagged this file as malicious 54 | Link: https://www.virustotal.com/gui/file/1d05616e41534b46657f235878ab3f4fa63365a652b3eda5280624fb610c0578 55 | 56 | Build type: Standalone 57 | Compiler: Nuitka 2.1.6 Commercial 58 | Backend: gcc 13.2.0 59 | Signed: Yes (EV Code signing certificate) 60 | Build target: npbackup-cli-x64.exe 61 | Result: 2/72 security vendors and no sandboxes flagged this file as malicious 62 | Link: https://www.virustotal.com/gui/file/88e761959ea4a538b762d193d088358ce04a13d78c20e082a16622e709734f87 63 | 64 | ### 2024/06/05 65 | #### CLI compilation for 3.0.0-rc1 66 | Build type: Standalone 67 | Compiler: Nuitka 2.2.1 Commercial 68 | Backend: gcc 13.2.0 69 | Signed: No 70 | Build target: npbackup-cli.exe 71 | Result: 5/74 security vendors and no sandboxes flagged this file as malicious 72 | Link: https://www.virustotal.com/gui/file/864154e9ed5756225192467f4d6636cd1322f207cdfa4d4d798a75e1255bb326 73 | 74 | Build type: Standalone 75 | Compiler: Nuitka 2.2.1 Commercial 76 | Backend: gcc 13.2.0 77 | Signed: Yes (EV Code signing certificate) 78 | Build target: npbackup-cli.exe 79 | Result: 5/74 security vendors and no sandboxes flagged this file as malicious 80 | Link: https://www.virustotal.com/gui/file/2778043151df967a98c3ceeca46868a873453c45947383874b98e1d94312bb12 81 | 82 | ### 2024/07/28 83 | #### CLI compilation for 3.0.0-rc2 84 | Build type: Standalone 85 | Compiler: Nuitka 2.2.1 Commercial 86 | Backend: gcc 13.2.0 87 | Signed: Yes (EV Code signing certificate) 88 | Build target: npbackup-cli.exe 89 | Result: No security vendors flagged this file as malicious 90 | Link: https://www.virustotal.com/gui/file/5e67baf15018d7acbc9839e16e600924cc12f59f51327a7ea217e6204250cf88 91 | 92 | ## 2024/09/04 93 | #### CLI compilation for 3.0.0-rc4 94 | Build type: Standalone 95 | Compiler: Nuitka 2.4.8 Commercial 96 | Backend gcc 13.2.0 97 | Signed: Yes (EV Code signing certificate) 98 | Build target: npbackup-cli.exe 99 | Result: 2/75 security vendors flagged this file as malicious 100 | Link: https://www.virustotal.com/gui/file/e0895e3d752c589f405c4200447496d42664eb1900f7ed1d9b6a3d321893311f 101 | 102 | ## 2024/10/25 103 | #### CLI compilation for 3.0.0-rc7 104 | Build type: Standalone 105 | Compiler: Nuitka 2.4.10 Commercial 106 | Backend gcc 13.2.0 107 | Signed: Yes (EV Code signing certificate) 108 | Build target: npbackup-cli.exe 109 | Result: No security vendors flagged this file as malicious 110 | Link: https://www.virustotal.com/gui/file/f008db7e323c832f47a778b1526b326b915042c0de5b2916b3052fc2a311040e?nocache=1 111 | 112 | ## 2025/01/10 113 | #### CLI compilation for 3.0.0-rc13 114 | Build type: Standalone 115 | Compiler: Nuitka 2.5.9 Commercial 116 | Backend gcc 13.2.0 117 | Signed: Yes (EV Code signing certificate) 118 | Build target: npbackup-cli.exe 119 | Result: 2/72 security vendors flagged this file as malicious 120 | Link: https://www.virustotal.com/gui/file/48c7d828f878638ccf8736f1b87bccd0f8e544aa90d1659b318dbca7b5166b65 121 | 122 | ## 2025/02/16 123 | #### CLI Compilation for 3.0.0-rc14 124 | Build type: Standalone 125 | Compiler: Nuitka 2.6.4 commercial 126 | Backend msvc 143 127 | Signed: Yes (EV Code signing certificate) 128 | Build target: npbackup-cli.exe 129 | Result: No security vendors flagged this file as malicious 130 | Link: https://www.virustotal.com/gui/file/50a12c697684194853e6e4ec1778f50f86798d426853c4e0a744f2a5c4d02def -------------------------------------------------------------------------------- /COMPATIBILITY.md: -------------------------------------------------------------------------------- 1 | ## Compatibility for various platforms 2 | 3 | ### Linux 4 | 5 | We need Python 3.7 to compile on RHEL 7, which uses glibc 2.17 6 | These builds will be "legacy" builds, 64 bit builds are sufficient. 7 | 8 | ### Windows 9 | 10 | We need Python 3.7 to compile on Windows 7 / Server 2008 R2 11 | These builds will be "legacy" builds, and will run quite slower than their modern counterparts. 12 | 13 | Also, last restic version to run on Windows 7 is 0.16.2, see https://github.com/restic/restic/issues/4636 (basically go1.21 is not windows 7 compatible anymore) 14 | So we actually need to compile restic ourselves with go1.20.12 which is done via restic_legacy_build.cmd script 15 | 16 | -------------------------------------------------------------------------------- /PRIVATE/README.md: -------------------------------------------------------------------------------- 1 | This folder may contain overrides for NPBackup secrets. 2 | If these files exist, NPBackup will be compiled as "private build". 3 | 4 | Overrides are used by default if found at execution time. 5 | 6 | In order to use them at compile time, one needs to run `compile.py --audience private` 7 | 8 | 1. You can create a file called _private_secret_keys.py to override default secret_keys.py file from npbackup 9 | 2. You may obfuscate the AES key at runtime by creating a file called `_obfuscation.py` that must contain 10 | a function `obfuscation(bytes) -> bytes` like `aes_key_derivate = obfuscation(aes_key)` where aes_key_derivate must be 32 bytes. 11 | -------------------------------------------------------------------------------- /PRIVATE/__no_init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # Placeholder to remind this directory should not become a package 5 | # In order to avoid bundling it with the main package as wheel / bdist 6 | 7 | # THIS DIRECTORY MAY CONTAIN OVERRIDE FILES 8 | # 9 | # 10 | # `_private_secret_keys.py` here will override secret_keys.py 11 | # `_obfuscation.py` here will override obfuscation.py 12 | # 13 | # If these files exist, build type will become "private" instead of "public" 14 | -------------------------------------------------------------------------------- /RESTIC_CMDLINE_CONSISTENCY.md: -------------------------------------------------------------------------------- 1 | ## List of various restic problems encountered while developing NPBackup 2 | 3 | As of 2024/01/02, version 0.16.2: 4 | 5 | ### json inconsistencies 6 | 7 | - `restic check --json` does not produce json output, probably single str on error 8 | - `restic unlock --json` does not produce any output, probably single str on error 9 | - `restic repair index --json` does not produce json output 10 | ``` 11 | loading indexes... 12 | getting pack files to read... 13 | rebuilding index 14 | [0:00] 100.00% 28 / 28 packs processed 15 | deleting obsolete index files 16 | done 17 | ``` 18 | - `restic repair snapshots --json` does not produce json output 19 | ``` 20 | snapshot 00ecc4e3 of [c:\git\npbackup] at 2024-01-02 19:15:35.3779691 +0100 CET) 21 | 22 | snapshot 1066f045 of [c:\git\npbackup] at 2023-12-28 13:46:41.3639521 +0100 CET) 23 | 24 | no snapshots were modified 25 | ``` 26 | - `restic forget --json` does not produce any output, and produces str output on error. Example on error: 27 | ``` 28 | Ignoring "ff20970b": no matching ID found for prefix "ff20970b" 29 | ``` 30 | - `restic list index|blobs|snapshots --json` produce one result per line output, not json, example for blobs: 31 | ``` 32 | tree 0d2eef6a1b06aa0650a08a82058d57a42bf515a4c84bf4f899e391a4b9906197 33 | tree 9e61b5966a936e2e8b4ef4198b86ad59000c5cba3fc6250ece97cb13621b3cd1 34 | tree 1fe90879bd35d90cd4fde440e64bfc16b331297cbddb776a43eb3fdf94875540 35 | ``` 36 | 37 | - `restic key list --json` produces direct parseable json 38 | - `restic stats --json` produces direct parseable json 39 | - `restic find --json` produces direct parseable json 40 | - `restic snapshots --json` produces direct parseable json 41 | - `restic backup --json` produces multiple state lines, each one being valid json, which makes sense 42 | - `restic restore --target --json` produces multiple state lines, each one being valid json, which makes sense 43 | 44 | ### backup results inconsistency 45 | 46 | When using `restic backup`, we get different results depending on if we're using `--json`or not: 47 | 48 | - "data_blobs": Not present in string output 49 | - "tree_blobs": Not present in string output 50 | - "data_added": Present in both outputs, is "4.425" in `Added to the repository: 4.425 MiB (1.431 MiB stored)` 51 | - "data_stored": Not present in json output, is "1.431" in `Added to the repository: 4.425 MiB (1.431 MiB stored)` 52 | 53 | `restic backup` results 54 | ``` 55 | repository 962d5924 opened (version 2, compression level auto) 56 | using parent snapshot 325a2fa1 57 | [0:00] 100.00% 4 / 4 index files loaded 58 | 59 | Files: 216 new, 21 changed, 5836 unmodified 60 | Dirs: 29 new, 47 changed, 817 unmodified 61 | Added to the repository: 4.425 MiB (1.431 MiB stored) 62 | 63 | processed 6073 files, 116.657 MiB in 0:03 64 | snapshot b28b0901 saved 65 | ``` 66 | 67 | `restic backup --json` results 68 | ``` 69 | {"message_type":"summary","files_new":5,"files_changed":15,"files_unmodified":6058,"dirs_new":0,"dirs_changed":27,"dirs_unmodified":866,"data_blobs":17,"tree_blobs":28,"data_added":281097,"total_files_processed":6078,"total_bytes_processed":122342158,"total_duration":1.2836983,"snapshot_id":"360333437921660a5228a9c1b65a2d97381f0bc135499c6e851acb0ab84b0b0a"} 70 | ``` 71 | 72 | ### init results inconsistency (v0.16.4) 73 | 74 | init command with `--json` parameter doesn't return JSON when it fails 75 | 76 | `restic init --json` results 77 | 78 | - on success 79 | ``` 80 | {"message_type":"initialized","id":"8daef59e2ac4c86535ae3f7414fcac6534f270077176af3ebddd34c364cac3c2","repository":"c:\\testy"} 81 | ``` 82 | - on already existing repo 83 | Fatal: create repository at c:\testy failed: config file already exists 84 | 85 | - on bogus path 86 | Fatal: create repository at x:\testy failed: mkdir \\?: La syntaxe du nom de fichier, de répertoire ou de volume est incorrecte. 87 | 88 | ### forget results insconsistenct (v0.17.2) 89 | # NPF-RESTIC-00001 90 | 91 | forget command returns exit code 0 when 92 | 93 | Example with `restic forget myunknwonsnapshot`: 94 | ``` 95 | repository 73d4b440 opened (version 2, compression level auto) 96 | Ignoring "myunknwonsnapshot": no matching ID found for prefix "myunknwonsnapshot" 97 | 98 | echo $? 99 | 0 100 | ``` -------------------------------------------------------------------------------- /RESTIC_SOURCE_FILES/README.txt: -------------------------------------------------------------------------------- 1 | This folder should contain restic binaries directly downloaded from github.com/restic/restic for Linux / macOS and Windows and restic legacy builds -------------------------------------------------------------------------------- /RESTIC_SOURCE_FILES/legacy_restic_builder.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | :: Blatantly copied from https://gist.github.com/DRON-666/6e29eb6a8635fae9ab822782f34d8fd6 4 | :: with some mods to specify restic versions and produce both 32 and 64 bit executables 5 | 6 | 8 7 | SET RESTIC_VERSION=0.18.0 8 | SET GO_BINARIES_VERSION=1.21.3 9 | SET GO23_VERSION=1.23.8 10 | SET GO24_VERSION=1.24.2 11 | 12 | SET LOG_FILE=%~n0.log 13 | SET BUILD_DIR=%~dp0BUILD 14 | IF NOT EXIST "%BUILD_DIR%" MKDIR "%BUILD_DIR%" || GOTO ERROR 15 | PUSHD BUILD 16 | 17 | set RESTIC_URL=https://github.com/restic/restic/releases/download/v%RESTIC_VERSION%/restic-%RESTIC_VERSION%.tar.gz 18 | set GO21_BIN_URL=https://go.dev/dl/go%GO_BINARIES_VERSION%.windows-amd64.zip 19 | set GO23_SRC_URL=https://go.dev/dl/go%GO23_VERSION%.src.tar.gz 20 | set GO24_SRC_URL=https://go.dev/dl/go%GO24_VERSION%.src.tar.gz 21 | set PATCH_URL=https://gist.github.com/DRON-666/6e29eb6a8635fae9ab822782f34d8fd6/raw/win7sup.diff 22 | set BUSYBOX_URL=https://web.archive.org/web/20250314144220id_/https://frippery.org/files/busybox/busybox64.exe 23 | set BUSYBOX="%BUILD_DIR%\busybox64.exe" 24 | set GOTOOLCHAIN=local 25 | 26 | call:Log "Running legacy restic builder" 27 | 28 | call:Log "Fetching busybox" 29 | if not exist %BUSYBOX% (powershell -Command (New-Object System.Net.WebClient^).DownloadFile('"%BUSYBOX_URL%"','%BUSYBOX%'^) || GOTO ERROR) 30 | call:Log "Fetching GO %GO_BINARIES_VERSION% binaries which are Window 7 compatible" 31 | call:process %GO21_BIN_URL% %BUILD_DIR%\go 32 | call:Log "Building Go %GO23_VERSION% with Windows 7 support patch" 33 | call:process %GO23_SRC_URL% %BUILD_DIR%\go23 %PATCH_URL% %BUILD_DIR%\go 34 | call:Log "Building Go %GO24_VERSION% with Windows 7 support patch" 35 | call:process %GO24_SRC_URL% %BUILD_DIR%\go24 %PATCH_URL% %BUILD_DIR%\go23 36 | call:Log "Downloading restic %RESTIC_VERSION% source code" 37 | call:process %RESTIC_URL% restic-%RESTIC_VERSION% 38 | 39 | call:build_restic 386 40 | call:build_restic amd64 41 | goto END 42 | 43 | :GetTime 44 | :: US Date /T returns Day MM/DD/YYYY whereas other languages may DD/MM/YYYY, Try to catch both 45 | FOR /F "tokens=1,2,3,4 delims=/" %%a IN ('Date /T') DO ( 46 | IF "%%d"=="" set now_date=%%a-%%b-%%c 47 | IF NOT "%%d"=="" set now_date=%%a-%%b-%%c-%%d 48 | ) 49 | set now_time=%TIME:~0,2%:%TIME:~3,2%:%TIME:~6,2% 50 | set start_date=%now_date%-%TIME:~0,2%_%TIME:~3,2%_%TIME:~6,2% 51 | GOTO:EOF 52 | 53 | :Log 54 | call:GetTime 55 | echo %now_date% - %now_time% %~1 >> "%LOG_FILE%" 56 | IF "%DEBUG%"=="yes" echo %~1 57 | GOTO:EOF 58 | 59 | :process 60 | :: Download and extract archives 61 | if not exist %~nx1 (%BUSYBOX% wget %~1 || GOTO ERROR) 62 | if not exist %~2 if /I "%~x1"==".zip" (%BUSYBOX% unzip -q %~nx1 || GOTO ERROR) else (md %~2 && %BUSYBOX% tar x -z --strip-components 1 -C %~2 -f %~nx1 || GOTO ERROR) 63 | if "%~3"=="" goto :EOF 64 | :: Patch go sources 65 | if not exist %~nx3 (%BUSYBOX% wget %~3 || GOTO ERROR) 66 | if not exist %~2\patched (pushd %~2 && %BUSYBOX% patch -p 0 -i "%BUILD_DIR%\%~nx3" && %BUSYBOX% sed -E -i "s/^go1.+[0-9]$/\0-win7sup/" VERSION && md patched && popd || GOTO ERROR) 67 | :: Build go from sources - Don't put a space between '%4' and '&&' or else it will be interpreted as fullpath 68 | if not exist %~2\bin\go.exe (pushd %~2\src && set GOROOT_BOOTSTRAP=%4&& call make.bat && popd || GOTO ERROR) 69 | GOTO:EOF 70 | 71 | :build_restic 72 | if NOT "%~1"=="" SET GOARCH=%~1 73 | call:Log "Building restic %RESTIC_VERSION% %GOARCH% with Windows 7 Support" 74 | :: Setting path without previous paths prevents further runs and calling binaries like powershell 75 | set PATH=%BUILD_DIR%\go24\bin;%PATH% 76 | if not exist restic_%RESTIC_VERSION%_windows_legacy_%GOARCH%.exe (PUSHD %BUILD_DIR%\restic-%RESTIC_VERSION% && go.exe run build.go && move /y restic.exe ../restic_%RESTIC_VERSION%_windows_legacy_%GOARCH%.exe && popd || GOTO ERROR) 77 | restic_%RESTIC_VERSION%_windows_legacy_%GOARCH%.exe version 78 | GOTO:EOF 79 | 80 | :ERROR 81 | call:Log "Build failure" 82 | GOTO EXIT 83 | 84 | :END 85 | call:Log "Build success" 86 | GOTO EXIT 87 | 88 | :EXIT 89 | POPD 90 | ::exit /b 1 -------------------------------------------------------------------------------- /RESTIC_SOURCE_FILES/restic_0.18.0_windows_legacy_386.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/RESTIC_SOURCE_FILES/restic_0.18.0_windows_legacy_386.exe -------------------------------------------------------------------------------- /RESTIC_SOURCE_FILES/restic_0.18.0_windows_legacy_amd64.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/RESTIC_SOURCE_FILES/restic_0.18.0_windows_legacy_amd64.exe -------------------------------------------------------------------------------- /RESTIC_SOURCE_FILES/update_restic.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | __intname__ = "npbackup.restic_update" 6 | __author__ = "Orsiris de Jong" 7 | __copyright__ = "Copyright (C) 2024-2025 NetInvent" 8 | __license__ = "BSD-3-Clause" 9 | __build__ = "2025040801" 10 | 11 | import os 12 | import sys 13 | import bz2 14 | from pathlib import Path 15 | import requests 16 | import json 17 | import shutil 18 | from pprint import pprint 19 | 20 | 21 | sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))) 22 | 23 | from npbackup.path_helper import BASEDIR 24 | 25 | 26 | def download_restic_binaries(arch: str = "amd64") -> bool: 27 | """ 28 | We must first download latest restic binaries to make sure we can run all tests and/or compile 29 | """ 30 | org = "restic" 31 | repo = "restic" 32 | 33 | response = requests.get( 34 | f"https://api.github.com/repos/{org}/{repo}/releases/latest" 35 | ) 36 | # print("RESPONSE: ", response) 37 | if response.status_code != 200: 38 | print(f"ERROR: Cannot get latest restic release: {response.status_code}") 39 | print("RESPONSE TEXT: ", response.text) 40 | return False 41 | json_response = json.loads(response.text) 42 | current_version = json_response["tag_name"].lstrip("v") 43 | 44 | # print("JSON RESPONSE") 45 | # pprint(json_response, indent=5) 46 | 47 | dest_dir = Path(BASEDIR).absolute().parent.joinpath("RESTIC_SOURCE_FILES") 48 | if os.name == "nt": 49 | fname = f"_windows_{arch}" 50 | suffix = ".exe" 51 | arch_suffix = ".zip" 52 | else: 53 | fname = f"_linux_{arch}" 54 | suffix = "" 55 | arch_suffix = ".bz2" 56 | 57 | if not dest_dir.joinpath("ARCHIVES").is_dir(): 58 | os.makedirs(dest_dir.joinpath("ARCHIVES")) 59 | 60 | dest_file = dest_dir.joinpath("restic_" + current_version + fname + suffix) 61 | 62 | if dest_file.is_file(): 63 | print(f"RESTIC SOURCE ALREADY PRESENT. NOT DOWNLOADING {dest_file}") 64 | return True 65 | else: 66 | print(f"DOWNLOADING RESTIC {dest_file}") 67 | # Also we need to move any earlier file that may not be current version to archives 68 | for file in dest_dir.glob(f"restic_*{fname}{suffix}"): 69 | # We need to keep legacy binary for Windows 7 / Server 2008 70 | if "legacy" in file.name: 71 | try: 72 | archive_file = dest_dir.joinpath("ARCHIVES").joinpath(file.name) 73 | if archive_file.is_file(): 74 | archive_file.unlink() 75 | shutil.move( 76 | file, 77 | archive_file, 78 | ) 79 | except OSError as exc: 80 | print( 81 | f"CANNOT MOVE OLD FILES ARCHIVE: {file} to {archive_file}: {exc}" 82 | ) 83 | return False 84 | 85 | downloaded = False 86 | for entry in json_response["assets"]: 87 | if f"{fname}{arch_suffix}" in entry["browser_download_url"]: 88 | file_request = requests.get( 89 | entry["browser_download_url"], allow_redirects=True 90 | ) 91 | print("FILE REQUEST RESPONSE", file_request) 92 | filename = entry["browser_download_url"].rsplit("/", 1)[1] 93 | full_path = dest_dir.joinpath(filename) 94 | print("PATH TO DOWNLOADED ARCHIVE: ", full_path) 95 | if arch_suffix == ".bz2": 96 | final_executable = str(full_path).rstrip(arch_suffix) 97 | with open(final_executable, "wb") as fp: 98 | fp.write(bz2.decompress(file_request.content)) 99 | # We also need to make that file executable 100 | os.chmod(str(full_path).rstrip(arch_suffix), 0o775) 101 | else: 102 | with open(full_path, "wb") as fp: 103 | fp.write(file_request.content) 104 | # Assume we have a zip or tar.gz 105 | shutil.unpack_archive(full_path, dest_dir) 106 | final_executable = dest_dir.joinpath(filename) 107 | try: 108 | # We don't drop the bz2 files on disk, so no need to move them to ARCHIVES 109 | if arch_suffix != ".bz2": 110 | if dest_dir.joinpath("ARCHIVES").joinpath(filename).is_file(): 111 | dest_dir.joinpath("ARCHIVES").joinpath(filename).unlink() 112 | shutil.move( 113 | full_path, dest_dir.joinpath("ARCHIVES").joinpath(filename) 114 | ) 115 | except OSError as exc: 116 | print( 117 | f'CANNOT MOVE TO ARCHIVE: {full_path} to {dest_dir.joinpath("ARCHIVES").joinpath(filename)}: {[exc]}' 118 | ) 119 | return False 120 | print(f"DOWNLOADED {final_executable}") 121 | downloaded = True 122 | break 123 | if not downloaded: 124 | print(f"NO RESTIC BINARY FOUND for {arch}") 125 | return False 126 | return True 127 | 128 | 129 | def download_restic_binaries_for_arch(): 130 | """ 131 | Shortcut to be used in compile script 132 | """ 133 | if os.name == "nt": 134 | if not download_restic_binaries("amd64") or not download_restic_binaries("386"): 135 | sys.exit(1) 136 | else: 137 | if ( 138 | not download_restic_binaries("amd64") 139 | or not download_restic_binaries("arm64") 140 | or not download_restic_binaries("arm") 141 | ): 142 | sys.exit(1) 143 | return True 144 | 145 | 146 | if __name__ == "__main__": 147 | download_restic_binaries_for_arch() 148 | -------------------------------------------------------------------------------- /RESTIC_SOURCE_FILES/update_restic.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Quick script to update restic binaries 4 | 5 | export ORG=restic 6 | export REPO=restic 7 | LATEST_VERSION=$(curl -s https://api.github.com/repos/${ORG}/${REPO}/releases/latest | grep "tag_name" | cut -d'"' -f4) 8 | echo Latest restic version ${LATEST_VERSION} 9 | 10 | errors=false 11 | 12 | if [[ $(uname -s) == *"CYGWIN"* ]]; then 13 | platforms=(windows_amd64 windows_386) 14 | else 15 | platforms=(linux_arm linux_arm64 linux_amd64 darwin_amd64 freebsd_amd64) 16 | fi 17 | 18 | for platform in "${platforms[@]}"; do 19 | echo "Checking for ${platform}" 20 | restic_filename="restic_${LATEST_VERSION//v}_${platform}" 21 | if [ ! -f "${restic_filename}" ]; then 22 | echo "Moving earlier version to archive" 23 | [ -d ARCHIVES ] || mkdir ARCHIVES 24 | mv -f restic_*_${platform} ARCHIVES/ > /dev/null 2>&1 25 | # Move all restic executables except restic legacy binary 26 | find ./ -mindepth 1 -maxdepth 1 -type f -name "restic_*${platform}.exe" -and ! -name "restic_0.16.2_windows_386.exe" -exec mv '{}' ARCHIVES \; 27 | # Avoid moving restic 28 | echo "Downloading ${restic_filename}" 29 | if [ "${platform:0:7}" == "windows" ]; then 30 | ext=zip 31 | else 32 | ext=bz2 33 | fi 34 | curl -OL https://github.com/restic/restic/releases/download/${LATEST_VERSION}/restic_${LATEST_VERSION//v}_{$platform}.${ext} 35 | if [ $? -ne 0 ]; then 36 | echo "Failed to download ${restic_filename}" 37 | errors=true 38 | else 39 | if [ -f "${restic_filename}.bz2" ]; then 40 | bzip2 -d "${restic_filename}.bz2" && chmod +x "${restic_filename}" 41 | elif [ -f "${restic_filename}.zip" ]; then 42 | unzip "${restic_filename}.zip" 43 | else 44 | echo "Archive ${restic_filename} not found" 45 | errors=true 46 | fi 47 | if [ $? -ne 0 ]; then 48 | echo "Failed to decompress ${restic_filename}.bz2" 49 | errors=true 50 | fi 51 | [ -f "${restic_filename}.zip" ] && rm -f "${restic_filename}.zip" 52 | fi 53 | fi 54 | done 55 | 56 | if [ "${errors}" = true ]; then 57 | echo "Errors occurred during update" 58 | exit 1 59 | else 60 | echo "Finished updating restic binaries" 61 | fi 62 | -------------------------------------------------------------------------------- /ROADMAP.md: -------------------------------------------------------------------------------- 1 | ## What's planned / considered post v3 2 | 3 | ### Daemon mode (planned) 4 | Instead of relying on scheduled tasks, we could launch backup & housekeeping operations as daemon. 5 | Caveats: 6 | - We need a windows service (nuitka commercial implements one) 7 | - We need to use apscheduler (wait for v4) 8 | - We need a resurrect service config for systemd and windows service 9 | - Upgrade checks will be done via service 10 | 11 | ### Fallback (considered) 12 | - Repository uri should allow to have a fallback server 13 | - Prometheus support should have a push gateway fallback server option. 14 | - Upgrade server should have a fallback server 15 | 16 | ### Web interface (planned) 17 | Since runner can discuss in JSON mode, we could simply wrap it all in FastAPI 18 | Caveats: 19 | - We'll need a web interface, with templates, whistles and bells 20 | - We'll probably need an executor (Celery ?) in order to not block threads 21 | 22 | ### KVM Backup plugin (planned, already exists as external script) 23 | Since we run cube backup, we could "bake in" full KVM support 24 | Caveats: 25 | - We'll need to re-implement libvirt controller class for linux 26 | 27 | ### Proxmox Backup plugin 28 | Assumption: Since we can backup KVM, we can also backup Proxmox ? 29 | Caveats: 30 | - vzdump produces a specific archive format (using lzop) which would need to be decompressed to allow good deduplication (https://git.proxmox.com/?p=pve-qemu.git;a=blob;f=vma_spec.txt; ) 31 | - vzdump vanilla files can be deduped with block dedup tech, but badly (test done using two vzdump backups, one after another to zfs 2.3.0 with fast dedup and a 160GB ddt table limit): 32 | - 1M recordsize = 0% 33 | - 128K recordsize = 1% 34 | - 64K recordsize = 19% 35 | - 32K recordsize = 65% 36 | - 16k recordsize = 79% 37 | - So basically block dedup is bad for vzdump files. restic uses 4M pack sizes minimum. Tests have shown restic dedup algorithm to dedup 0% of data properly in the same file (rsync tests have shown about 40% file differences using patch method) 38 | - We need to "open" the vzdump files in order to store them properly (with vma utility), but present them to proxmox as vzdump files. We could check if vzdump can produce proper tar files with --stdout, or we could also just not use vzdump but rather do it the good old way of backing up qcow2 + xml files via qm 39 | 40 | ### SQL Backups 41 | That's a pre-script job ;) 42 | Perhaps, provide pre-scripts for major SQL engines 43 | Perhaps, provide an alternative dump | npbackup-cli syntax. 44 | In the latter case, shell (bash, zsh, ksh) would need `shopt -o pipefail`, and minimum backup size set. 45 | The pipefail will not be given to npbackup-cli, so we'd need to wrap everything into a script, which defeats the prometheus metrics. 46 | 47 | ### Key management 48 | Possibility to add new keys to current repo, and delete old keys if more than one key present 49 | 50 | ### Provision server (planned) 51 | Possibility to auto load repo settings for new instances from central server 52 | We actually could improve upgrade_server to do so 53 | 54 | ### Hyper-V Backup plugin 55 | That's another story. Creating snapshots and dumping VM is easy 56 | Shall we go that route since a lot of good commercial products exist ? Probably not 57 | 58 | ### Full disk cloning 59 | Out of scope of NPBackup. There are plenty of good tools out there, designed for that job 60 | 61 | ### Rust rewrite 62 | That would be my "dream" project in order to learn a new language in an existing usecase. 63 | But this would need massive sponsoring as I couldn't get the non-paid time to do so. 64 | 65 | ### More backends support 66 | Rustic is a current alternative backend candidate I tested. Might happen if enough traction. 67 | 68 | ### Branding manager 69 | We might want to put all files into `resources` directory and have `customization.py` files generated from there. 70 | 71 | ### New installer 72 | We might need to code an installer script for Linux, and perhaps a NSIS installer for Windows. 73 | 74 | ### Security 75 | Check that the config file is not world writable, so nobody can inject pre/post commands -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # NPF-SEC-00001: SECURITY-ADMIN-BACKUP-PASSWORD ONLY AVAILABLE ON PRIVATE COMPILED BUILDS 2 | 3 | Note: This security entry has been retired since v2.3.0, and totally reimplemented in v3.0 4 | In gui.config we have a function that allows to show unencrypted values of the yaml config file 5 | While this is practical, it should never be allowed on non compiled builds or with the default backup admin password 6 | 7 | # NPF-SEC-00002: pre & post execution as well as password commands can be a security risk 8 | 9 | All these commands are run with npbackup held privileges. 10 | In order to avoid a potential attack, the config file has to be world readable only. 11 | 12 | # NPF-SEC-00003: Avoid password command divulgation 13 | 14 | Password command is encrypted in order to avoid it's divulgation if config file is world readable. 15 | Password command is also not logged. 16 | 17 | # NPF-SEC-00004: Client should never know the repo password 18 | 19 | Partially covered with password_command feature, and alternative aes key management. 20 | We should have a central password server that holds repo passwords, so password is never actually stored in config. 21 | This will prevent local backups, so we need to think of a better zero knowledge strategy here. 22 | 23 | # NPF-SEC-00005: Viewer mode can bypass permissions 24 | 25 | Since viewer mode requires actual knowledge of repo URI and repo password, there's no need to manage local permissions. 26 | Viewer mode permissions are set to "restore-only". 27 | 28 | # NPF-SEC-00006: Never inject permissions if some are already present 29 | 30 | Since v3.0.0, we insert permissions directly into the encrypted repo URI. 31 | Hence, update permissions should only happen in two cases: 32 | - CLI: Recreate repo_uri entry and add permission field from YAML file 33 | - GUI: Enter permission password to update permissions 34 | 35 | # NPF-SEC-00007: Encrypted data needs to be protected 36 | 37 | Since encryption is symmetric, we need to protect our sensitive data. 38 | Best ways: 39 | - Compile with alternative aes-key 40 | - Use `NPBACKUP_KEY_LOCATION` or `NPBACKUP_KEY_COMMAND` to specify alternative AES keys 41 | 42 | # NPF-SEC-00008: Don't show manager password / sensitive data with --show-config 43 | 44 | Using `--show-config` should hide sensitive data, and manager password. 45 | 46 | # NPF-SEC-00009: Option to show sensitive data 47 | 48 | When using `--show-config` or right click `show unencrypted`, we should only show unencrypted config if password is set. 49 | Environment variable `NPBACKUP_MANAGER_PASSWORD` will be read to verify access, or GUI may ask for password. 50 | Also, when wrong password is entered, we should wait in order to reduce brute force attacks. 51 | 52 | # NPF-SEC-00010: Date attacks 53 | 54 | When using retention policies, we need to make sure that current system date is good, in order to avoid wrong retention deletions. 55 | When set, an external NTP server is used to get the offset. If offset is high enough (10 min), we avoid executing the retention policies. 56 | 57 | # NPF-SEC-00011: Default AES key obfuscation 58 | 59 | Using obfuscation() symmetric function in order to not store the bare AES key. 60 | 61 | # NPF-SEC-00012: Don't add PRIVATE directory to wheel / bdist builds 62 | 63 | The PRIVATE directory might contain alternative AES keys and obfuscation functions which should never be bundled for a PyPI release. 64 | 65 | # NPF-SEC-00013: Don't leave encrypted environment variables for script usage 66 | 67 | Sensitive environment variables aren't available for scripts / additional parameters and will be replaced by a given string from __env__.py -------------------------------------------------------------------------------- /UPGRADE_SERVER.md: -------------------------------------------------------------------------------- 1 | ## Setup upgrade server 2 | 3 | Clone git repository into a directory and create a venv environment 4 | 5 | ``` 6 | cd /opt 7 | git clone https://github.com/netinvnet/npbackup 8 | cd npbackup 9 | python -m venv venv 10 | ``` 11 | 12 | Install requirements 13 | ``` 14 | venv/bin/python -m pip install -r upgrade_server/requirements.txt 15 | ``` 16 | 17 | ## Configuration file 18 | 19 | Create a configuration file ie `/etc/npbackup_upgrade_server.conf` with the following content 20 | 21 | ``` 22 | # NPBackup v3 upgrade server 23 | http_server: 24 | # These are the passwords that need to be set in the upgrade settings of NPBackup client 25 | username: npbackup_upgrader 26 | password: SomeSuperSecurePassword 27 | listen: 0.0.0.0 28 | port: 8080 29 | upgrades: 30 | data_root: /opt/upgrade_server_root 31 | statistics_file: /opt/upgrade_server_root/stats.csv 32 | ``` 33 | 34 | You should also create the upgrade server root path 35 | ``` 36 | mkdir /opt/upgrade_server_root 37 | ``` 38 | 39 | ## Provisioning files 40 | 41 | Basically, upgrade_server serves compressed versions of NPBackup, than can directly be downloaded from git releases. Those compressed files will contain a single directory, like `npbackup-gui` or `npbackup-cli` which contain all the files (.zip and .tar.gz files are supported) 42 | 43 | 44 | When uploading new versions, you need to create a file in the data root called `VERSION` which should contain current NPBackup version, example `3.0.1` 45 | This way, every NPBackup client will download this file and compare with it's current version in order to verify if an upgrade is needed. 46 | 47 | If an upgrade is needed, NPBackup will try to download it from `/download/{platform}/{arch}/{build_type}` 48 | 49 | Current platforms are: `windows`, `linux` 50 | Current arches are `x64`, `x64-legacy`, `x86-legacy`, `arm-legacy` and `arm64-legacy`. 51 | 52 | Basically, if you want to update the current NPBackup client, you should have copy your new npbackup archives to the according locations 53 | ``` 54 | /opt/upgrade_server_root/linux/x64/npbackup-cli.tar.gz 55 | /opt/upgrade_server_root/windows/x64/npbackup-gui.zip 56 | ``` 57 | 58 | 59 | ## Run server 60 | 61 | You can run server with 62 | ``` 63 | venv/bin/python upgrade_server/upgrade_server.py -c /etc/npbackup_upgrade_server.conf 64 | ``` 65 | 66 | ## Create a service 67 | 68 | You can create a systemd service for the upgrade server as `/etc/systemd/system/npbackup_upgrade_server.service`, see the systemd file in the example directory. 69 | 70 | ## Statistics 71 | 72 | You can find the CSV file containing statistics, which contain: 73 | 74 | - Operation: check_version|get_file_info|download_upgrade 75 | - IP Address 76 | - Machine ID as defined in client 77 | - NPBackup version 78 | - Machine Group as defined in client 79 | - Platform 80 | - Arch 81 | -------------------------------------------------------------------------------- /bin/COMPILE.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | :: This is an example compiler script 4 | 5 | SET PYTHON64=c:\python313-64\python.exe 6 | SET PYTHON64-LEGACY=c:\python38-64\python.exe 7 | SET PYTHON32=c:\python313-32\python.exe 8 | SET PYTHON32-LEGACY=c:\python38-32\python.exe 9 | 10 | 11 | cd C:\GIT\npbackup 12 | git pull || GOTO ERROR 13 | 14 | :: Make sure we add npbackup in python path so bin and npbackup subfolders become packages 15 | SET OLD_PYTHONPATH=%PYTHONPATH% 16 | SET PYTHONPATH=c:\GIT\npbackup 17 | 18 | "%PYTHON64%" RESTIC_SOURCE_FILES/update_restic.py || GOTO ERROR 19 | 20 | :: BUILD 64-BIT VERSION 21 | "%PYTHON64%" -m pip install --upgrade pip || GOTO ERROR 22 | "%PYTHON64%" -m pip install pytest 23 | "%PYTHON64%" -m pip install --upgrade -r npbackup/requirements.txt || GOTO ERROR 24 | 25 | "%PYTHON64%" -m pytest C:\GIT\npbackup\tests || GOTO ERROR 26 | 27 | "%PYTHON64%" bin\compile.py --sign "C:\ODJ\KEYS\NetInventEV.dat" %* 28 | 29 | :: BUILD 64-BIT LEGACY VERSION 30 | "%PYTHON64-LEGACY%" -m pip install --upgrade pip || GOTO ERROR 31 | "%PYTHON64-LEGACY%" -m pip install pytest 32 | "%PYTHON64-LEGACY%" -m pip install --upgrade -r npbackup/requirements.txt || GOTO ERROR 33 | 34 | "%PYTHON64-LEGACY%" -m pytest C:\GIT\npbackup\tests || GOTO ERROR 35 | 36 | "%PYTHON64-LEGACY%" bin\compile.py --sign "C:\ODJ\KEYS\NetInventEV.dat" %* 37 | 38 | :: BUILD 32-BIT VERSION 39 | "%PYTHON32%" -m pip install --upgrade pip || GOTO ERROR 40 | "%PYTHON32%" -m pip install pytest 41 | "%PYTHON32%" -m pip install --upgrade -r npbackup/requirements-win32.txt || GOTO ERROR 42 | 43 | "%PYTHON32%" -m pytest C:\GIT\npbackup\tests || GOTO ERROR 44 | 45 | "%PYTHON32%" bin\compile.py --sign "C:\ODJ\KEYS\NetInventEV.dat" %* 46 | 47 | "%PYTHON64%" RESTIC_SOURCE_FILES/update_restic.py || GOTO ERROR 48 | 49 | :: BUILD 32-BIT LEGACY VERSION 50 | "%PYTHON32-LEGACY%" -m pip install --upgrade pip || GOTO ERROR 51 | "%PYTHON32-LEGACY%" -m pip install pytest 52 | "%PYTHON32-LEGACY%" -m pip install --upgrade -r npbackup/requirements-win32.txt || GOTO ERROR 53 | 54 | "%PYTHON32-LEGACY%" -m pytest C:\GIT\npbackup\tests || GOTO ERROR 55 | 56 | "%PYTHON32-LEGACY%" bin\compile.py --sign "C:\ODJ\KEYS\NetInventEV.dat" %* 57 | GOTO END 58 | 59 | :ERROR 60 | echo "Failed to run build script" 61 | :END 62 | SET PYTHONPATH=%OLD_PYTHONPATH% 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /bin/COMPILE.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This is an example compiler script 4 | 5 | cd /opt/npbackup 6 | git pull || exit 1 7 | 8 | OLD_PYTHONPATH="$PYTHONPATH" 9 | export PYTHONPATH=/opt/npbackup 10 | 11 | # For RHEL 7 based builds, we need to define path to locally built tcl8.6 12 | [ -d /usr/local/lib/tcl8.6 ] && export LD_LIBRARY_PATH=/usr/local/lib 13 | 14 | /opt/npbackup/venv/bin/python RESTIC_SOURCE_FILES/update_restic.py || exit 1 15 | 16 | /opt/npbackup/venv/bin/python -m pip install --upgrade pip || exit 1 17 | /opt/npbackup/venv/bin/python -m pip install pytest ||exit 1 18 | /opt/npbackup/venv/bin/python -m pip install --upgrade -r npbackup/requirements.txt || exit 1 19 | 20 | /opt/npbackup/venv/bin/python -m pytest /opt/npbackup/tests || exit 1 21 | 22 | /opt/npbackup/venv/bin/python bin/compile.py $@ 23 | 24 | export PYTHONPATH="$OLD_PYTHONPATH" -------------------------------------------------------------------------------- /bin/npbackup-cli: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup, and is really just a binary shortcut to launch npbackup.__main__ 5 | 6 | import os 7 | import sys 8 | 9 | sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))) 10 | 11 | import npbackup.__env__ 12 | npbackup.__env__.BUILD_TYPE = "cli" 13 | from npbackup.__main__ import main 14 | del sys.path[0] 15 | if __name__ == "__main__": 16 | while "--run-as-cli" in sys.argv: 17 | # Drop --run-as-cli argument since cli doesn't know about it 18 | sys.argv.pop(sys.argv.index("--run-as-cli")) 19 | main() 20 | -------------------------------------------------------------------------------- /bin/npbackup-gui: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup, and is really just a binary shortcut to launch npbackup.gui.__main__ 5 | 6 | import os 7 | import sys 8 | 9 | sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))) 10 | 11 | import npbackup.__env__ 12 | npbackup.__env__.BUILD_TYPE = "gui" 13 | from npbackup.gui.__main__ import main_gui 14 | from npbackup.__main__ import main 15 | 16 | del sys.path[0] 17 | 18 | if __name__ == "__main__": 19 | if "--run-as-cli" in sys.argv or "--check-config" in sys.argv: 20 | # Drop --run-as-cli argument since cli doesn't know about it 21 | while "--run-as-cli" in sys.argv: 22 | sys.argv.pop(sys.argv.index("--run-as-cli")) 23 | main() 24 | else: 25 | main_gui() 26 | -------------------------------------------------------------------------------- /bin/npbackup-viewer: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup, and is really just a binary shortcut to launch npbackup.gui.__main__ 5 | 6 | import os 7 | import sys 8 | 9 | sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))) 10 | 11 | import npbackup.__env__ 12 | npbackup.__env__.BUILD_TYPE = "viewer" 13 | from npbackup.gui.__main__ import main_gui 14 | 15 | del sys.path[0] 16 | 17 | if __name__ == "__main__": 18 | main_gui(viewer_mode=True) 19 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | This directory contains various example config files. 2 | -------------------------------------------------------------------------------- /examples/kvm-qemu/npbackup-cube.conf.template: -------------------------------------------------------------------------------- 1 | conf_version: 3.0 2 | repos: 3 | default: 4 | repo_uri: 5 | repo_group: default_group 6 | backup_opts: 7 | paths: 8 | - ___SOURCE___ 9 | source_type: files_from_verbatim 10 | exclude_files_larger_than: 0.0 KiB 11 | tags: 12 | - ___VM___ 13 | repo_opts: 14 | repo_password: 15 | retention_policy: {} 16 | prometheus: {} 17 | env: {} 18 | groups: 19 | default_group: 20 | backup_opts: 21 | paths: [] 22 | source_type: 23 | stdin_from_command: 24 | stdin_filename: 25 | tags: [] 26 | compression: auto 27 | use_fs_snapshot: false 28 | ignore_cloud_files: true 29 | exclude_caches: true 30 | one_file_system: true 31 | priority: low 32 | excludes_case_ignore: false 33 | exclude_files: 34 | - excludes/generic_excluded_extensions 35 | - excludes/generic_excludes 36 | - excludes/windows_excludes 37 | - excludes/linux_excludes 38 | exclude_patterns: [] 39 | exclude_files_larger_than: 40 | additional_parameters: 41 | additional_backup_only_parameters: 42 | minimum_backup_size_error: 2 GiB 43 | pre_exec_commands: 44 | - '[ -f /opt/cube/SNAPSHOT_FAILED ] && echo "Snapshot failed for $(cat /opt/cube/SNAPSHOT_FAILED)" && exit 1 || exit 0' 45 | pre_exec_per_command_timeout: 3600 46 | pre_exec_failure_is_fatal: false 47 | post_exec_commands: [] 48 | post_exec_per_command_timeout: 3600 49 | post_exec_failure_is_fatal: false 50 | post_exec_execute_even_on_backup_error: true 51 | repo_opts: 52 | repo_password: 53 | repo_password_command: 54 | minimum_backup_age: 1435 55 | upload_speed: 800 Mib 56 | download_speed: 0 Mib 57 | backend_connections: 0 58 | retention_policy: 59 | last: 3 60 | hourly: 72 61 | daily: 30 62 | weekly: 4 63 | monthly: 12 64 | yearly: 3 65 | tags: [] 66 | keep_within: true 67 | ntp_server: 68 | prometheus: 69 | backup_job: ___VM___ 70 | group: ${MACHINE_GROUP} 71 | env: 72 | env_variables: {} 73 | encrypted_env_variables: {} 74 | is_protected: false 75 | identity: 76 | machine_id: ${HOSTNAME}_${RANDOM}[4] 77 | machine_group: SOME_ARBITRARY_GROUP_NAME 78 | global_prometheus: 79 | metrics: false 80 | instance: ___VM___ 81 | destination: 82 | http_username: 83 | http_password: 84 | additional_labels: 85 | npf_tenant: ___TENANT___ 86 | backup_type: vm 87 | no_cert_verify: false 88 | global_options: 89 | auto_upgrade: false 90 | auto_upgrade_interval: 10 91 | auto_upgrade_server_url: 92 | auto_upgrade_server_username: 93 | auto_upgrade_server_password: 94 | auto_upgrade_host_identity: ${MACHINE_ID} 95 | auto_upgrade_group: ${MACHINE_GROUP} -------------------------------------------------------------------------------- /examples/npbackup.linux.conf.dist: -------------------------------------------------------------------------------- 1 | # NPBackup default configuration 2 | conf_version: 3.0.1 3 | audience: public 4 | repos: 5 | default: 6 | repo_uri: 7 | repo_group: default_group 8 | backup_opts: 9 | source_type: folder_list 10 | exclude_files_larger_than: 0.0 KiB 11 | repo_opts: 12 | repo_password: 13 | retention_policy: {} 14 | prometheus: {} 15 | env: {} 16 | is_protected: false 17 | groups: 18 | default_group: 19 | backup_opts: 20 | paths: [] 21 | source_type: 22 | stdin_from_command: 23 | stdin_filename: 24 | tags: [] 25 | compression: auto 26 | use_fs_snapshot: false 27 | ignore_cloud_files: true 28 | one_file_system: false 29 | priority: low 30 | exclude_caches: true 31 | excludes_case_ignore: false 32 | exclude_files: 33 | - excludes/generic_excluded_extensions 34 | - excludes/generic_excludes 35 | - excludes/linux_excludes 36 | - excludes/synology_excludes 37 | exclude_patterns: [] 38 | exclude_files_larger_than: 39 | additional_parameters: 40 | additional_backup_only_parameters: 41 | minimum_backup_size_error: 10 MiB 42 | pre_exec_commands: [] 43 | pre_exec_per_command_timeout: 3600 44 | pre_exec_failure_is_fatal: false 45 | post_exec_commands: [] 46 | post_exec_per_command_timeout: 3600 47 | post_exec_failure_is_fatal: false 48 | post_exec_execute_even_on_backup_error: true 49 | post_backup_housekeeping_percent_chance: 0 50 | post_backup_housekeeping_interval: 0 51 | repo_opts: 52 | repo_password: 53 | repo_password_command: 54 | minimum_backup_age: 1435 55 | upload_speed: 800 Mib 56 | download_speed: 0 Mib 57 | backend_connections: 0 58 | retention_policy: 59 | last: 3 60 | hourly: 72 61 | daily: 30 62 | weekly: 4 63 | monthly: 12 64 | yearly: 3 65 | tags: [] 66 | keep_within: true 67 | group_by_host: true 68 | group_by_tags: true 69 | group_by_paths: false 70 | ntp_server: 71 | prune_max_unused: 0 B 72 | prune_max_repack_size: 73 | prometheus: 74 | backup_job: ${MACHINE_ID} 75 | group: ${MACHINE_GROUP} 76 | env: 77 | env_variables: {} 78 | encrypted_env_variables: {} 79 | is_protected: false 80 | identity: 81 | machine_id: ${HOSTNAME}__${RANDOM}[4] 82 | machine_group: 83 | global_prometheus: 84 | metrics: false 85 | instance: ${MACHINE_ID} 86 | destination: 87 | http_username: 88 | http_password: 89 | additional_labels: {} 90 | no_cert_verify: false 91 | global_options: 92 | auto_upgrade: false 93 | auto_upgrade_percent_chance: 15 94 | auto_upgrade_interval: 0 95 | auto_upgrade_server_url: 96 | auto_upgrade_server_username: 97 | auto_upgrade_server_password: 98 | auto_upgrade_host_identity: ${MACHINE_ID} 99 | auto_upgrade_group: ${MACHINE_GROUP} 100 | -------------------------------------------------------------------------------- /examples/npbackup.windows.conf.dist: -------------------------------------------------------------------------------- 1 | # NPBackup default configuration 2 | conf_version: 3.0.1 3 | audience: public 4 | repos: 5 | default: 6 | repo_uri: 7 | repo_group: default_group 8 | backup_opts: 9 | source_type: folder_list 10 | exclude_files_larger_than: 0.0 KiB 11 | repo_opts: 12 | repo_password: 13 | retention_policy: {} 14 | prometheus: {} 15 | env: {} 16 | is_protected: false 17 | groups: 18 | default_group: 19 | backup_opts: 20 | paths: [] 21 | source_type: 22 | stdin_from_command: 23 | stdin_filename: 24 | tags: [] 25 | compression: auto 26 | use_fs_snapshot: true 27 | ignore_cloud_files: true 28 | one_file_system: false 29 | priority: low 30 | exclude_caches: true 31 | excludes_case_ignore: false 32 | exclude_files: 33 | - excludes/generic_excluded_extensions 34 | - excludes/generic_excludes 35 | - excludes/windows_excludes 36 | exclude_patterns: [] 37 | exclude_files_larger_than: 38 | additional_parameters: 39 | additional_backup_only_parameters: 40 | minimum_backup_size_error: 10 MiB 41 | pre_exec_commands: [] 42 | pre_exec_per_command_timeout: 3600 43 | pre_exec_failure_is_fatal: false 44 | post_exec_commands: [] 45 | post_exec_per_command_timeout: 3600 46 | post_exec_failure_is_fatal: false 47 | post_exec_execute_even_on_backup_error: true 48 | post_backup_housekeeping_percent_chance: 0 49 | post_backup_housekeeping_interval: 0 50 | repo_opts: 51 | repo_password: 52 | repo_password_command: 53 | minimum_backup_age: 1435 54 | upload_speed: 800 Mib 55 | download_speed: 0 Mib 56 | backend_connections: 0 57 | retention_policy: 58 | last: 3 59 | hourly: 72 60 | daily: 30 61 | weekly: 4 62 | monthly: 12 63 | yearly: 3 64 | tags: [] 65 | keep_within: true 66 | group_by_host: true 67 | group_by_tags: true 68 | group_by_paths: false 69 | ntp_server: 70 | prune_max_unused: 0 B 71 | prune_max_repack_size: 72 | prometheus: 73 | backup_job: ${MACHINE_ID} 74 | group: ${MACHINE_GROUP} 75 | env: 76 | env_variables: {} 77 | encrypted_env_variables: {} 78 | is_protected: false 79 | identity: 80 | machine_id: ${HOSTNAME}__${RANDOM}[4] 81 | machine_group: 82 | global_prometheus: 83 | metrics: false 84 | instance: ${MACHINE_ID} 85 | destination: 86 | http_username: 87 | http_password: 88 | additional_labels: {} 89 | no_cert_verify: false 90 | global_options: 91 | auto_upgrade: false 92 | auto_upgrade_percent_chance: 15 93 | auto_upgrade_interval: 0 94 | auto_upgrade_server_url: 95 | auto_upgrade_server_username: 96 | auto_upgrade_server_password: 97 | auto_upgrade_host_identity: ${MACHINE_ID} 98 | auto_upgrade_group: ${MACHINE_GROUP} 99 | -------------------------------------------------------------------------------- /examples/systemd/npbackup_upgrade_server.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=NPBackup upgrade server gunicorn service 3 | After=network.target 4 | 5 | [Service] 6 | User=npbackup 7 | Group=npbackup 8 | 9 | # Set this to whatever directory you installed the upgrade_server to 10 | ExecStart=/opt/upgrade_server/venv/bin/python /var/npbackup_upgrade_server/upgrade_server.py --config-file=/etc/npbackup_upgrade_server.conf 11 | WorkingDirectory=/var/npbackup_upgrade_server 12 | Environment="PYTHONPATH=/var/npbackup_upgrade_server" 13 | Restart=always 14 | RestartSec=60 15 | 16 | [Install] 17 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /examples/upgrade_server/npbackup_upgrade_server.conf.dist: -------------------------------------------------------------------------------- 1 | # NPBackup upgrade server configuration file 2 | 3 | http_server: 4 | listen: 0.0.0.0 5 | port: 8080 6 | users: 7 | - username: upgrade_client_user 8 | password: super_secret_password 9 | permissions: 10 | audience: 11 | - private 12 | - public 13 | 14 | 15 | upgrades: 16 | data_root: /var/npbackup_upgrade_server/dist 17 | # We'll store a CSV containing backup clients that upgrade here 18 | statistics_file: /var/npbackup_upgrade_server/stats.csv 19 | 20 | # See github wiki for more explanation of the contents of data_root dir 21 | 22 | -------------------------------------------------------------------------------- /examples/upgrade_server/upgrade_script.cmd: -------------------------------------------------------------------------------- 1 | :: Example upgrade script for NPBackup that will be pushed server side 2 | 3 | :: The following variables will be overwritten by the upgrade process 4 | :: {CURRENT_DIR} - The current directory of the distribution 5 | :: {backup_dist} - A directory where we try to move / copy the current distribution 6 | :: {upgrade_dist} - The directory where the new distribution is extracted to after download 7 | :: {downloaded_archive} - The path to the downloaded archive 8 | :: {log_file} - The log file where the output of this script will be written 9 | :: {original_args} - The arguments that were passed to the upgrade script 10 | 11 | :: Also, I really HATE batch files, from the bottom of my programmer heart 12 | :: Every try to write a one-liner batch with some variables and perhaps if statements ? 13 | :: With or without setlocal enabledelayedexpansion, you won't get what you want, it's a nightmare 14 | :: eg command & IF %ERRORLEVEL% EQU 0 (echo "Success") ELSE (echo "Failure") 15 | :: Run this a couple of times with good and bad exit code commands and you'll see the "memory" of previous runs 16 | :: Also, using !ERRORLEVEL! produces another type of "memory" 17 | :: So here we are, in GOTO land, like in the good old Commodore 64 days 18 | 19 | echo "Launching upgrade" >> "{log_file}" 2>&1 20 | echo "Moving current dist from {CURRENT_DIR} to {backup_dist}" >> "{log_file}" 2>&1 21 | move /Y "{CURRENT_DIR}" "{backup_dist}" >> "{log_file}" 2>&1 || GOTO MOVE_FAILED 22 | GOTO MOVE_OK 23 | :MOVE_FAILED 24 | echo "Moving current dist failed. Trying to copy it." >> "{log_file}" 2>&1 25 | xcopy /S /Y /I "{CURRENT_DIR}\*" "{backup_dist}" >> "{log_file}" 2>&1 26 | echo "Now trying to overwrite current dist with upgrade dist" >> "{log_file}" 2>&1 27 | xcopy /S /Y /I "{upgrade_dist}\*" "{CURRENT_DIR}" >> "{log_file}" 2>&1 28 | GOTO TESTRUN 29 | :MOVE_OK 30 | echo "Moving upgraded dist from {upgrade_dist} to {CURRENT_DIR}" >> "{log_file}" 2>&1 31 | move /Y "{upgrade_dist}" "{CURRENT_DIR}" >> "{log_file}" 2>&1 32 | echo "Copying optional configuration files from {backup_dist} to {CURRENT_DIR}" >> "{log_file}" 2>&1 33 | xcopy /S /Y /I "{backup_dist}\*conf" {CURRENT_DIR} > NUL 2>&1 34 | GOTO TESTRUN 35 | :TESTRUN 36 | echo "Loading new executable {CURRENT_EXECUTABLE} --run-as-cli --check-config {original_args}" >> "{log_file}" 2>&1 37 | "{CURRENT_EXECUTABLE}" --run-as-cli --check-config {original_args} >> "{log_file}" 2>&1 || GOTO FAILED_TEST_RUN 38 | GOTO TEST_RUN_OK 39 | :FAILED_TEST_RUN 40 | echo "New executable failed. Rolling back" >> "{log_file}" 2>&1 41 | echo "Trying to move back" >> "{log_file}" 2>&1 42 | move /Y "{CURRENT_DIR}" "{backup_dist}.original" >> "{log_file}" 2>&1 || GOTO MOVE_BACK_FAILED 43 | move /Y "{backup_dist}" "{CURRENT_DIR}" >> "{log_file}" 2>&1 44 | GOTO RUN_AS_PLANNED 45 | :MOVE_BACK_FAILED 46 | echo "Moving files back failed. Trying to overwrite" >> "{log_file}" 2>&1 47 | xcopy /S /Y /I "{backup_dist}\*" "{CURRENT_DIR}" >> "{log_file}" 2>&1 48 | GOTO RUN_AS_PLANNED 49 | :TEST_RUN_OK 50 | echo "Upgrade successful" >> "{log_file}" 2>&1 51 | rd /S /Q "{backup_dist}" >> "{log_file}" 2>&1 52 | rd /S /Q "{upgrade_dist}" > NUL 2>&1 53 | del /F /S /Q "{downloaded_archive}" >> "{log_file}" 2>&1 54 | GOTO RUN_AS_PLANNED 55 | :RUN_AS_PLANNED 56 | echo "Running as initially planned:" >> "{log_file}" 2>&1 57 | echo "{CURRENT_EXECUTABLE} {original_args}" >> "{log_file}" 2>&1 58 | "{CURRENT_EXECUTABLE}" {original_args} 59 | echo "Upgrade script run finished" >> "{log_file}" 2>&1 -------------------------------------------------------------------------------- /examples/upgrade_server/upgrade_script.sh: -------------------------------------------------------------------------------- 1 | #/usr/bin/env bash 2 | # Example upgrade script for NPBackup that will be pushed server side 3 | 4 | # The following variables will be overwritten by the upgrade process 5 | # {CURRENT_DIR} - The current directory of the distribution 6 | # {backup_dist} - A directory where we try to move / copy the current distribution 7 | # {upgrade_dist} - The directory where the new distribution is extracted to after download 8 | # {downloaded_archive} - The path to the downloaded archive 9 | # {log_file} - The log file where the output of this script will be written 10 | # {original_args} - The arguments that were passed to the upgrade script 11 | 12 | 13 | echo "Launching upgrade" >> "{log_file}" 2>&1 14 | echo "Moving current dist from {CURRENT_DIR} to {backup_dist}" >> "{log_file}" 2>&1 15 | mv -f "{CURRENT_DIR}" "{backup_dist}" >> "{log_file}" 2>&1 16 | echo "Moving upgraded dist from {upgrade_dist} to {CURRENT_DIR}" >> "{log_file}" 2>&1 17 | mv -f "{upgrade_dist}" "{CURRENT_DIR}" >> "{log_file}" 2>&1 18 | echo "Copying optional configuration files from {backup_dist} to {CURRENT_DIR}" >> "{log_file}" 2>&1 19 | find "{backup_dist}" -name "*.conf" -exec cp --parents "{}" "{CURRENT_DIR}" \; 20 | echo "Adding executable bit to new executable" >> "{log_file}" 2>&1 21 | chmod +x "{CURRENT_EXECUTABLE}" >> "{log_file}" 2>&1 22 | echo "Loading new executable {CURRENT_EXECUTABLE} --run-as-cli --check-config {original_args}" >> "{log_file}" 2>&1 23 | "{CURRENT_EXECUTABLE}" --run-as-cli --check-config {original_args} >> "{log_file}" 2>&1 24 | if [ $? -ne 0 ]; then 25 | echo "New executable failed. Rolling back" >> "{log_file}" 2>&1 26 | mv -f "{CURRENT_DIR}" "{backup_dist}.original">> "{log_file}" 2>&1 27 | mv -f "{backup_dist}" "{CURRENT_DIR}" >> "{log_file}" 2>&1 28 | else 29 | echo "Upgrade successful" >> "{log_file}" 2>&1 30 | rm -rf "{backup_dist}" >> "{log_file}" 2>&1 31 | rm -rf "{upgrade_dist}" >> "{log_file}" 2>&1 32 | rm -rf "{downloaded_archive}" >> "{log_file}" 2>&1 33 | fi 34 | echo "Running as initially planned:" >> "{log_file}" 2>&1 35 | echo "{CURRENT_EXECUTABLE} {original_args}" >> "{log_file}" 2>&1 36 | "{CURRENT_EXECUTABLE}" {original_args} 37 | echo "Upgrade script run finished" >> "{log_file}" 2>&1 -------------------------------------------------------------------------------- /excludes/generic_excluded_extensions: -------------------------------------------------------------------------------- 1 | # File extensions exclude patterns for backup 2 | # patterns are FileMatch compatible (restic) 3 | 4 | # rev 2023010701 5 | 6 | *.back 7 | *.bak 8 | *.bkp 9 | *.cache 10 | *.chk 11 | *.dmp 12 | *.dump 13 | *.err 14 | *.lock 15 | *.lockfile 16 | *.log 17 | *.log[0-9] 18 | *.log.[0-9] 19 | *.log.[0-9][0-9] 20 | *.old 21 | *.tmp 22 | *.temp 23 | *.pid 24 | 25 | # Browser not finished downloads 26 | *.download 27 | *.crdownload 28 | *.part 29 | 30 | # Adobe lightroom preview files 31 | *.lrprev 32 | 33 | # AutoCAD 34 | *.dwl 35 | *.dwl2 36 | *.atmp 37 | 38 | # Microsoft Access lock file 39 | *.laccdb 40 | *.swp 41 | 42 | # Microsoft Outlook Exchange sync files (can still be backed up since we may convert from ost to pst) 43 | #*.ost 44 | 45 | # Microsoft Tracelog files 46 | *.etl 47 | 48 | # Python compiled files 49 | *.py[cod] 50 | -------------------------------------------------------------------------------- /excludes/generic_excludes: -------------------------------------------------------------------------------- 1 | # patterns are FileMatch compatible (restic) 2 | 3 | # rev 2023010701 4 | 5 | # Microsoft Office lock files (eg */~$somefile.doc) 6 | **/~$$* 7 | 8 | # MacOS files 9 | **/.DS_Store 10 | 11 | # odrive sync dir 12 | **/.odrive 13 | 14 | # Generic directories / files 15 | **/cache*.db 16 | **/TemporaryFiles 17 | 18 | # Generic Thumbs files 19 | **/thumb*.db 20 | **/icon*.db 21 | **/icons*.db 22 | **/*cache.db 23 | **/iconcache*.db 24 | 25 | # Generic Thumbs directories 26 | **/thumbnails 27 | **/thumbcache*.db 28 | 29 | # Generic tmp dir 30 | **/Temp 31 | **/tmp 32 | 33 | # Generic caches dir for most navigators 34 | 35 | **/.cache 36 | **/Font*Cache 37 | **/File*Cache 38 | **/Dist*Cache 39 | **/Native*Cache 40 | **/Play*Cache 41 | **/Asset*Cache 42 | **/Activities*Cache 43 | **/Script*Cache 44 | **/Gpu*Cache 45 | **/Code*Cache 46 | **/Local*Cache 47 | **/Session*Cache 48 | **/Web*Cache 49 | **/JS*Cache 50 | **/CRL*Cache 51 | **/GrShader*Cache 52 | **/Shader*Cache 53 | **/Cache_data 54 | **/Cache 55 | **/Caches 56 | **/CacheStorage 57 | **/Cachedata 58 | **/CachedFiles 59 | **/Cache*Storage 60 | **/Cache*data 61 | **/Cached*Files 62 | # firefox specific 63 | **/cache2 64 | 65 | # npm 66 | **/npm-cache 67 | 68 | # Thumbnails folder 69 | **/Thumbnails 70 | 71 | # Lock files 72 | **/lock 73 | **/lockfile 74 | 75 | # Generic cookie files and folders 76 | **/Cookie 77 | 78 | # Python cache files 79 | **/__pycache__ 80 | 81 | # Synology NAS working directory 82 | #exclude_regex = .*/.SynologyWorkingDirectory 83 | **/.SynologyWorkingDirectory 84 | 85 | **/.tmp.drivedownload 86 | 87 | # Adobe CS files 88 | **/web-cache-temp 89 | 90 | # Speific coverage files 91 | **/.tox 92 | **/.nox 93 | **/.coverage 94 | **/.pytest_cache 95 | 96 | # Python VENV back files 97 | **/env.bak 98 | **/venv.bak 99 | 100 | # PyCharm 101 | **/.idea 102 | 103 | # VSCore 104 | **/.vscode 105 | -------------------------------------------------------------------------------- /excludes/linux_excludes: -------------------------------------------------------------------------------- 1 | # Unixes Fs exclude patterns for backup 2 | # patterns are FileMatch compatible (restic) 3 | 4 | # rev 2025010901 5 | 6 | # Generic unix sys path excludes 7 | /dev 8 | lost+found 9 | /media 10 | /proc 11 | /sys 12 | /run 13 | /selinux 14 | /var/cache 15 | /var/log 16 | /var/run 17 | /var/tmp 18 | /tmp 19 | # Let's keep /mnt since it's a common point for servers with external disks 20 | #/mnt 21 | 22 | # More MacOS specific sys path excludes 23 | /afs 24 | /Network 25 | /automount 26 | /private/Network 27 | /private/tmp 28 | /private/var/tmp 29 | /private/var/folders 30 | /private/var/run 31 | /private/var/spool/postfix 32 | /private/var/automount 33 | /private/var/db/fseventsd 34 | /Previous Systems 35 | 36 | # For user file exclusions, we'll keep both $HOME/ and /home/*/ syntaxes so we are sure to exclude users which home isn't /home, but still be able to exclude all other /home/user directories 37 | # Home directory excludes mostly found on unixes 38 | $HOME/Downloads 39 | $HOME/Library 40 | $HOME/snap 41 | $HOME/.Trash 42 | $HOME/.bundle 43 | $HOME/.cache 44 | $HOME/.dbus 45 | $HOME/.debug 46 | $HOME/.gvfs 47 | $HOME/.local/share/gvfs-metadata 48 | $HOME/.local/share/Trash 49 | $HOME/.dropbox 50 | $HOME/.dropbox-dist 51 | $HOME/.local/pipx 52 | $HOME/.local/share/Trash 53 | $HOME/.npm 54 | $HOME/.pyenv 55 | $HOME/.thumbnails 56 | $HOME/.virtualenvs 57 | $HOME/.Trash 58 | $HOME/.recently-used 59 | $HOME/.xession-errors 60 | $HOME/OneDrive 61 | $HOME/Dropbox 62 | $HOME/SkyDrive* 63 | 64 | /home/*/Downloads 65 | /home/*/Library 66 | /home/*/snap 67 | /home/*/.Trash 68 | /home/*/.bundle 69 | /home/*/.cache 70 | /home/*/.dbus 71 | /home/*/.debug 72 | /home/*/.gvfs 73 | /home/*/.local/share/gvfs-metadata 74 | /home/*/.local/share/Trash 75 | /home/*/.dropbox 76 | /home/*/.dropbox-dist 77 | /home/*/.local/pipx 78 | /home/*/.local/share/Trash 79 | /home/*/.npm 80 | /home/*/.pyenv 81 | /home/*/.thumbnails 82 | /home/*/.virtualenvs 83 | /home/*/.Trash 84 | /home/*/.recently-used 85 | /home/*/.xession-errors 86 | /home/*/OneDrive 87 | /home/*/Dropbox 88 | /home/*/SkyDrive* 89 | 90 | # Some morre generic MacOS exclusions 91 | **/Network Trash Folder 92 | **/.fseventsd* 93 | **/.Spotlight-* 94 | **/*Mobile*Backups 95 | -------------------------------------------------------------------------------- /excludes/synology_excludes: -------------------------------------------------------------------------------- 1 | # Don't backup system related directories from Synology NAS 2 | /volume*/@* 3 | # Avoid sync directories too 4 | **/@eaDir 5 | # Avoid recycle bin directories 6 | **/#recylce -------------------------------------------------------------------------------- /excludes/windows_excludes: -------------------------------------------------------------------------------- 1 | # Windows Fs exclude patterns for backup 2 | # patterns are FileMatch compatible (restic) 3 | 4 | # rev 2023011501 5 | 6 | # Generic Microsoft excludes 7 | ?:\Users\*\AppData\Local\Temp 8 | ?:\Users\*\AppData\LocalLow 9 | ?:\Users\*\Documents and Settings\*\Cookies 10 | ?:\Users\*\Documents and Settings\*\Recent 11 | ?:\Users\*\Documents and Settings\*\Local Settings\Temp 12 | ?:\Users\*\AppData\Roaming\Microsoft\Windows\Recent 13 | ?:\Users\*\AppData\Local\Temp 14 | ?:\Users\*\AppData\Local\History 15 | ?:\Users\*\AppData\Local\Application Data 16 | ?:\Users\*\AppData\Local\Microsoft\Internet Explorer 17 | ?:\Users\*\AppData\Local\Microsoft\Windows\History 18 | ?:\Users\*\AppData\Local\Microsoft\Terminal Server Client\Cache 19 | ?:\Users\*\AppData\**\Temp 20 | ?:\Users\*\AppData\Roaming\Microsoft\Windows\Themes\CachedFiles 21 | ?:\Users\*\AppData\Internet Explorer 22 | ?:\Users\*\AppData\*\Microsoft\*\InetCookies 23 | ?:\Users\*\AppData\*\Microsoft Office\*\OfficeFileCache 24 | ?:\Users\*\AppData\*\Microsoft Office\*\WebServiceCache 25 | ?:\Users\*\AppData\*\Microsoft Office\*\Lync\Tracing 26 | ?:\Users\*\AppData\*\Microsoft Office\*\Recent 27 | ?:\Users\*\AppData\Local\Microsoft\Office\*OfficeFileCache 28 | ?:\Users\*\AppData\Local\Microsoft\Office\Recent 29 | ?:\Users\*\AppData\Local\Microsoft\Outlook\RoamCache 30 | ?:\Users\*\AppData\LocalLow\Microsoft\CryptnetUrlCache 31 | ?:\Users\*\AppData\Local\Downloaded Installations 32 | ?:\Users\*\AppData\Local\GroupPolicy 33 | ?:\Users\*\AppData\Local\Microsoft\AppV 34 | ?:\Users\*\AppData\Local\Microsoft\Messenger 35 | ?:\Users\*\AppData\Local\Microsoft\OneNote 36 | ?:\Users\*\AppData\Local\Microsoft\Terminal Server Client 37 | ?:\Users\*\AppData\Local\Microsoft\UEV 38 | ?:\Users\*\AppData\Local\Microsoft\Windows Live 39 | ?:\Users\*?:\Users\*\AppData\Local\Microsoft\Windows Live Contacts 40 | ?:\Users\*\AppData\Local\Microsoft\Application Shortcuts 41 | ?:\Users\*\AppData\Local\Microsoft\Notifications 42 | ?:\Users\*\AppData\Local\Microsoft\Windows\UsrClass.dat.LOG* 43 | ?:\Users\*\ntuser.dat.LOG* 44 | ?:\Users\*\**\Temporary Internet Files 45 | ?:\Users\*\AppData\**\Microsoft\Windows\INetCache\* 46 | 47 | 48 | # Temp folder for files that are ready to be burned on DVD 49 | ?:\Users\*\AppData\Local\Microsoft\Windows\Burn 50 | ?:\Users\*\AppData\Local\Microsoft\Windows\CD Burning 51 | 52 | # Generic application cache & temp folders (excludes all aaaCacheaaa or bbbTempbbb dirs) 53 | ?:\Users\*\AppData\Local\*\*Cache* 54 | ?:\Users\*\AppData\LocalLow\*\*Cache* 55 | ?:\Users\*\AppData\Roaming\*\*Cache* 56 | ?:\Users\*\AppData\Local\*\*Temp* 57 | ?:\Users\*\AppData\LocalLow\*\*Temp* 58 | ?:\Users\*\AppData\Roaming\*\*Temp* 59 | 60 | # Various Win 10 caches 61 | **\OfficeFileCache 62 | **\SmartLookupCache 63 | **\BackstageInAppNavCache 64 | **\MruServiceCache 65 | 66 | # Error reports 67 | ?:\Users\*\AppData\Local\Microsoft\Windows\WER 68 | ?:\Users\*\AppData\Local\CrashDumps 69 | ?:\Users\*\AppData\Local\Diagnostics 70 | 71 | # Windows 10 Edge 72 | ?:\Users\*\AppData\Local\Microsoft\Windows\*Cache 73 | ?:\Users\*\MicrosoftEdgeBackups 74 | ?:\Users\*\AppData\Local\MicrosoftEdge\SharedCacheContainers 75 | ?:\Users\*\AppData\Local\Microsoft\Edge\User Data\Default\DawnCache 76 | 77 | # Windows 10 Store Application cache and state 78 | ?:\Users\*\AppData\Local\Packages\*\AC 79 | ?:\Users\*\AppData\Local\Packages\*\TempState 80 | ?:\Users\*\AppData\Local\Packages\*\LocalState 81 | ?:\Users\*\AppData\Local\Packages\*\LocalCache 82 | ?:\Users\*\AppData\Local\Packages\*\RoamingState 83 | ?:\Users\*\AppData\Local\Packages\*\AppData\User\Default\CacheStorage 84 | ?:\Users\*\AppData\Local\Packages\*\AppData\CacheStorage 85 | ?:\Users\*\AppData\Local\Package Cache 86 | 87 | # Windows 10 Windows application reparse points 88 | ?:\Users\*\AppData\Local\Microsoft\WindowsApps 89 | 90 | # Windows 10 various stuff 91 | ?:\Users\*\AppData\Local\Microsoft\Windows\Notifications 92 | ?:\Users\*\AppData\Local\Microsoft\Windows\Explorer 93 | 94 | # Windows downloads 95 | ?:\Users\*\Downloads 96 | 97 | # Windows update cache 98 | ?:\Windows\SoftwareDistribution\Download 99 | 100 | # Windows offline files 101 | ?:\Windows\CSC 102 | 103 | # Generic Windows folders 104 | ?:\Windows\Temp 105 | ?:\Windows\Downloaded Program Files 106 | ?:\RECYCLER 107 | ?:\$$recycle.bin 108 | ?:\System Volume Information 109 | 110 | # VSS mountpoints 111 | \\$?\GLOBALROOT\Device\HarddiskVolumeShadowCopy* 112 | 113 | # swap file (Windows XP, 7, 8) 114 | ?:\pagefile.sys 115 | 116 | # swap file (Windows 8) 117 | ?:\swapfile.sys 118 | 119 | # hibernation file 120 | ?:\hiberfil.sys 121 | 122 | # Windows Upgrade temp download folder 123 | ?:\$$WINDOWS.~BT 124 | 125 | # Windows 10 Upgrade previous install 126 | ?:\Windows.old 127 | 128 | # Windows performance logs 129 | ?:\PerfLogs 130 | 131 | # Windows filesystem directories 132 | ?:\$$mft 133 | ?:\$$logfile 134 | ?:\$$volume 135 | ?:\$$bitmap 136 | ?:\$$extend 137 | ?:\$$reparse 138 | 139 | # Onedrive AppData 140 | ?:\Users\*\AppData\Local\Microsoft\OneDrive 141 | 142 | # Unnecessary folder exclusions 143 | ?:\Users\AppData\Roaming\Microsoft\Windows\Cookies 144 | ?:\Users\*\NetHood 145 | ?:\Users\*\PrintHood 146 | ?:\Users\*\Cookies 147 | ?:\Users\*\Recent 148 | ?:\Users\*\SentTo 149 | ?:\Users\*\LocalService 150 | ?:\Users\*\NetworkService 151 | ?:\Users\*\AppData\LocalLow 152 | ?:\Users\*\Tracing 153 | 154 | # Generic system file exclusions 155 | **\MSOCache 156 | **\MSOCache.* 157 | **\Config.Msi 158 | 159 | #### Applications 160 | 161 | # Office telemetry data 162 | ?:\Users\*\AppData\Local\Microsoft\Office\OTeleData* 163 | 164 | 165 | # Blink based navigators (can be in AppData\Local or AppData\Roaming) 166 | ?:\Users\*\AppData\**\Local Storage 167 | ?:\Users\*\AppData\**\Session Storage 168 | ?:\Users\*\AppData\**\Crash Reports 169 | ?:\Users\*\AppData\**\sessionstore\.bak 170 | ?:\Users\*\AppData\**\DawnCache 171 | 172 | # Chrome 66+ 173 | ?:\Users\*\AppData\**\Chrome\User Data\**\LOG 174 | ?:\Users\*\AppData\**\Chrome\User Data\**\File System 175 | ?:\Users\*\AppData\**\Chrome\User Data\**\SwReporter 176 | ?:\Users\*\AppData\**\Chrome\User Data\**\PepperFlash 177 | 178 | # Opera 41+ 179 | ?:\Users\*\AppData\**\Opera\Opera\profile\cache4 180 | ?:\Users\*\AppData\Roaming\Opera Software\**\Sessions 181 | ?:\Users\*\AppData\Roaming\Opera Software\**\LOG 182 | 183 | # Vivaldi 1.x+ 184 | ?:\Users\*\AppData\**\Vivaldi\User Data\Application 185 | ?:\Users\*\AppData\**\Vivaldi\User Data\LOG 186 | 187 | 188 | # Thunderbird 189 | ?:\Users\*\AppData\Local\Thunderbird\Mozilla Thunderbird\updates 190 | ?:\Users\*\AppData\Roaming\Thunderbird\Profiles\*\crashes 191 | 192 | 193 | 194 | # Github Desktop 195 | ?:\Users\*\AppData\Local\GitHubDesktop 196 | 197 | # Google Apps Sync 198 | ?:\Users\*\AppData\Local\Google\Google Apps Sync\Tracing 199 | 200 | # Adobe Acrobat DC 201 | ?:\Users\*\AppData\Local\Adobe\AcroCef\DC\Acrobat\Cache 202 | 203 | # Apple Logs 204 | ?:\Users\*\AppData\Local\Apple Computer\Logs 205 | ?:\Users\*\AppData\Roaming\Apple Computer\Logs 206 | 207 | # Apple iPhone backups :( 208 | ?:\Users\*\AppData\Roaming\Apple Computer\MobileSync\Backup 209 | 210 | # iTunes downloaded album artwork 211 | ?:\Users\*\Music\iTunes\Album Artwork\Download 212 | 213 | # Java 214 | ?:\Users\*\AppData\Local\Sun 215 | ?:\Users\*\AppData\LocalLow\Sun\Java\Deployment\log 216 | ?:\Users\*\AppData\Roaming\Sun\Java\Deployment\log 217 | 218 | # Cisco Webex 219 | ?:\Users\*\AppData\Local\webEx\wbxcache 220 | 221 | # Ignite Realtime Spark client logs 222 | ?:\Users\*\AppData\Roaming\Spark\logs 223 | 224 | # TeamViewer \ SimpleHelp quick support 225 | ?:\Users\*\AppData\*\Teamviewer 226 | ?:\Users\*\AppData\*\JWrapper-Remote Support 227 | ?:\Users\*\AppData\*\JWrapper-SimpleHelp Technician 228 | 229 | 230 | # Zoom remote tool 231 | ?:\Users\*\AppData\*\Zoom 232 | 233 | 234 | # Dropbox, OneDrive, SkyDrive data directories (not excluded by default because of cryptolockers attacks) 235 | #?:\Users\*\SkyDrive* 236 | #?:\Users\*\Dropbox 237 | #?:\Users\*\OneDrive 238 | 239 | # Dropbox config directory 240 | ?:\Users\*\AppData\Local\Dropbox 241 | 242 | # Owncloud \ nextcloud logs 243 | ?:\Users\AppData\Local\owncloud 244 | ?:\Users\AppData\Local\Nextcloud 245 | ?:\Users\AppData\Roaming\Nextcloud\logs 246 | 247 | # AMD cache files 248 | ?:\Users\*\AppData\Local\AMD\*Cache 249 | 250 | # DirectX Cache 251 | ?:\Users\*\AppData\Local\D3DSCache 252 | 253 | # Restic caches 254 | ?:\Users\*\AppData\Local\restic 255 | 256 | # VSCode history + logs 257 | ?:\Users\*\AppData\Roaming\Code\User\History 258 | ?:\Users\*\AppData\Roaming\Code\logs 259 | -------------------------------------------------------------------------------- /excludes/windows_program: -------------------------------------------------------------------------------- 1 | # windows sys Fs exclude patterns for backup 2 | # patterns are FileMatch compatible (restic) 3 | 4 | # rev 2023010701 5 | 6 | # Exclusion list that removes most of Windows system and program files 7 | # This list is a complementary to windows_excludes 8 | 9 | ?:\Windows 10 | ?:\Program Files 11 | ?:\Program Files (x86) 12 | ?:\ProgramData 13 | 14 | 15 | ## Optional program data (should be safe to exclude) 16 | # Edge 17 | ?:\Users\*\AppData\Local\Microsoft\Edge 18 | # Microsoft Teams 19 | ?:\Users\*\AppData\Local\Microsoft\Teams 20 | # UWP Binaries 21 | ?:\Users\*\AppData\Local\Packages 22 | 23 | ?:\Users\*\AppData\Local\Logseq 24 | ?:\Users\*\AppData\Local\BraveSoftware 25 | # .pylint.d exclusion does not work 26 | ?:\Users\*\AppData\.pylint.d 27 | ?:\Users\*\AppData\Local\GoToMeeting 28 | ?:\Users\*\AppData\Local\Programs\GIMP* 29 | ?:\Users\*\AppData\Local\Programs\Opera 30 | ?:\Users\*\AppData\Local\pypa 31 | ?:\Users\*\AppData\Local\JetBrains 32 | 33 | ## Optional program data, less safe to exclude, enable if you know what you are doing 34 | # ?:\Users\*\AppData\Roaming\npm 35 | # ?:\Users\*\AppData\Roaming\JetBrains 36 | # ?:\Users\*\AppData\Roaming\Microsoft\Teams 37 | # ?:\Users\*\AppData\Roaming\Roaming\LibreOffice 38 | # ?:\Users\*\AppData\Roaming\REAPER -------------------------------------------------------------------------------- /img/backup_window_v2.2.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/backup_window_v2.2.0.png -------------------------------------------------------------------------------- /img/configuration_v2.1.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/configuration_v2.1.0.png -------------------------------------------------------------------------------- /img/configuration_v2.2.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/configuration_v2.2.0.png -------------------------------------------------------------------------------- /img/configuration_v2.2.0rc2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/configuration_v2.2.0rc2.png -------------------------------------------------------------------------------- /img/configuration_v3.0.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/configuration_v3.0.0.png -------------------------------------------------------------------------------- /img/grafana_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/grafana_dashboard.png -------------------------------------------------------------------------------- /img/grafana_dashboard_2.2.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/grafana_dashboard_2.2.0.png -------------------------------------------------------------------------------- /img/grafana_dashboard_20250211.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/grafana_dashboard_20250211.png -------------------------------------------------------------------------------- /img/grafana_dashboard_20250226.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/grafana_dashboard_20250226.png -------------------------------------------------------------------------------- /img/interface_v2.1.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/interface_v2.1.0.png -------------------------------------------------------------------------------- /img/interface_v2.2.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/interface_v2.2.0.png -------------------------------------------------------------------------------- /img/interface_v3.0.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/interface_v3.0.0.png -------------------------------------------------------------------------------- /img/orchestrator_v3.0.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/orchestrator_v3.0.0.png -------------------------------------------------------------------------------- /img/restore_window_v2.1.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/restore_window_v2.1.0.png -------------------------------------------------------------------------------- /img/restore_window_v2.2.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/restore_window_v2.2.0.png -------------------------------------------------------------------------------- /img/restore_window_v3.0.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/restore_window_v3.0.0.png -------------------------------------------------------------------------------- /img/viewer_v3.0.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/img/viewer_v3.0.0.png -------------------------------------------------------------------------------- /misc/npbackup-cli.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | setlocal 4 | 5 | if exist "%~dp0..\python.exe" ( 6 | "%~dp0..\python" -m npbackup %* 7 | ) else if exist "%~dp0python.exe" ( 8 | "%~dp0python" -m npbackup %* 9 | ) else ( 10 | "python" -m npbackup %* 11 | ) 12 | 13 | endlocal 14 | -------------------------------------------------------------------------------- /npbackup/__debug__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.__debug__" 7 | __author__ = "Orsiris de Jong" 8 | __site__ = "https://www.netperfect.fr/npbackup" 9 | __description__ = "NetPerfect Backup Client" 10 | __copyright__ = "Copyright (C) 2023-2025 NetInvent" 11 | __build__ = "2025021901" 12 | 13 | 14 | import sys 15 | import os 16 | import traceback 17 | from typing import Callable 18 | from functools import wraps 19 | from logging import getLogger 20 | import json 21 | 22 | 23 | logger = getLogger() 24 | 25 | 26 | # If set, debugging will be enabled by setting environment variable to __SPECIAL_DEBUG_STRING content 27 | # Else, a simple true or false will suffice 28 | __SPECIAL_DEBUG_STRING = "" 29 | __debug_os_env = os.environ.get("_DEBUG", "False").strip("'\"") 30 | 31 | 32 | if not __SPECIAL_DEBUG_STRING: 33 | if "--debug" in sys.argv: 34 | _DEBUG = True 35 | sys.argv.pop(sys.argv.index("--debug")) 36 | 37 | 38 | if not "_DEBUG" in globals(): 39 | _DEBUG = False 40 | if __SPECIAL_DEBUG_STRING: 41 | if __debug_os_env == __SPECIAL_DEBUG_STRING: 42 | _DEBUG = True 43 | elif __debug_os_env.lower().capitalize() == "True": 44 | _DEBUG = True 45 | 46 | 47 | _NPBACKUP_ALLOW_AUTOUPGRADE_DEBUG = ( 48 | True 49 | if os.environ.get("_NPBACKUP_ALLOW_AUTOUPGRADE_DEBUG", "False") 50 | .strip("'\"") 51 | .lower() 52 | .capitalize() 53 | == "True" 54 | else False 55 | ) 56 | 57 | 58 | def exception_to_string(exc): 59 | """ 60 | Transform a caught exception to a string 61 | https://stackoverflow.com/a/37135014/2635443 62 | """ 63 | stack = traceback.extract_stack()[:-3] + traceback.extract_tb( 64 | exc.__traceback__ 65 | ) # add limit=?? 66 | pretty = traceback.format_list(stack) 67 | return "".join(pretty) + "\n {} {}".format(exc.__class__, exc) 68 | 69 | 70 | def catch_exceptions(fn: Callable): 71 | """ 72 | Catch any exception and log it so we don't loose exceptions in thread 73 | """ 74 | 75 | @wraps(fn) 76 | def wrapper(self, *args, **kwargs): 77 | try: 78 | # pylint: disable=E1102 (not-callable) 79 | return fn(self, *args, **kwargs) 80 | except Exception as exc: 81 | # pylint: disable=E1101 (no-member) 82 | operation = fn.__name__ 83 | logger.error(f"General catcher: Function {operation} failed with: {exc}") 84 | logger.error("Trace:", exc_info=True) 85 | return None 86 | 87 | return wrapper 88 | 89 | 90 | def fmt_json(js: dict): 91 | """ 92 | Just a quick and dirty shorthand for pretty print which doesn't require pprint 93 | to be loaded 94 | """ 95 | js = json.dumps(js, indent=4) 96 | return js 97 | -------------------------------------------------------------------------------- /npbackup/__env__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.__env__" 7 | __author__ = "Orsiris de Jong" 8 | __site__ = "https://www.netperfect.fr/npbackup" 9 | __description__ = "NetPerfect Backup Client" 10 | __copyright__ = "Copyright (C) 2023-2025 NetInvent" 11 | 12 | 13 | ################## 14 | # CONSTANTS FILE # 15 | ################## 16 | 17 | # Interval for timeout in queue reads 18 | # The lower, the faster we get backend results, but at the expense of cpu 19 | CHECK_INTERVAL = 0.005 20 | 21 | # The lower the snappier the GUI, but also more cpu hungry 22 | # Should not be lower than CHECK_INTERVAL 23 | GUI_CHECK_INTERVAL = 0.005 24 | 25 | 26 | # Interval on which we log a status message stating we're still alive 27 | # This is useful for long running operations 28 | HEARTBEAT_INTERVAL = 3600 29 | 30 | # Arbitrary timeout for init / init checks. 31 | # If init takes more than a minute, we really have a problem in our backend 32 | FAST_COMMANDS_TIMEOUT = 180 33 | 34 | # # Wait x seconds before we actually do the upgrade so current program could quit before being erased 35 | UPGRADE_DEFER_TIME = 60 36 | 37 | # Maximum allowed time offset in seconds to allow policy operations to run 38 | MAX_ALLOWED_NTP_OFFSET = 600.0 39 | 40 | if not "BUILD_TYPE" in globals(): 41 | BUILD_TYPE = "UnknownBuildType" 42 | 43 | 44 | def set_build_type(build_type: str) -> None: 45 | global BUILD_TYPE 46 | BUILD_TYPE = build_type 47 | 48 | 49 | # Allowed server ids for upgrade 50 | ALLOWED_UPGRADE_SERVER_IDS = ("npbackup.upgrader", "npbackup.deployment_server") 51 | 52 | # Replacement string for sensitive data 53 | HIDDEN_BY_NPBACKUP = "_[o_O]_hidden_by_npbackup" 54 | -------------------------------------------------------------------------------- /npbackup/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | # Placeholder so this directory becomes a package 7 | -------------------------------------------------------------------------------- /npbackup/__version__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup" 7 | __author__ = "Orsiris de Jong" 8 | __site__ = "https://www.netperfect.fr/npbackup" 9 | __description__ = "NetPerfect Backup Client" 10 | __copyright__ = "Copyright (C) 2022-2025 NetInvent" 11 | __license__ = "GPL-3.0-only" 12 | __build__ = "2025050201" 13 | __version__ = "3.0.2" 14 | 15 | 16 | import sys 17 | import psutil 18 | from ofunctions.platform import python_arch, get_os 19 | import npbackup.__env__ 20 | from npbackup.key_management import IS_PRIV_BUILD 21 | from npbackup.core.nuitka_helper import IS_COMPILED 22 | 23 | 24 | # Python 3.7 versions are considered legacy since they don't support msgspec 25 | # msgspec is only supported on Python 3.8 64-bit and above 26 | # Since development currently follows Python 3.12, let's consider anything below 3.12 as legacy 27 | IS_LEGACY = True if (sys.version_info[1] < 12 or python_arch() == "x86") else False 28 | 29 | try: 30 | CURRENT_USER = psutil.Process().username() 31 | except Exception: 32 | CURRENT_USER = "unknown" 33 | version_dict = { 34 | "name": __intname__, 35 | "version": __version__, 36 | "build_type": npbackup.__env__.BUILD_TYPE, 37 | "audience": "private" if IS_PRIV_BUILD else "public", 38 | "os": get_os().lower(), 39 | "arch": python_arch() + ("-legacy" if IS_LEGACY else ""), 40 | "pv": sys.version_info, 41 | "comp": IS_COMPILED, 42 | "build": __build__, 43 | "copyright": __copyright__, 44 | } 45 | version_string = f"{version_dict['name']} {version_dict['version']}-{version_dict['os']}-{version_dict['build_type']}-{version_dict['arch']}-{version_dict['audience']}-{version_dict['pv'][0]}.{version_dict['pv'][1]}-{'c' if IS_COMPILED else 'i'} {version_dict['build']} - {version_dict['copyright']} running as {CURRENT_USER}" 46 | -------------------------------------------------------------------------------- /npbackup/common.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.common" 7 | __author__ = "Orsiris de Jong" 8 | __site__ = "https://www.netperfect.fr/npbackup" 9 | __description__ = "NetPerfect Backup Client" 10 | __copyright__ = "Copyright (C) 2023-2025 NetInvent" 11 | __license__ = "GPL-3.0-only" 12 | __build__ = "2023121801" 13 | 14 | 15 | from datetime import datetime, timezone 16 | from logging import getLogger 17 | import ofunctions.logger_utils 18 | 19 | 20 | logger = getLogger() 21 | 22 | 23 | def execution_logs(start_time: datetime) -> None: 24 | """ 25 | Try to know if logger.warning or worse has been called 26 | logger._cache contains a dict of values like {10: boolean, 20: boolean, 30: boolean, 40: boolean, 50: boolean} 27 | where 28 | 10 = debug, 20 = info, 30 = warning, 40 = error, 50 = critical 29 | so "if 30 in logger._cache" checks if warning has been triggered 30 | ATTENTION: logger._cache does only contain cache of current main, not modules, deprecated in favor of 31 | ofunctions.logger_utils.ContextFilterWorstLevel 32 | 33 | ATTENTION: For ofunctions.logger_utils.ContextFilterWorstLevel will only check current logger instance 34 | So using logger = getLogger("anotherinstance") will create a separate instance from the one we can inspect 35 | Makes sense ;) 36 | """ 37 | 38 | end_time = datetime.now(timezone.utc) 39 | 40 | logger_worst_level = 0 41 | for flt in logger.filters: 42 | if isinstance(flt, ofunctions.logger_utils.ContextFilterWorstLevel): 43 | logger_worst_level = flt.worst_level 44 | 45 | log_level_reached = "success" 46 | try: 47 | if logger_worst_level >= 50: 48 | log_level_reached = "critical" 49 | elif logger_worst_level >= 40: 50 | log_level_reached = "errors" 51 | elif logger_worst_level >= 30: 52 | log_level_reached = "warnings" 53 | except AttributeError as exc: 54 | logger.error(f"Cannot get worst log level reached: {exc}") 55 | logger.info( 56 | f"ExecTime = {end_time - start_time}, finished, state is: {log_level_reached}." 57 | ) 58 | # using sys.exit(code) in a atexit function will swallow the exitcode and render 0 59 | # Using sys.exit(logger.get_worst_logger_level()) is the way to go, when using ofunctions.logger_utils >= 2.4.1 60 | -------------------------------------------------------------------------------- /npbackup/core/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # Placeholder so this directory becomes a package 5 | -------------------------------------------------------------------------------- /npbackup/core/i18n_helper.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.core.i18n_helper" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2022-2025 NetInvent" 9 | __license__ = "BSD-3-Clause" 10 | __build__ = "2023032101" 11 | 12 | 13 | import os 14 | from logging import getLogger 15 | from locale import getlocale 16 | import i18n 17 | from npbackup.path_helper import BASEDIR 18 | 19 | 20 | logger = getLogger() 21 | 22 | 23 | TRANSLATIONS_DIR = os.path.join(BASEDIR, "translations") 24 | 25 | # getdefaultlocale returns a tuple like ('fr-FR', 'cp1251') 26 | # Let's only use the fr part, so other french speaking countries also have french translation 27 | _locale = os.environ.get("NPBACKUP_LOCALE", getlocale()[0]) 28 | try: 29 | _locale, _ = _locale.split("_") 30 | except (ValueError, AttributeError): 31 | try: 32 | _locale, _ = _locale.split("-") 33 | except (ValueError, AttributeError): 34 | _locale = "en" 35 | 36 | try: 37 | i18n.load_path.append(TRANSLATIONS_DIR) 38 | except OSError as exc: 39 | logger.error("Cannot load translations: {}".format(exc)) 40 | i18n.set("locale", _locale) 41 | i18n.set("fallback", "en") 42 | 43 | 44 | def _t(*args, **kwargs): 45 | try: 46 | return i18n.t(*args, **kwargs) 47 | except OSError as exc: 48 | logger.error("Translation not found in {}: {}".format(TRANSLATIONS_DIR, exc)) 49 | except TypeError as exc: 50 | logger.error("Translation failed: {}".format(exc)) 51 | logger.error("Arguments: {}".format(*args)) 52 | if len(args) > 0: 53 | return args[0] 54 | return args 55 | -------------------------------------------------------------------------------- /npbackup/core/jobs.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.gui.core.jobs" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2022-2025 NetInvent" 9 | __license__ = "GPL-3.0-only" 10 | __build__ = "2025031201" 11 | 12 | # This module helps scheduling jobs without using a daemon 13 | # We can schedule on a random percentage or a fixed interval 14 | 15 | 16 | import os 17 | from typing import Optional 18 | import tempfile 19 | from logging import getLogger 20 | from random import randint 21 | from npbackup.path_helper import CURRENT_DIR 22 | 23 | 24 | logger = getLogger() 25 | 26 | 27 | def schedule_on_interval(job_name: str, interval: int) -> bool: 28 | """ 29 | Basic counter that returns true only every X times this is called 30 | 31 | We need to make to select a write counter file that is writable 32 | So we actually test a local file and a temp file (less secure for obvious reasons, ie tmp file deletions) 33 | We just have to make sure that once we can write to one file, we stick to it unless proven otherwise 34 | 35 | The for loop logic isn't straight simple, but allows file fallback 36 | """ 37 | if not interval: 38 | logger.debug(f"No interval given for job {job_name}: {interval}") 39 | return False 40 | 41 | try: 42 | interval = int(interval) 43 | except ValueError: 44 | logger.error(f"No valid interval given for job {job_name}: {interval}") 45 | return False 46 | 47 | # file counter, local, home, or temp if not available 48 | counter_file = f"{__intname__}.{job_name}.log" 49 | 50 | def _write_count(file: str, count: int) -> bool: 51 | try: 52 | with open(file, "w", encoding="utf-8") as fpw: 53 | fpw.write(str(count)) 54 | return True 55 | except OSError: 56 | # We may not have write privileges, hence we need a backup plan 57 | return False 58 | 59 | def _get_count(file: str) -> Optional[int]: 60 | try: 61 | with open(file, "r", encoding="utf-8") as fp: 62 | count = int(fp.read()) 63 | return count 64 | except OSError as exc: 65 | # We may not have read privileges 66 | logger.error(f"Cannot read {job_name} counter file {file}: {exc}") 67 | except ValueError as exc: 68 | logger.error(f"Bogus {job_name} counter in {file}: {exc}") 69 | return None 70 | 71 | path_list = [ 72 | os.path.join(tempfile.gettempdir(), counter_file), 73 | os.path.join(CURRENT_DIR, counter_file), 74 | ] 75 | if os.name != "nt": 76 | path_list = [os.path.join("/var/log", counter_file)] + path_list 77 | else: 78 | path_list = [os.path.join(r"C:\Windows\Temp", counter_file)] + path_list 79 | 80 | for file in path_list: 81 | if not os.path.isfile(file): 82 | if _write_count(file, 1): 83 | logger.debug(f"Initial job {job_name} counter written to {file}") 84 | else: 85 | logger.debug(f"Cannot write {job_name} counter file {file}") 86 | continue 87 | count = _get_count(file) 88 | # Make sure we can write to the file before we make any assumptions 89 | result = _write_count(file, count + 1) 90 | if result: 91 | if count >= interval: 92 | # Reinitialize counter before we actually approve job run 93 | if _write_count(file, 1): 94 | logger.info( 95 | f"schedule on interval has decided {job_name} is required" 96 | ) 97 | return True 98 | break 99 | else: 100 | logger.debug(f"Cannot write {job_name} counter to {file}") 101 | continue 102 | return False 103 | 104 | 105 | def schedule_on_chance(job_name: str, chance_percent: int) -> bool: 106 | """ 107 | Randomly decide if we need to run a job according to chance_percent 108 | """ 109 | if not chance_percent: 110 | logger.debug(f"No chance percent given for job {job_name}: {chance_percent}") 111 | return False 112 | try: 113 | chance_percent = int(chance_percent) 114 | except ValueError: 115 | logger.error( 116 | f"No valid chance percent given for job {job_name}: {chance_percent}" 117 | ) 118 | return False 119 | if randint(1, 100) <= chance_percent: 120 | logger.debug(f"schedule on chance has decided {job_name} is required") 121 | return True 122 | return False 123 | 124 | 125 | def schedule_on_chance_or_interval( 126 | job_name: str, chance_percent: int, interval: int 127 | ) -> bool: 128 | """ 129 | Decide if we will run a job according to chance_percent or interval 130 | """ 131 | if schedule_on_chance(job_name, chance_percent) or schedule_on_interval( 132 | job_name, interval 133 | ): 134 | return True 135 | return False 136 | -------------------------------------------------------------------------------- /npbackup/core/nuitka_helper.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.core.nuitka_helper" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2022-2025 NetInvent" 9 | __license__ = "BSD-3-Clause" 10 | __build__ = "2023040401" 11 | 12 | 13 | IS_COMPILED = "__compiled__" in globals() 14 | -------------------------------------------------------------------------------- /npbackup/core/restic_source_binary.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.gui.core.restic_source_binary" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2022-2025 NetInvent" 9 | __license__ = "GPL-3.0-only" 10 | __build__ = "2025021401" 11 | 12 | 13 | import os 14 | import sys 15 | import glob 16 | from logging import getLogger 17 | from npbackup.__version__ import IS_LEGACY 18 | from npbackup.path_helper import BASEDIR 19 | 20 | logger = getLogger() 21 | 22 | RESTIC_SOURCE_FILES_DIR = os.path.join(BASEDIR, os.pardir, "RESTIC_SOURCE_FILES") 23 | 24 | 25 | def get_restic_internal_binary(arch: str) -> str: 26 | binary = None 27 | if os.path.isdir(RESTIC_SOURCE_FILES_DIR): 28 | if os.name == "nt": 29 | if IS_LEGACY or "legacy" in "arch": 30 | # Last compatible restic binary for Windows 7, see https://github.com/restic/restic/issues/5065 31 | # We build a legacy version of restic for windows 7 and Server 2008R2 32 | logger.info( 33 | "Dealing with special case for Windows 7 32 bits that doesn't run with restic >= 0.16.2" 34 | ) 35 | if arch == "x86": 36 | binary = "restic_*_windows_legacy_386.exe" 37 | else: 38 | binary = "restic_*_windows_legacy_amd64.exe" 39 | elif arch == "x86": 40 | binary = "restic_*_windows_386.exe" 41 | else: 42 | binary = "restic_*_windows_amd64.exe" 43 | else: 44 | # We don't have restic legacy builds for unixes 45 | # so we can drop the -legacy suffix 46 | arch = arch.replace("-legacy", "") 47 | if sys.platform.lower() == "darwin": 48 | if arch == "arm64": 49 | binary = "restic_*_darwin_arm64" 50 | else: 51 | binary = "restic_*_darwin_amd64" 52 | else: 53 | if arch == "arm": 54 | binary = "restic_*_linux_arm" 55 | elif arch == "arm64": 56 | binary = "restic_*_linux_arm64" 57 | elif arch == "x64": 58 | binary = "restic_*_linux_amd64" 59 | else: 60 | binary = "restic_*_linux_386" 61 | if binary: 62 | guessed_path = glob.glob(os.path.join(RESTIC_SOURCE_FILES_DIR, binary)) 63 | if guessed_path: 64 | # Take glob results reversed so we get newer version 65 | # Does not always compute, but is g00denough(TM) for our dev 66 | return guessed_path[-1] 67 | return None 68 | -------------------------------------------------------------------------------- /npbackup/core/upgrade_runner.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.gui.core.upgrade_runner" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2022-2025 NetInvent" 9 | __license__ = "GPL-3.0-only" 10 | __build__ = "2025030701" 11 | 12 | 13 | from logging import getLogger 14 | from random import randint 15 | from npbackup.upgrade_client.upgrader import auto_upgrader, _check_new_version 16 | import npbackup.configuration 17 | 18 | 19 | logger = getLogger() 20 | 21 | 22 | def check_new_version(full_config: dict) -> bool: 23 | upgrade_url = full_config.g("global_options.auto_upgrade_server_url") 24 | username = full_config.g("global_options.auto_upgrade_server_username") 25 | password = full_config.g("global_options.auto_upgrade_server_password") 26 | if not upgrade_url or not username or not password: 27 | logger.warning( 28 | "Missing auto upgrade info, cannot check new version for auto upgrade" 29 | ) 30 | return None 31 | else: 32 | return _check_new_version(upgrade_url, username, password) 33 | 34 | 35 | def run_upgrade( 36 | config_file: str, full_config: dict, ignore_errors: bool = False 37 | ) -> bool: 38 | upgrade_url = full_config.g("global_options.auto_upgrade_server_url") 39 | username = full_config.g("global_options.auto_upgrade_server_username") 40 | password = full_config.g("global_options.auto_upgrade_server_password") 41 | if not upgrade_url or not username or not password: 42 | logger.warning("Missing auto upgrade info, cannot launch auto upgrade") 43 | return False 44 | 45 | evaluated_full_config = npbackup.configuration.evaluate_variables( 46 | full_config, full_config 47 | ) 48 | auto_upgrade_host_identity = evaluated_full_config.g( 49 | "global_options.auto_upgrade_host_identity" 50 | ) 51 | group = evaluated_full_config.g("global_options.auto_upgrade_group") 52 | 53 | result = auto_upgrader( 54 | config_file=config_file, 55 | upgrade_url=upgrade_url, 56 | username=username, 57 | password=password, 58 | auto_upgrade_host_identity=auto_upgrade_host_identity, 59 | group=group, 60 | ignore_errors=ignore_errors, 61 | ) 62 | return result 63 | -------------------------------------------------------------------------------- /npbackup/gui/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # Placeholder so this directory becomes a package 5 | -------------------------------------------------------------------------------- /npbackup/gui/handle_window.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.gui.windows_gui_helper" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2022-2025 NetInvent" 9 | __license__ = "GPL-3.0-only" 10 | __build__ = "2023020601" 11 | 12 | 13 | import sys 14 | import os 15 | 16 | 17 | def handle_current_window(action: str = "minimize") -> None: 18 | """ 19 | Minimizes / hides current commandline window in GUI mode 20 | This helps when Nuitka cmdline hide action does not work 21 | """ 22 | if os.name == "nt": 23 | # pylint: disable=E0401 (import-error) 24 | import win32gui 25 | 26 | # pylint: disable=E0401 (import-error) 27 | import win32con 28 | 29 | current_executable = os.path.abspath(sys.argv[0]) 30 | # console window will have the name of current executable 31 | hwndMain = win32gui.FindWindow(None, current_executable) 32 | if hwndMain: 33 | if action == "minimize": 34 | win32gui.ShowWindow(hwndMain, win32con.SW_MINIMIZE) 35 | elif action == "hide": 36 | win32gui.ShowWindow(hwndMain, win32con.SW_HIDE) 37 | else: 38 | raise ValueError( 39 | f"Bad action parameter for handling current window: {action}" 40 | ) 41 | -------------------------------------------------------------------------------- /npbackup/gui/viewer.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.gui.viewer" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2022-2025 NetInvent" 9 | __license__ = "GPL-3.0-only" 10 | 11 | 12 | from npbackup.gui.__main__ import main_gui 13 | 14 | 15 | def viewer_gui(): 16 | main_gui(viewer_mode=True) 17 | 18 | 19 | if __name__ == "__main__": 20 | viewer_gui() 21 | -------------------------------------------------------------------------------- /npbackup/key_management.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.key_management" 7 | 8 | 9 | import sys 10 | import os 11 | from logging import getLogger 12 | from command_runner import command_runner 13 | from cryptidy.symmetric_encryption import generate_key 14 | from npbackup.obfuscation import obfuscation 15 | 16 | sys.path.insert(0, os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))) 17 | 18 | 19 | logger = getLogger() 20 | 21 | 22 | # Try to import a private key, if not available, fallback to the default key 23 | try: 24 | from PRIVATE._private_secret_keys import AES_KEY 25 | from PRIVATE._obfuscation import obfuscation 26 | 27 | AES_KEY = obfuscation(AES_KEY) 28 | IS_PRIV_BUILD = True 29 | try: 30 | from PRIVATE._private_secret_keys import EARLIER_AES_KEY 31 | 32 | EARLIER_AES_KEY = obfuscation(EARLIER_AES_KEY) 33 | except ImportError: 34 | EARLIER_AES_KEY = None 35 | except ImportError: 36 | # If no private keys are used, then let's use the public ones 37 | try: 38 | from npbackup.secret_keys import AES_KEY 39 | from npbackup.obfuscation import obfuscation 40 | 41 | AES_KEY = obfuscation(AES_KEY) 42 | IS_PRIV_BUILD = False 43 | try: 44 | from npbackup.secret_keys import EARLIER_AES_KEY 45 | except ImportError: 46 | EARLIER_AES_KEY = None 47 | except ImportError: 48 | print("No secret_keys file. Please read documentation.") 49 | sys.exit(1) 50 | 51 | 52 | def get_aes_key(): 53 | """ 54 | Get encryption key from environment variable or file 55 | """ 56 | key = None 57 | 58 | try: 59 | key_location = os.environ.get("NPBACKUP_KEY_LOCATION", None) 60 | if key_location and os.path.isfile(key_location): 61 | try: 62 | with open(key_location, "rb") as key_file: 63 | key = key_file.read() 64 | msg = f"Encryption key file read" 65 | except OSError as exc: 66 | msg = f"Cannot read encryption key file: {exc}" 67 | return False, msg 68 | else: 69 | key_command = os.environ.get("NPBACKUP_KEY_COMMAND", None) 70 | if key_command: 71 | exit_code, output = command_runner( 72 | key_command, encoding=False, shell=True 73 | ) 74 | if exit_code != 0: 75 | msg = f"Cannot run encryption key command: {output}" 76 | return False, msg 77 | key = bytes(output) 78 | msg = f"Encryption key read from command" 79 | except Exception as exc: 80 | msg = f"Error reading encryption key: {exc}" 81 | return False, msg 82 | if key: 83 | return obfuscation(key), msg 84 | return None, "" 85 | 86 | 87 | def create_key_file(key_location: str): 88 | try: 89 | with open(key_location, "wb") as key_file: 90 | key_file.write(obfuscation(generate_key())) 91 | logger.info(f"Encryption key file created at {key_location}") 92 | return True 93 | except OSError as exc: 94 | logger.critical(f"Cannot create encryption key file: {exc}") 95 | return False 96 | -------------------------------------------------------------------------------- /npbackup/obfuscation.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.obfuscation" 7 | 8 | 9 | # NPF-SEC-00011: Default AES key obfuscation 10 | 11 | 12 | def obfuscation(key: bytes) -> bytes: 13 | """ 14 | Symmetric obfuscation of bytes 15 | """ 16 | if key: 17 | keyword = b"/*NPBackup 2024*/" 18 | key_length = len(keyword) 19 | return bytes(c ^ keyword[i % key_length] for i, c in enumerate(key)) 20 | return key 21 | -------------------------------------------------------------------------------- /npbackup/path_helper.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.path_helper" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2023-2025 NetInvent" 9 | __license__ = "BSD-3-Clause" 10 | __build__ = "2024091701" 11 | 12 | 13 | # This file must exist at the root of the package, for basedir to be detected as root 14 | 15 | 16 | import sys 17 | import os 18 | 19 | 20 | # This is the path to a python script, a standalone or a onefile nuitka generated binary 21 | # When running python interpreter without any script, sys.argv is empty hence CURRENT_EXECUTABLE would become current directory 22 | CURRENT_EXECUTABLE = os.path.abspath(sys.argv[0]) 23 | CURRENT_DIR = os.path.dirname(CURRENT_EXECUTABLE) 24 | # When run with nuitka onefile, this will be the temp directory, else, this will be the path to current file 25 | BASEDIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) 26 | -------------------------------------------------------------------------------- /npbackup/requirements-compilation.txt: -------------------------------------------------------------------------------- 1 | nuitka>=2.4.8 2 | windows_tools.signtool>=0.5.0 -------------------------------------------------------------------------------- /npbackup/requirements-win32.txt: -------------------------------------------------------------------------------- 1 | command_runner>=1.7.0 2 | cryptidy>=1.2.4 3 | python-dateutil 4 | ofunctions.logger_utils>=2.4.2 5 | ofunctions.misc>=1.8.0 6 | ofunctions.process>=2.1.0 7 | ofunctions.threading>=2.2.0 8 | ofunctions.platform>=1.5.1 9 | ofunctions.random 10 | ofunctions.requestor>=1.2.2 11 | python-pidfile>=3.0.0 12 | # pysimplegui 5 has gone commercial, let's switch to freesimplegui 13 | # keep in mind that freesimplegui might higher required python version in the future 14 | freesimplegui==5.2.0 15 | requests 16 | ruamel.yaml 17 | psutil 18 | pyyaml # Required for python-i18n / i18nice which does not work with ruamel.yaml 19 | # python-i18n 20 | # Replaced python-i18n with a fork that prevents boolean keys from being interpreted 21 | # Also fixes some portability issues (still boolean key name issues) encountered when compiling on Centos 7 and executing on Almalinux 9 22 | # python-i18n@https://github.com/Krutyi-4el/python-i18n/archive/master.zip 23 | # python-i18n @ git+https://github.com/Krutyi-4el/python-i18n.git@0.6.0#8999a0d380be8a08beed785e46fbb31dfc03c605 24 | # Since PyPI and twine don't allow usage of direct references (git addresses) 25 | # we'll use an inline version for now 26 | i18nice>=0.6.2 27 | packaging 28 | pywin32; platform_system == "Windows" 29 | imageio; platform_system == "Darwin" 30 | ntplib>=0.4.0 31 | # msgspec needs python 3.8+ and is not compatible with win 32-bit 32 | 33 | -------------------------------------------------------------------------------- /npbackup/requirements.txt: -------------------------------------------------------------------------------- 1 | command_runner>=1.7.3 2 | cryptidy>=1.2.4 3 | python-dateutil 4 | ofunctions.logger_utils>=2.4.2 5 | ofunctions.misc>=1.8.0 6 | ofunctions.process>=2.1.0 7 | ofunctions.threading>=2.2.0 8 | ofunctions.platform>=1.5.1 9 | ofunctions.random 10 | ofunctions.requestor>=1.2.2 11 | python-pidfile>=3.0.0 12 | # pysimplegui 5 has gone commercial, let's switch to freesimplegui 13 | # keep in mind that freesimplegui might higher required python version in the future 14 | freesimplegui==5.2.0 15 | requests 16 | ruamel.yaml 17 | psutil 18 | pyyaml # Required for python-i18n / i18nice which does not work with ruamel.yaml 19 | # python-i18n 20 | # Replaced python-i18n with a fork that prevents boolean keys from being interpreted 21 | # Also fixes some portability issues (still boolean key name issues) encountered when compiling on Centos 7 and executing on Almalinux 9 22 | # python-i18n@https://github.com/Krutyi-4el/python-i18n/archive/master.zip 23 | # python-i18n @ git+https://github.com/Krutyi-4el/python-i18n.git@0.6.0#8999a0d380be8a08beed785e46fbb31dfc03c605 24 | # Since PyPI and twine don't allow usage of direct references (git addresses) 25 | # we'll use an inline version for now 26 | i18nice>=0.6.2 27 | packaging 28 | pywin32; platform_system == "Windows" 29 | imageio; platform_system == "Darwin" 30 | ntplib>=0.4.0 31 | # msgspec needs python 3.8+ (see pep-0496 for environment markers) 32 | msgspec; python_version >= "3.8" 33 | -------------------------------------------------------------------------------- /npbackup/restic_metrics/requirements.txt: -------------------------------------------------------------------------------- 1 | ofunctions.misc>=1.5.2 -------------------------------------------------------------------------------- /npbackup/restic_wrapper/schema.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.restic_wrapper.schema" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2024 NetInvent" 9 | __license__ = "GPL-3.0-only" 10 | __description__ = "Restic json output schemas" 11 | 12 | 13 | from typing import Optional 14 | from datetime import datetime 15 | from enum import Enum 16 | 17 | try: 18 | from msgspec import Struct 19 | 20 | HAVE_MSGSPEC = True 21 | except ImportError: 22 | 23 | class Struct: 24 | def __init_subclass__(self, *args, **kwargs): 25 | pass 26 | 27 | class StrEnum: 28 | pass 29 | 30 | HAVE_MSGSPEC = False 31 | 32 | 33 | class LsNodeType(str, Enum): 34 | FILE = "file" 35 | DIR = "dir" 36 | SYMLINK = "symlink" 37 | IRREGULAR = "irregular" 38 | 39 | 40 | class LsNode(Struct, omit_defaults=True): 41 | """ 42 | restic ls outputs lines of 43 | {"name": "b458b848.2024-04-28-13h07.gz", "type": "file", "path": "/path/b458b848.2024-04-28-13h07.gz", "uid": 0, "gid": 0, "size": 82638431, "mode": 438, "permissions": "-rw-rw-rw-", "mtime": "2024-04-29T10:32:18+02:00", "atime": "2024-04-29T10:32:18+02:00", "ctime": "2024-04-29T10:32:18+02:00", "message_type": "node", "struct_type": "node"} 44 | # In order to save some memory in GUI, let's drop unused data 45 | """ 46 | 47 | # name: str # We don't need name, we have path from which we extract name, which is more memory efficient 48 | type: LsNodeType 49 | path: str 50 | mtime: datetime 51 | size: Optional[int] = None 52 | -------------------------------------------------------------------------------- /npbackup/runner_interface.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.runner_interface" 7 | __author__ = "Orsiris de Jong" 8 | __site__ = "https://www.netperfect.fr/npbackup" 9 | __description__ = "NetPerfect Backup Client" 10 | __copyright__ = "Copyright (C) 2022-2025 NetInvent" 11 | __license__ = "GPL-3.0-only" 12 | __build__ = "2024103001" 13 | 14 | 15 | import sys 16 | from logging import getLogger 17 | 18 | try: 19 | import msgspec.json 20 | 21 | HAVE_MSGSPEC = True 22 | json = None # linter E0601 fix 23 | except ImportError: 24 | import json 25 | 26 | HAVE_MSGSPEC = False 27 | import datetime 28 | from npbackup.core.runner import NPBackupRunner 29 | 30 | 31 | logger = getLogger() 32 | 33 | 34 | def serialize_datetime(obj): 35 | """ 36 | By default, datetime objects aren't serialisable to json directly 37 | Here's a quick converter from https://www.geeksforgeeks.org/how-to-fix-datetime-datetime-not-json-serializable-in-python/ 38 | """ 39 | if isinstance(obj, datetime.datetime): 40 | return obj.isoformat() 41 | raise TypeError("Type not serializable") 42 | 43 | 44 | def entrypoint(*args, **kwargs): 45 | repo_config = kwargs.pop("repo_config", None) 46 | json_output = kwargs.pop("json_output") 47 | operation = kwargs.pop("operation") 48 | backend_binary = kwargs.pop("backend_binary", None) 49 | 50 | npbackup_runner = NPBackupRunner() 51 | if repo_config: 52 | npbackup_runner.repo_config = repo_config 53 | npbackup_runner.dry_run = kwargs.pop("dry_run") 54 | npbackup_runner.verbose = kwargs.pop("verbose") 55 | npbackup_runner.live_output = not json_output 56 | npbackup_runner.json_output = json_output 57 | npbackup_runner.no_cache = kwargs.pop("no_cache", False) 58 | npbackup_runner.no_lock = kwargs.pop("no_lock", False) 59 | if backend_binary: 60 | npbackup_runner.binary = backend_binary 61 | result = npbackup_runner.__getattribute__(operation)( 62 | **kwargs.pop("op_args"), __no_threads=True 63 | ) 64 | if not json_output: 65 | if not isinstance(result, bool): 66 | # We need to temporarily remove the stdout handler 67 | # Since we already get live output from the runner 68 | # Unless operation is "ls", because it's too slow for command_runner poller method that allows live_output 69 | # But we still need to log the result to our logfile 70 | if not operation == "ls": 71 | handler = None 72 | for handler in logger.handlers: 73 | if handler.stream == sys.stdout: 74 | logger.removeHandler(handler) 75 | break 76 | logger.info(f"\n{result}") 77 | if not operation == "ls" and handler: 78 | logger.addHandler(handler) 79 | if result: 80 | logger.info("Operation finished") 81 | else: 82 | logger.error("Operation finished") 83 | else: 84 | if HAVE_MSGSPEC: 85 | print(msgspec.json.encode(result).decode("utf-8", errors="ignore")) 86 | else: 87 | print(json.dumps(result, default=serialize_datetime)) 88 | sys.exit(0) 89 | -------------------------------------------------------------------------------- /npbackup/secret_keys.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.secret_keys" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2023-2025 NetInvent" 9 | __license__ = "GPL-3.0-only" 10 | __build__ = "2024050901" 11 | 12 | 13 | # Encryption key to keep repo settings safe in plain text yaml config file 14 | 15 | # This is the default key that comes with NPBackup.. You should change it (and keep a backup copy in case you need to decrypt a config file data) 16 | # You can overwrite this by copying this file to `../PRIVATE/_private_secret_keys.py` and generating a new key 17 | # Obtain a new key with: 18 | # python3 -c "from cryptidy import symmetric_encryption as s; print(s.generate_key())" 19 | # You may also create a new keyfile via 20 | # npbackup-cli --create-key keyfile.key 21 | # Given keyfile can then be loaded via environment variables, see documentation for more 22 | 23 | AES_KEY = b"\xc3T\xdci\xe3[s\x87o\x96\x8f\xe5\xee.>\xf1,\x94\x8d\xfe\x0f\xea\x11\x05 \xa0\xe9S\xcf\x82\xad|" 24 | 25 | """ 26 | If someday we need to change the AES_KEY, copy it's content to EARLIER_AES_KEY and generate a new one 27 | Keeping EARLIER_AES_KEY allows to migrate from old configuration files to new ones 28 | """ 29 | EARLIER_AES_KEY = b"\x9e\xbck\xe4\xc5nkT\x1e\xbf\xb5o\x06\xd3\xc6(\x0e:'i\x1bT\xb3\xf0\x1aC e\x9bd\xa5\xc6" 30 | -------------------------------------------------------------------------------- /npbackup/translations/config_gui.en.yml: -------------------------------------------------------------------------------- 1 | en: 2 | # tabs 3 | backup: Backup 4 | backup_destination: Destination 5 | exclusions: Exclusions 6 | pre_post: Pre/Post exec 7 | 8 | encrypted_data: Encrypted_Data 9 | compression: Compression 10 | backup_paths: Backup paths 11 | use_fs_snapshot: Use VSS snapshots 12 | ignore_cloud_files: Ignore in-cloud files 13 | windows_only: Windows only 14 | exclude_patterns: Exclude patterns 15 | exclude_files: Files containing exclude patterns 16 | excludes_case_ignore: Ignore case for excludes patterns/files 17 | exclude_files_larger_than: Exclude files larger than 18 | windows_always: always enabled for Windows 19 | exclude_cache_dirs: Exclude cache dirs 20 | one_file_system: Do not follow mountpoints 21 | minimum_backup_size_error: Minimum size under which backup is considered failed 22 | pre_exec_commands: Pre-exec commands 23 | maximum_exec_time: Maximum exec time 24 | exec_failure_is_fatal: Execution failure is fatal 25 | post_exec_commands: Post-exec commands 26 | execute_even_on_backup_error: Execute even if backup failed 27 | tags: Tags 28 | one_per_line: one per line 29 | backup_priority: Backup priority 30 | additional_parameters: Additional parameters 31 | additional_backup_only_parameters: Additional backup only parameters 32 | additional_restore_only_parameters: Additional restore only parameters 33 | 34 | minimum_backup_age: Minimum delay between two backups 35 | backup_repo_uri: backup repo URI / path 36 | backup_repo_password: Backup repo encryption password 37 | backup_repo_password_command: Command that returns backup repo encryption password 38 | upload_speed: Upload speed limit (KB/s) 39 | download_speed: Download speed limit (KB/s) 40 | backend_connections: Simultaneous repo connections 41 | 42 | prometheus_config: Prometheus configuration 43 | available_variables: Available variables ${HOSTNAME}, ${BACKUP_JOB}, ${MACHINE_ID}, ${MACHINE_GROUP}, ${RANDOM}[n] 44 | available_variables_id: Available variables ${HOSTNAME}, ${RANDOM}[n] where n is the number of random chars 45 | enable_prometheus: Enable prometheus metrics 46 | job_name: Job name (backup_job) 47 | metrics_destination: Metrics destination (Push URI / file) 48 | no_cert_verify: Do not verify SSL certificate 49 | metrics_username: HTTP metrics username 50 | metrics_password: HTTP metrics password 51 | instance: Prometheus instance 52 | additional_labels: Additional labels 53 | 54 | no_config_available: No configuration file found. Please use --config-file "path" to specify one or copy a config file next to the NPBackup binary 55 | create_new_config: Would you like to create a new configuration ? 56 | saved_initial_config: If you saved your configuration, you may now reload this program 57 | bogus_config_file: Bogus configuration file found 58 | 59 | encrypted_env_variables: Encrypted environment variables 60 | env_variables: Environment variables 61 | 62 | no_runner: Cannot connect to backend. Please see logs 63 | runner_not_configured: Backend not configured properly. Please see logs 64 | no_binary: Cannot find backup backend. Please install restic binary from restic.net 65 | key_error: Key from configuration has no match in GUI 66 | delete_bad_keys: Do you want to delete the bogus keys ? Note that this only affects current object. If key is inherited, you will need to load corresponding group. Concerned keys in current object 67 | 68 | configuration_saved: Configuration saved 69 | cannot_save_configuration: Could not save configuration. See logs for further info 70 | repo_uri_cannot_be_empty: Repo URI / path cannot be empty 71 | set_manager_password: Manager password 72 | wrong_password: Wrong password 73 | remove_password: Remove password 74 | 75 | auto_upgrade: Auto upgrade 76 | auto_upgrade_server_url: Server URL 77 | auto_upgrade_server_username: Server username 78 | auto_upgrade_server_password: Server password 79 | auto_upgrade_percent_chance: Auto upgrade percent chance (%%) 80 | auto_upgrade_interval: Auto upgrade runs interval 81 | auto_upgrade_launch: Launch auto upgrade 82 | auto_upgrade_will_quit: Warning, launching an upgrade procedure will quit this program without notice. You will have to wait 5 minutes before launching it again for the upgrade to complete 83 | auto_upgrade_failed: Auto upgrade procedure failed, see logs for further details 84 | auto_upgrade_disabled: Auto upgrade is disabled or server is not reachable 85 | 86 | create_backup_scheduled_task_every: Create scheduled backup task every 87 | create_backup_scheduled_task_at: Create scheduled backup task every day at 88 | create_housekeeping_scheduled_task_at: Create housekeeping scheduled every day at 89 | scheduled_task_explanation: Task can run at a given time to run a backup which is great to make server backups, or run every x minutes, but only run actual backup when more than maximum_backup_age minutes was reached, which is the best way to backup laptops which have flexible power on hours. 90 | scheduled_task_creation_success: Scheduled task created successfully 91 | scheduled_task_creation_failure: Scheduled task could not be created. See logs for further info 92 | 93 | machine_identification: Machine identification 94 | machine_id: Machine identifier 95 | machine_group: Machine group 96 | 97 | show_decrypted: Show sensitive data 98 | no_manager_password_defined: No manager password defined, cannot show unencrypted. If you just set one, you need to save the configuration before you can use it 99 | 100 | # compression 101 | auto: Automatic 102 | max: Maximum 103 | off: Disabled 104 | 105 | # priorities 106 | low: Low 107 | normal: Normal 108 | high: High 109 | 110 | # source types 111 | source_type: Sources type 112 | folder_list: Folder / file list 113 | files_from: From file 114 | files_from_verbatim: From verbatim 115 | files_from_raw: From raw 116 | stdin_from_command: Standard input from command 117 | stdin_filename: Optional filename for stdin backed up data 118 | 119 | # retention policy 120 | retention_policy: Retention policy 121 | keep: Keep 122 | last: last snapshots 123 | hourly: hourly snapshots 124 | daily: daily snapshots 125 | weekly: weekly snapshots 126 | monthly: monthly snapshots 127 | yearly: yearly snapshots 128 | keep_within: Keep snapshots within time period relative to current snapshot 129 | keep_tags: Keep snapshots with the following tags 130 | post_backup_housekeeping_percent_chance: Post backup housekeeping run chance (%%) 131 | post_backup_housekeeping_percent_chance_explanation: Randomize housekeeping runs after backup (0-100%%, 0 = never, 100 = always) 132 | post_backup_housekeeping_interval: Post backup housekeeping interval 133 | post_backup_housekeeping_interval_explanation: Interval in number of runs between housekeeping runs 134 | optional_ntp_server_uri: Optional NTP server URI 135 | prune_max_unused: Prune max unused data 136 | prune_max_unused_explanation: Maximum percentage or bytes of unused data to keep in when pruning with maximum parameter 137 | prune_max_repack_size: Prune max repack size 138 | prune_max_repack_size_explanation: Maximum size of repacks when pruning (limits needed storage size for prune operation) 139 | # repo / group managmeent 140 | repo_group: Repo group 141 | group_inherited: Group inherited 142 | repo_group_config: Repos and groups configuration 143 | global_config: Global config 144 | select_object: Select configuration object 145 | add_object: Add another repo or group 146 | delete_object: Delete selected repo or group 147 | are_you_sure_to_delete: Are you sure you want to delete 148 | no_object_to_delete: No object to delete 149 | repo_already_exists: Repo already exists 150 | group_already_exists: Group already exists 151 | cannot_remove_group_inherited_settings: Cannot remove group inherited settings. Please remove directly in group configuration 152 | object_name_cannot_be_empty: Object name cannot be empty and may not contain dots 153 | object_name_cannot_be_all: Object name cannot be '__all__' which is a reserved name 154 | cannot_delete_default_repo: Cannot delete default repo 155 | cannot_delete_default_group: Cannot delete default group 156 | 157 | # permissions 158 | set_permissions: Set permissions and password 159 | permissions_only_for_repos: Permissions can only be applied for repos 160 | permissions: Permissions 161 | backup_perms: Backup only 162 | restore_perms: Backup, verify, recover and restore 163 | restore_only_perms: Restore only 164 | full_perms: Full permissions 165 | setting_permissions_requires_manager_password: Setting permissions requires manager password 166 | manager_password_too_simple: Manager password needs at least 8 uppercase, lowercase and digits characters 167 | current_permissions: Current permissions (no inheritance) 168 | manager_password_set: Manager password initialized (no inheritance) 169 | 170 | unknown_error_see_logs: Unknown error, please check logs 171 | 172 | enter_tag: Enter tag 173 | enter_pattern: Enter pattern 174 | enter_command: Enter command 175 | enter_var_name: Enter variable name 176 | enter_var_value: Enter variable value 177 | enter_label_name: Enter label name 178 | enter_label_value: Enter label value 179 | enter_labvel: Enter label 180 | 181 | suggested_encrypted_env_variables: Suggested encrypted environment variables 182 | 183 | policiy_group_by: Apply retention policy by grouping snapshots 184 | group_by_host: Group by host 185 | group_by_paths: Group by paths 186 | group_by_tags: Group by tags 187 | policiy_group_by_explanation: If none are chosen, snapshots will be grouped by host and paths 188 | 189 | add_identity: Add Cloud identities 190 | value_cannot_be_empty: Value cannot be empty 191 | repo_uri_cloud_hint: Cloud repo URI requires to set encrypted environment variables (see environment tab) -------------------------------------------------------------------------------- /npbackup/translations/config_gui.fr.yml: -------------------------------------------------------------------------------- 1 | fr: 2 | # tabs 3 | backup: Sauvegarde 4 | backup_destination: Destination 5 | exclusions: Exclusions 6 | pre_post: Pré/Post exec 7 | 8 | 9 | encrypted_data: Donnée_Chiffrée 10 | compression: Compression 11 | backup_paths: Chemins à sauvegarder 12 | use_fs_snapshot: Utiliser les instantanés VSS 13 | ignore_cloud_files: Exclure le fichiers dans le cloud 14 | windows_only: Windows seulement 15 | exclude_patterns: Patterns d'exclusion 16 | exclude_files: Fichiers contenant des patterns d'exclusions 17 | exclude_files_larger_than: Exclure les fichiers plus grands que 18 | excludes_case_ignore: Ignorer la casse des exclusions patterns/fichiers 19 | windows_always: toujours actif pour Windows 20 | exclude_cache_dirs: Exclure dossiers cache 21 | one_file_system: Ne pas suivre les points de montage 22 | minimum_backup_size_error: Taille minimale en dessous de laquelle la sauvegarde est considérée échouée 23 | pre_exec_commands: Commandes pré-sauvegarde 24 | maximum_exec_time: Temps maximal d'execution 25 | exec_failure_is_fatal: L'échec d'execution est fatal 26 | post_exec_commands: Commandes post-sauvegarde 27 | execute_even_on_backup_error: Executer même si la sauvegarde a échouée 28 | tags: Tags 29 | one_per_line: un par ligne 30 | backup_priority: Priorité de sauvegarde 31 | additional_parameters: Paramètres supplémentaires 32 | additional_backup_only_parameters: Paramètres supplémentaires de sauvegarde 33 | additional_restore_only_parameters: Paramètres supplémentaires de restauration 34 | 35 | minimum_backup_age: Délai minimal entre deux sauvegardes 36 | backup_repo_uri: URI / chemin local dépot de sauvegarde 37 | backup_repo_password: Mot de passe (chiffrement) dépot de sauvegarde 38 | backup_repo_password_command: Commande qui retourne le mot de passe de chiffrement dépot 39 | upload_speed: Vitesse limite de téléversement (KB/s) 40 | download_speed: Vitesse limite de téléchargement (KB/s) 41 | backend_connections: Connexions simultanées au dépot 42 | 43 | prometheus_config: Configuration prometheus 44 | available_variables: Variables disponibles ${HOSTNAME}, ${BACKUP_JOB}, ${MACHINE_ID}, ${MACHINE_GROUP}, ${RANDOM}[n] 45 | available_variables_id: Variables disponibles ${HOSTNAME}, ${RANDOM}[n] où n est le nombre de caractères aléatoires 46 | enable_prometheus: Activer les métriques prometheus 47 | job_name: Nom du travail (backup_job) 48 | metrics_destination: Destination métriques (URI Push / fichier) 49 | no_cert_verify: Ne pas vérifier le certificat SSL 50 | metrics_username: Nom d'utilisateur métriques HTTP 51 | metrics_password: Mot de passe métriques HTTP 52 | instance: Instance Prometheus 53 | additional_labels: Etiquettes supplémentaires 54 | 55 | no_config_available: Aucun fichier de configuration trouvé. Merci d'utiliser --config-file "chemin" pour spécifier un fichier, ou copier un fichier de configuration a côté du binaire NPBackup. 56 | create_new_config: Souhaitez-vous créer une nouvelle configuration ? 57 | saved_initial_config: Si vous avez enregistré une configuration, vous pouvez à présent recharger le programme. 58 | bogus_config_file: Fichier de configuration érroné 59 | 60 | encrypted_env_variables: Variables d'envrionnement chiffrées 61 | env_variables: Variables d'environnement 62 | 63 | no_runner: Impossible de se connecter au backend. Verifier les logs 64 | runner_not_configured: Backend non configuré proprement. Verifier les logs 65 | no_binary: Impossible de trouver le coeur de sauvegarde. Merci d'installer le binaire restic depuis restic.net 66 | key_error: Un entrée de la configuration n'a pas d'équivalent dans l'interface 67 | delete_bad_keys: Souhaitez-vous supprimer les entrées non conformes ? Notez que seul l'objet courant sera affecté. Si l'entrée est un héritage, il faudra charger le groupe correspondant. Entrées concernées dans l'objet courant 68 | 69 | configuration_saved: Configuration sauvegardée 70 | cannot_save_configuration: Impossible d'enregistrer la configuration. Veuillez consulter les journaux pour plus de détails 71 | repo_uri_cannot_be_empty: L'URI / chemin local dépot de sauvegarde ne peut-être vide 72 | set_manager_password: Mot de passe gestionnaire 73 | wrong_password: Mot de passe érroné 74 | remove_password: Supprimer le mot de passe 75 | 76 | auto_upgrade: Mise à niveau 77 | auto_upgrade_server_url: Serveur de mise à niveau 78 | auto_upgrade_server_username: Nom d'utilisateur serveur 79 | auto_upgrade_server_password: Mot de passe serveur 80 | auto_upgrade_percent_chance: Chance de mise à niveau automatique (%%) 81 | auto_upgrade_interval: Intervalle de mise à niveau 82 | auto_upgrade_launch: Lancer une mise à niveau 83 | auto_upgrade_will_quit: Attnetion, la procédure de mise à niveau va quitter ce programme sans notification. Vous devrez attendre 5 minutes pour laisser la procédure se terminer avant de relancer le programme 84 | auto_upgrade_failed: Procédure de mise à niveau échouée, veuillez consulter les journaux pour plus de détails 85 | auto_upgrade_disabled: Mise à niveau automatique désactivée ou serveur injoignable 86 | 87 | create_backup_scheduled_task_every: Créer une tâche planifiée de sauvegarde toutes les 88 | create_backup_scheduled_task_at: Créer une tâche planifiée de sauvegarde tous les jours à 89 | create_housekeeping_scheduled_task_at: Créer une tâche planifiée de maintenance tous les jours à 90 | scheduled_task_explanation: Les tâches planifiées peuvent être crées à heure fixe pour réaliser des sauvegardes serveur, ou toutes les x minutes, auquel cas la sauvegarde aura lieue uniquement si le temps minimal entre deux sauvegardes est dépassé, ce qui est adapté aux ordinateurs portables qui ont des heures de travail flexibles. 91 | scheduled_task_creation_success: Tâche planifiée crée avec succès 92 | scheduled_task_creation_failure: Impossible de créer la tâche planifiée. Veuillez consulter les journaux pour plus de détails 93 | 94 | machine_identification: Identification machine 95 | machine_id: Identificateur machine 96 | machine_group: Groupe machine 97 | 98 | show_decrypted: Voir les données sensibles 99 | no_manager_password_defined: Mot de passe gestionnaire non initialisé, ne peut montrer la version déchiffrée. Si vous venez d'en définir un, vous devez sauvegarder la configuration actuelle avant de pouvoir l'utiliser 100 | 101 | # compression 102 | auto: Automatique 103 | max: Maximale 104 | off: Aucune 105 | 106 | # priorities 107 | low: Basse 108 | normal: Normale 109 | high: Haute 110 | 111 | # source types 112 | source_type: Type de sources 113 | folder_list: Liste de dossiers / fichiers 114 | files_from: Liste depuis un fichier 115 | files_from_verbatim: Liste depuis un fichier "exact" 116 | files_from_raw: Liste depuis un fichier "raw" 117 | stdin_from_command: Entrée standard depuis une commande 118 | stdin_filename: Nom optionel pour les données sauvegardées depuis l'entrée standard 119 | 120 | # retention policy 121 | retention_policy: Politique de rétention 122 | keep: Garder 123 | last: derniers instantanés 124 | hourly: instantanés horaires 125 | daily: instantanés journalières 126 | weekly: instantanés hebdomadaires 127 | monthly: instantanés mensuels 128 | yearly: instantanés annuelles 129 | keep_within: Garder les instantanées dans une période relative au dernier instantané 130 | keep_tags: Garder les instantanés avec les tags suivants 131 | post_backup_housekeeping_percent_chance: Chance (%%) de lancer maintenance post-sauvegarde 132 | post_backup_housekeeping_percent_chance_explanation: Rend aléatoire la maintenance après sauvegarde (0-100%%, 0 = jamais, 100 = toujours) 133 | post_backup_housekeeping_interval: Intervalle de maintenance post-sauvegarde 134 | post_backup_housekeeping_interval_explanation: Intervalle de nombre d'exécutions de sauvegarde avant lancement operations de maintenance 135 | optional_ntp_server_uri: URI optionnelle serveur NTP 136 | prune_max_unused: Données inutilisées max lors des purges 137 | prune_max_unused_explanation: Taille maximale en pourcentage ou octets de données à conserver lors des opérations de purge maximales 138 | prune_max_repack_size: Taille maximale de repack 139 | prune_max_repack_size_explanation: Taille maximale de repack en bytes (permet de limiter la taille du stockage nécessaire lors des purges) 140 | 141 | # repo management 142 | repo_group: Groupe de dépots 143 | group_inherited: Hérité du groupe 144 | repo_group_config: Configuration des dépots et groupes 145 | global_config: Configuration globale 146 | select_object: Selectionner l'objet à configurer 147 | add_object: Ajouter un autre dépot ou groupe 148 | delete_object: Supprimer le dépot ou groupe actuel 149 | are_you_sure_to_delete: Êtes-vous sûr de vouloir supprimer le 150 | no_object_to_delete: Aucun objet à supprimer 151 | repo_already_exists: Dépot déjà existant 152 | group_already_exists: Groupe déjà existant 153 | cannot_remove_group_inherited_settings: Impossible de supprimer une option héritée de groupe. Veuillez supprimer l'option directement dans la configuration de groupe 154 | object_name_cannot_be_empty: Le nom de l'objet ne peut être vide et ne peut contenir de points 155 | object_name_cannot_be_all: Le nom de l'objet ne peut être '__all__' qui est un nom réservé 156 | cannot_delete_default_repo: Impossible de supprimer le dépot par défaut 157 | cannot_delete_default_group: Impossible de supprimer le groupe par défaut 158 | 159 | # permissions 160 | set_permissions: Définir les permissions et le mot de passe 161 | permissions_only_for_repos: Les permissions peuvent être appliquées uniquement à des dépots 162 | permissions: Permissions 163 | backup_perms: Sauvegardes uniquement 164 | restore_perms: Sauvegarde, vérification, récupération et restauration 165 | restore_only_perms: Restauration uniquement 166 | full_perms: Accès total 167 | setting_permissions_requires_manager_password: Un mot de passe gestionnaire est requis pour définir des permissions 168 | manager_password_too_simple: Le mot de passe gestionnaire nécessite au moins 8 caractères majuscules, minuscules et chiffres 169 | current_permissions: Permissions actives (non herité) 170 | manager_password_set: Mot de passe gestionnaire initialisé (non hérité) 171 | 172 | unknown_error_see_logs: Erreur inconnue, merci de vérifier les journaux 173 | 174 | enter_tag: Entrer tag 175 | enter_pattern: Entrer pattern 176 | enter_command: Entrer command 177 | enter_var_name: Entrer le nom de la variable 178 | enter_var_value: Entrer sa valeur 179 | enter_label_name: Entrer le nom de l'étiquette 180 | enter_label_value: Entrer la valeur de l'étiquette 181 | enter_label: Entrer étiquette 182 | 183 | suggested_encrypted_env_variables: Variables chiffrées suggérées 184 | 185 | policiy_group_by: Appliquer la politique de rétention par groupes d'instantanés 186 | group_by_host: Grouper par nom d'hôte 187 | group_by_paths: Grouper par chemins sauvegardés 188 | group_by_tags: Grouper par tags 189 | policiy_group_by_explanation: Si aucun groupement n'est choisi, le groupement par nom d'hôte et chemin de sauvegarde sera utilisé 190 | 191 | add_identity: Ajouter identités Cloud 192 | value_cannot_be_empty: La valeur ne peut être vide 193 | repo_uri_cloud_hint: L'URI du dépot Cloud nécessite de définir des variables d'environnement chiffrées (voir l'onglet environnement) -------------------------------------------------------------------------------- /npbackup/translations/generic.en.yml: -------------------------------------------------------------------------------- 1 | en: 2 | accept: Accept 3 | cancel: Cancel 4 | quit: Exit 5 | return: Return 6 | configure: Configure 7 | about: About 8 | options: Options 9 | create: Create 10 | change: Change 11 | close: Close 12 | finished: Finished 13 | 14 | yes: Yes 15 | no: No 16 | 17 | seconds: seconds 18 | minutes: minutes 19 | hours: hours 20 | 21 | user: User 22 | group: Group 23 | identity: Identity 24 | 25 | old: Old 26 | up_to_date: Up to date 27 | unknown: Unknown 28 | not_connected_yet: Not connected to repo 29 | 30 | size: Size 31 | path: Path 32 | paths: Paths 33 | modification_date: Modification date 34 | 35 | content: Content 36 | version: Version 37 | destination: Destination 38 | 39 | decrypt: Decrypt 40 | encrypt: Encrypt 41 | 42 | is_uptodate: Program Up to date 43 | 44 | success: Success 45 | successfully: successfully 46 | failure: Failure 47 | 48 | scheduled_task: Scheduled task 49 | 50 | forget: Forget 51 | forgotten: Forgotten 52 | forgetting: Forgetting 53 | 54 | no_snapshots: No snapshots 55 | 56 | are_you_sure: Are you sure ? 57 | 58 | select_file: Select file 59 | name: Name 60 | type: Type 61 | value: Value 62 | 63 | bogus_data_given: Bogus data given 64 | 65 | please_wait: Please wait 66 | 67 | bad_file: Bad file 68 | file_does_not_exist: File does not exist 69 | 70 | add_files: Add files 71 | add_folder: Add folder 72 | add_manually: Add manually 73 | remove_selected: Remove selected 74 | 75 | refresh: Refresh repo info 76 | load: Load -------------------------------------------------------------------------------- /npbackup/translations/generic.fr.yml: -------------------------------------------------------------------------------- 1 | fr: 2 | accept: Valider 3 | cancel: Annuler 4 | quit: Quitter 5 | return: Retour 6 | configure: Configurer 7 | about: A propos 8 | options: Options 9 | create: Créer 10 | change: Changer 11 | close: Fermer 12 | finished: Terminé 13 | 14 | yes: Oui 15 | no: Non 16 | 17 | seconds: secondes 18 | minutes: minutes 19 | hours: heures 20 | 21 | user: Utilisateur 22 | group: Groupe 23 | identity: Identité 24 | 25 | old: Ancien 26 | up_to_date: A jour 27 | unknown: Inconnu 28 | not_connected_yet: Non connecté au dépot 29 | 30 | size: Taille 31 | path: Chemin 32 | paths: Chemins 33 | modification_date: Date de modification 34 | 35 | content: Contenu 36 | version: Version 37 | destination: Destination 38 | 39 | decrypt: Déchiffrer 40 | encrypt: Chiffrer 41 | 42 | is_uptodate: Logiciel à jour 43 | 44 | success: Succès 45 | successfully: avec succès 46 | failure: Echec 47 | 48 | scheduled_task: Tâche planifiée 49 | 50 | forget: Oublier 51 | forgotten: oublié 52 | forgetting: Oubli de 53 | 54 | no_snapshots: Aucun instantané 55 | 56 | are_you_sure: Etes-vous sûr ? 57 | 58 | select_file: Selection fichier 59 | name: Nom 60 | type: Type 61 | value: Valeur 62 | 63 | bogus_data_given: Données invalides 64 | 65 | please_wait: Merci de patienter 66 | 67 | bad_file: Fichier erroné 68 | file_does_not_exist: Fichier inexistant 69 | 70 | add_files: Ajouter fichiers 71 | add_folder: Ajouter dossier 72 | add_manually: Ajouter manuellement 73 | remove_selected: Enlever la sélection 74 | 75 | refresh: Recharger les informations du dépot 76 | load: Charger -------------------------------------------------------------------------------- /npbackup/translations/main_gui.en.yml: -------------------------------------------------------------------------------- 1 | en: 2 | loading_snapshot_list_from_repo: Loading snapshot list from repo 3 | loading_last_snapshot_date: Loading last snapshot date 4 | this_will_take_a_while: This will take a while 5 | cannot_get_content: Cannot get content. Pleasse check the logs 6 | cannot_get_repo_status: Cannot get repo status 7 | creating_tree: Creating tree 8 | restore_to: Restore to 9 | restore: Restore 10 | restoration: Restoration 11 | restore_in_progress: Restore in progress 12 | restore_done: Restore done successfully 13 | restore_failed: Restore failed. Please check the logs 14 | select_folder: Please select a folder 15 | only_include: Only include 16 | destination_folder: Destination folder 17 | backup_state: Backup state 18 | repo_type: Repo type 19 | backup_list_to: List of backups to repo 20 | local_folder: Local folder 21 | external_server: external server 22 | launch_backup: Launch backup 23 | see_content: See content 24 | backup_content_from: Backup content from 25 | backup_from: Backup from 26 | gui_activity: Current activity 27 | backup_in_progress: Backup in progress 28 | backup_done: Backup done 29 | backup_failed: Backup failed. Please check the logs 30 | select_backup: Please select a backup 31 | run_as: run as 32 | identified_by: identified by 33 | unknown_repo: Repo format unknown 34 | repository_not_configured: Repository not configured 35 | execute_operation: Executing operation 36 | forget_failed: Failed to forget. Please check the logs 37 | operations: Operations 38 | select_config_file: Load config file 39 | repo_and_password_cannot_be_empty: Repo and password cannot be empty 40 | viewer_mode: Repo view-only mode 41 | open_repo: Open repo 42 | new_config: Create new config 43 | load_config: Load configuration 44 | config_error: Configuration error 45 | no_config: Please load / create a configuration before proceeding 46 | cannot_load_config_keep_current: Cannot load configuration. Keep current configuration 47 | snapshot_is_empty: Snapshot is empty 48 | select_only_one_snapshot: Please select only one snapshot 49 | cancel_operation: Are you sure you want to cancel the operation? Try to avoid interrupting write operations 50 | open_existing_file: Open existing config file 51 | failed_operation: Failed operation, Please check the logs 52 | auto_upgrade_checking: Checking for upgrade 53 | upgrade_in_progress: Upgrade process will close this program. Please wait for the upgrade to finish. The program will reload itself. 54 | # logs 55 | last_messages: Last messages 56 | error_messages: Error messages -------------------------------------------------------------------------------- /npbackup/translations/main_gui.fr.yml: -------------------------------------------------------------------------------- 1 | fr: 2 | loading_snapshot_list_from_repo: Chargement liste des instantanés depuis le dépot 3 | loading_last_snapshot_date: Chargement date du dernier instantané 4 | this_will_take_a_while: Cela prendra quelques instants 5 | cannot_get_content: Impossible d'obtenir le contenu. Veuillez vérifier les journaux 6 | cannot_get_repo_status: Impossible d'obtenir l'état du dépot 7 | creating_tree: Création de l'arborescence 8 | restore_to: Restaurer vers 9 | restore: Restaurer 10 | restoration: Restauration 11 | restore_in_progress: Restauration en cours 12 | restore_done: Restauration terminée avec succès 13 | restore_failed: Restauration échouées. veuillez vérifier les journaux 14 | select_folder: Veuillez sélectionner un dossier 15 | only_include: Inclure seulement 16 | destination_folder: Dossier de destination 17 | backup_state: Etat de sauvegarde 18 | repo_type: Type de dépot 19 | backup_list_to: Liste des sauvegardes vers le dépot 20 | local_folder: Dossier local 21 | external_server: serveur externalisé 22 | launch_backup: Sauvegarder 23 | see_content: Voir contenu 24 | backup_content_from: Contenu de la sauvegarde du 25 | backup_from: Sauvegarde du 26 | gui_activity: Activité courante 27 | backup_in_progress: Sauvegarde en cours 28 | backup_done: Sauvegarde terminée 29 | backup_failed: Sauvegarde échouée. veuillez vérifier les journaux 30 | select_backup: Veuillez sélectionner une sauvegarde 31 | run_as: faite en tant que 32 | identified_by: identifiée en tant que 33 | unknown_repo: Format de dépot inconnu 34 | repository_not_configured: Dépot non configuré 35 | execute_operation: Opération en cours 36 | forget_failed: Oubli impossible. Veuillez vérifier les journaux 37 | operations: Opérations 38 | select_config_file: Charger fichier de configuration 39 | repo_and_password_cannot_be_empty: Le dépot et le mot de passe ne peuvent être vides 40 | viewer_mode: Visualisation de dépot uniquement 41 | open_repo: Ouvrir dépot 42 | new_config: Créer nouvelle configuration 43 | load_config: Charger configuration 44 | config_error: Erreur de configuration 45 | no_config: Veuillez charger / créer une configuration avant de procéder 46 | cannot_load_config_keep_current: Impossible de charger la configuration. Configuration actuelle conservée 47 | snapshot_is_empty: L'instantané est vide 48 | select_only_one_snapshot: Veuillez sélectionner un seul instantané 49 | cancel_operation: Etes-vous sûr de vouloir annuler l'opération? Essayez d'éviter d'interrompre les opérations d'écriture 50 | open_existing_file: Ouvrir fichier de configuration existant 51 | failed_operation: Operation échouée. Vérifier les journaux 52 | auto_upgrade_checking: Vérification de mise à jour 53 | upgrade_in_progress: Le processus de mise à jour va fermer ce programme. Veuillez attendre que le processus de mise à jour soit terminé. Le programme se relancera tout seul. 54 | 55 | 56 | # logs 57 | last_messages: Last messages 58 | error_messages: Error messages -------------------------------------------------------------------------------- /npbackup/translations/operations_gui.en.yml: -------------------------------------------------------------------------------- 1 | en: 2 | select_repositories: Select one or more repositories, or groups of repositories 3 | configured_repositories: Configured repositories 4 | housekeeping: Housekeeping and retention operation 5 | task_scheduler: Task scheduler 6 | quick_check: Quick repo check 7 | full_check: Full repo check 8 | repair_index: Repair repo index 9 | repair_packs: Repair repo packs 10 | pack_ids: Pack IDs 11 | repair_snapshots: Repair repo snapshots 12 | recover: Recover deleted snapshots 13 | unlock: Unlock repo 14 | forget_using_retention_policy: Forget using retention polic 15 | standard_prune: Normal prune data 16 | max_prune: Prune with maximum efficiency 17 | stats: Last snapshot repo statistics 18 | stats_raw: Full repo statistics 19 | apply_to_all: Apply to all repos 20 | apply_to_selected_groups: Apply to selected groups 21 | no_groups_selected: No groups selected 22 | add_repo: Add repo 23 | edit_repo: Edit repo 24 | remove_repo: Remove repo 25 | no_repo_selected: No repo selected 26 | no_repo_selected_apply_all: No repo selected, apply to all ? 27 | show_advanced: Show advanced options 28 | 29 | currently_configured_tasks: Currently configured tasks 30 | select_task_type: Select task type 31 | add_task: Add task -------------------------------------------------------------------------------- /npbackup/translations/operations_gui.fr.yml: -------------------------------------------------------------------------------- 1 | fr: 2 | select_repositories: Sélectionner un ou plusieurs dépots, ou groupes de dépots 3 | configured_repositories: Dépots configurés 4 | housekeeping: Opération de maintenance et rétention 5 | task_scheduler: Planificateur de tâches 6 | quick_check: Vérification rapide dépot 7 | full_check: Vérification complète dépot 8 | repair_index: Réparer les index du dépot 9 | repair_packs: Réparer les blocs du dépot 10 | pack_ids: Identifiants de blocs 11 | repair_snapshots: Réparer les instantanés du dépot 12 | recover: Récupérer des instantanés supprimés 13 | unlock: Déblocage de dépot 14 | forget_using_retention_policy: Oublier les instantanés en utilisant la stratégie de rétention 15 | standard_prune: Opération de purge normale 16 | max_prune: Opération de purge la plus efficace 17 | stats: Statistiques de dépot dernier instantané 18 | stats_raw: Statistiques globales du dépot 19 | apply_to_all: Appliquer à tous les dépots 20 | apply_to_selected_groups: Appliquer aux groupes sélectionnés 21 | no_groups_selected: Aucun groupe sélectionné 22 | add_repo: Ajouter dépot 23 | edit_repo: Modifier dépot 24 | remove_repo: Supprimer dépot 25 | no_repo_selected: Aucun dépot sélectionné 26 | no_repo_selected_apply_all: Aucun dépot sélectionné, appliquer à tous ? 27 | show_advanced: Afficher les options avancées 28 | 29 | currently_configured_tasks: Tâches actuellement configurées 30 | select_task_type: Sélectionner le type de tâche 31 | add_task: Ajouter tâche -------------------------------------------------------------------------------- /npbackup/upgrade_client/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # Placeholder so this directory becomes a package 5 | -------------------------------------------------------------------------------- /npbackup/windows/sign_windows.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.sign_windows" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2023-2025 NetInvent" 9 | __license__ = "GPL-3.0-only" 10 | __build__ = "2024090801" 11 | __version__ = "1.2.0" 12 | 13 | 14 | import os 15 | import sys 16 | import argparse 17 | from cryptidy.symmetric_encryption import decrypt_message 18 | 19 | try: 20 | from windows_tools.signtool import SignTool 21 | except ImportError: 22 | print("This tool needs windows_tools.signtool >= 0.4.0") 23 | 24 | 25 | basepath = r"C:\GIT\npbackup\BUILDS" 26 | audiences = ["private", "public"] 27 | arches = ["x86", "x64"] 28 | binaries = ["npbackup-cli", "npbackup-gui", "npbackup-viewer"] 29 | 30 | 31 | def check_private_ev(): 32 | """ 33 | Test if we have private ev data 34 | """ 35 | try: 36 | from PRIVATE._ev_data import AES_EV_KEY 37 | from PRIVATE._obfuscation import obfuscation 38 | 39 | print("We have private EV certificate DATA") 40 | return obfuscation(AES_EV_KEY) 41 | except ImportError as exc: 42 | print("ERROR: Cannot load private EV certificate DATA: {}".format(exc)) 43 | sys.exit(1) 44 | 45 | 46 | def get_ev_data(cert_data_path): 47 | """ 48 | This retrieves specific data for crypto env 49 | """ 50 | aes_key = check_private_ev() 51 | with open(cert_data_path, "rb") as fp: 52 | ev_cert_data = fp.read() 53 | try: 54 | timestamp, ev_cert = decrypt_message(ev_cert_data, aes_key=aes_key) 55 | ( 56 | pkcs12_certificate, 57 | pkcs12_password, 58 | container_name, 59 | cryptographic_provider, 60 | ) = ev_cert 61 | except Exception as exc: 62 | print("EV Cert data is corrupt") 63 | sys.exit(1) 64 | return pkcs12_certificate, pkcs12_password, container_name, cryptographic_provider 65 | 66 | 67 | def sign( 68 | executable: str = None, 69 | arch: str = None, 70 | ev_cert_data: str = None, 71 | dry_run: bool = False, 72 | ): 73 | if ev_cert_data: 74 | ( 75 | pkcs12_certificate, 76 | pkcs12_password, 77 | container_name, 78 | cryptographic_provider, 79 | ) = get_ev_data(ev_cert_data) 80 | signer = SignTool( 81 | certificate=pkcs12_certificate, 82 | pkcs12_password=pkcs12_password, 83 | container_name=container_name, 84 | cryptographic_provider=cryptographic_provider, 85 | ) 86 | else: 87 | signer = SignTool() 88 | 89 | if executable: 90 | print(f"Signing {executable}") 91 | result = signer.sign(executable, bitness=arch, dry_run=dry_run) 92 | if not result: 93 | # IMPORTANT: If using an automated crypto USB EV token, we need to stop on error so we don't lock ourselves out of the token with bad password attempts 94 | raise EnvironmentError( 95 | "Could not sign executable ! Is the PKI key connected ?" 96 | ) 97 | return result 98 | 99 | for audience in audiences: 100 | for arch in arches: 101 | for binary in binaries: 102 | one_file_exe_path = os.path.join( 103 | basepath, audience, "windows", arch, binary + f"-{arch}.exe" 104 | ) 105 | standalone_exe_path = os.path.join( 106 | basepath, 107 | audience, 108 | "windows", 109 | arch, 110 | binary + ".dist", 111 | binary + f".exe", 112 | ) 113 | for exe_file in (one_file_exe_path, standalone_exe_path): 114 | if os.path.isfile(exe_file): 115 | print(f"Signing {exe_file}") 116 | result = signer.sign(exe_file, bitness=arch, dry_run=dry_run) 117 | if not result: 118 | # IMPORTANT: If using an automated crypto USB EV token, we need to stop on error so we don't lock ourselves out of the token with bad password attempts 119 | raise EnvironmentError( 120 | "Could not sign executable ! Is the PKI key connected ?" 121 | ) 122 | 123 | 124 | if __name__ == "__main__": 125 | parser = argparse.ArgumentParser( 126 | prog="npbackup sign_windows.py", 127 | description="Windows executable signer for NPBackup", 128 | ) 129 | 130 | parser.add_argument( 131 | "--dry-run", 132 | action="store_true", 133 | default=False, 134 | required=False, 135 | help="Don't actually sign anything, just test command", 136 | ) 137 | parser.add_argument( 138 | "--ev-cert-data", 139 | type=str, 140 | default=None, 141 | required=False, 142 | help="Path to EV certificate data", 143 | ) 144 | 145 | args = parser.parse_args() 146 | 147 | sign(ev_cert_data=args.ev_cert_data, dry_run=args.dry_run) 148 | -------------------------------------------------------------------------------- /resources/__init__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | # Placeholder so this directory becomes a package 5 | -------------------------------------------------------------------------------- /resources/file_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/resources/file_icon.png -------------------------------------------------------------------------------- /resources/folder_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/resources/folder_icon.png -------------------------------------------------------------------------------- /resources/inherited_file_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/resources/inherited_file_icon.png -------------------------------------------------------------------------------- /resources/inherited_folder_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/resources/inherited_folder_icon.png -------------------------------------------------------------------------------- /resources/inherited_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/resources/inherited_icon.png -------------------------------------------------------------------------------- /resources/inherited_irregular_file_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/resources/inherited_irregular_file_icon.png -------------------------------------------------------------------------------- /resources/inherited_missing_file_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/resources/inherited_missing_file_icon.png -------------------------------------------------------------------------------- /resources/inherited_neutral_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/resources/inherited_neutral_icon.png -------------------------------------------------------------------------------- /resources/inherited_symlink_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/resources/inherited_symlink_icon.png -------------------------------------------------------------------------------- /resources/inherited_tree_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/resources/inherited_tree_icon.png -------------------------------------------------------------------------------- /resources/irregular_file_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/resources/irregular_file_icon.png -------------------------------------------------------------------------------- /resources/missing_file_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/resources/missing_file_icon.png -------------------------------------------------------------------------------- /resources/non_inherited_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/resources/non_inherited_icon.png -------------------------------------------------------------------------------- /resources/npbackup_icon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/resources/npbackup_icon.ico -------------------------------------------------------------------------------- /resources/symlink_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/resources/symlink_icon.png -------------------------------------------------------------------------------- /resources/tree_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/resources/tree_icon.png -------------------------------------------------------------------------------- /resources/update_custom_resources.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.customization_creator" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2024-2025 NetInvent" 9 | __license__ = "GPL-3.0-only" 10 | __build__ = "2025051101" 11 | __version__ = "1.0.0" 12 | 13 | import os 14 | import re 15 | import base64 16 | from npbackup.path_helper import BASEDIR 17 | 18 | 19 | """ 20 | Launching this file will update customization.py inline png images and gif animations with files from resources directory, if exist 21 | """ 22 | 23 | 24 | def image_to_data_url(filename, as_url: bool = False): 25 | ext = filename.split(".")[-1] 26 | with open(filename, "rb") as f: 27 | img = f.read() 28 | if as_url: 29 | return f"data:image/{ext};base64," + base64.b64encode(img).decode("utf-8") 30 | else: 31 | return base64.b64encode(img).decode("utf-8") 32 | 33 | 34 | def update_custom_icons(): 35 | """ 36 | Update customization.py file with icon content 37 | """ 38 | 39 | custom_resources = { 40 | "FILE_ICON": "file_icon.png", 41 | "FOLDER_ICON": "folder_icon.png", 42 | "INHERITED_FILE_ICON": "inherited_file_icon.png", 43 | "INHERITED_FOLDER_ICON": "inherited_folder_icon.png", 44 | "INHERITED_ICON": "inherited_icon.png", 45 | "INHERITED_IRREGULAR_FILE_ICON": "inherited_irregular_file_icon.png", 46 | "INHERITED_NEUTRAL_ICON": "inherited_neutral_icon.png", 47 | "INHERITED_TREE_ICON": "inherited_tree_icon.png", 48 | "IRREGULAR_FILE_ICON": "irregular_file_icon.png", 49 | "NON_INHERITED_ICON": "non_inherited_icon.png", 50 | "MISSING_FILE_ICON": "missing_file_icon.png", 51 | "INHERITED_MISSING_FILE_ICON": "inherited_missing_file_icon.png", 52 | "INHERITED_SYMLINK_ICON": "inherited_symlink_icon.png", 53 | "SYMLINK_ICON": "symlink_icon.png", 54 | "TREE_ICON": "tree_icon.png", 55 | "LOADING_ANIMATION": "loading.gif", 56 | "OEM_LOGO": "oem_logo.png", 57 | "OEM_ICON": "oem_icon.png", 58 | } 59 | 60 | resources_dir = os.path.join(BASEDIR, os.path.pardir, "resources") 61 | customization_py = os.path.join(resources_dir, "customization.py") 62 | with open(customization_py, "r", encoding="utf-8") as f: 63 | customization = f.read() 64 | for var_name, file in custom_resources.items(): 65 | file_path = os.path.join(resources_dir, file) 66 | if os.path.exists(file_path): 67 | print(f"Updating {var_name} with {file_path}") 68 | encoded_b64 = image_to_data_url(file_path) 69 | customization = re.sub( 70 | f'\n{var_name} = .*', f'\n{var_name} = b"{encoded_b64}"', customization, re.MULTILINE 71 | ) 72 | else: 73 | print("No file found for", var_name) 74 | with open(customization_py, "w", encoding="utf-8") as f: 75 | f.write(customization) 76 | 77 | 78 | if __name__ == "__main__": 79 | update_custom_icons() 80 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup package 5 | 6 | 7 | __intname__ = "npbackup.setup" 8 | __author__ = "Orsiris de Jong" 9 | __copyright__ = "Copyright (C) 2022-2025 NetInvent" 10 | __license__ = "GPL-3.0-only" 11 | __build__ = "2024060401" 12 | __setup_ver__ = "1.2.0" 13 | 14 | 15 | PACKAGE_NAME = "npbackup" 16 | DESCRIPTION = "One fits all solution for deduplicated and compressed backups on servers and laptops" 17 | 18 | import sys 19 | import os 20 | import pkg_resources 21 | import setuptools 22 | 23 | 24 | def _read_file(filename): 25 | here = os.path.abspath(os.path.dirname(__file__)) 26 | if sys.version_info[0] < 3: 27 | # With python 2.7, open has no encoding parameter, resulting in TypeError 28 | # Fix with io.open (slow but works) 29 | from io import open as io_open 30 | 31 | try: 32 | with io_open( 33 | os.path.join(here, filename), "r", encoding="utf-8" 34 | ) as file_handle: 35 | return file_handle.read() 36 | except IOError: 37 | # Ugly fix for missing requirements.txt file when installing via pip under Python 2 38 | return "" 39 | else: 40 | with open(os.path.join(here, filename), "r", encoding="utf-8") as file_handle: 41 | return file_handle.read() 42 | 43 | 44 | def get_metadata(package_file): 45 | """ 46 | Read metadata from package file 47 | """ 48 | 49 | _metadata = {} 50 | 51 | for line in _read_file(package_file).splitlines(): 52 | if line.startswith("__version__") or line.startswith("__description__"): 53 | delim = "=" 54 | _metadata[line.split(delim)[0].strip().strip("__")] = ( 55 | line.split(delim)[1].strip().strip("'\"") 56 | ) 57 | return _metadata 58 | 59 | 60 | def parse_requirements(filename): 61 | """ 62 | There is a parse_requirements function in pip but it keeps changing import path 63 | Let's build a simple one 64 | """ 65 | try: 66 | requirements_txt = _read_file(filename) 67 | install_requires = [ 68 | str(requirement) 69 | for requirement in pkg_resources.parse_requirements(requirements_txt) 70 | ] 71 | return install_requires 72 | except OSError: 73 | print( 74 | 'WARNING: No requirements.txt file found as "{}". Please check path or create an empty one'.format( 75 | filename 76 | ) 77 | ) 78 | 79 | 80 | # With this, we can enforce a binary package. 81 | class BinaryDistribution(setuptools.Distribution): 82 | """Distribution which always forces a binary package with platform name""" 83 | 84 | @staticmethod 85 | def has_ext_modules(): 86 | return True 87 | 88 | 89 | package_path = os.path.abspath(PACKAGE_NAME) 90 | for path in ["__main__.py", PACKAGE_NAME + ".py"]: 91 | package_file = os.path.join(package_path, "__version__.py") 92 | if os.path.isfile(package_file): 93 | break 94 | metadata = get_metadata(package_file) 95 | requirements = parse_requirements(os.path.join(package_path, "requirements.txt")) 96 | long_description = _read_file("README.md") 97 | 98 | package_data = {"": ["translations/*.yml"]} 99 | 100 | # if os.name == "nt": 101 | scripts = ["misc/npbackup-cli.cmd"] 102 | # console_scripts = [] 103 | # else: 104 | # scripts = [] 105 | console_scripts = [ 106 | "npbackup-cli = npbackup.__main__:main", 107 | "npbackup-gui = npbackup.gui.__main__:main_gui", 108 | "npbackup-viewer = npbackup.gui.viewer:viewer_gui", 109 | ] 110 | 111 | setuptools.setup( 112 | name=PACKAGE_NAME, 113 | # We may use find_packages in order to not specify each package manually 114 | # packages = ['command_runner'], 115 | packages=setuptools.find_packages(), 116 | version=metadata["version"], 117 | install_requires=requirements, 118 | package_data=package_data, 119 | classifiers=[ 120 | # command_runner is mature 121 | "Development Status :: 5 - Production/Stable", 122 | "Intended Audience :: End Users/Desktop", 123 | "Intended Audience :: System Administrators", 124 | "Intended Audience :: Information Technology", 125 | "Intended Audience :: Developers", 126 | "Intended Audience :: Science/Research", 127 | "Topic :: System :: Archiving :: Backup", 128 | "Topic :: System", 129 | "Topic :: System :: Monitoring", 130 | "Topic :: Utilities", 131 | "Programming Language :: Python", 132 | "Programming Language :: Python :: 3", 133 | "Programming Language :: Python :: 3.6", 134 | "Programming Language :: Python :: 3.7", 135 | "Programming Language :: Python :: 3.8", 136 | "Programming Language :: Python :: 3.9", 137 | "Programming Language :: Python :: 3.10", 138 | "Programming Language :: Python :: 3.11", 139 | "Programming Language :: Python :: Implementation :: CPython", 140 | "Programming Language :: Python :: Implementation :: PyPy", 141 | "Operating System :: POSIX :: Linux", 142 | "Operating System :: Microsoft :: Windows", 143 | "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", 144 | ], 145 | description=DESCRIPTION, 146 | license="GPLv3", 147 | author="NetInvent - Orsiris de Jong", 148 | author_email="contact@netinvent.fr", 149 | url="https://github.com/netinvent/npbackup", 150 | keywords=[ 151 | "shell", 152 | "backup", 153 | "deduplication", 154 | "compression", 155 | "prometheus", 156 | "windows", 157 | "linux", 158 | "gui", 159 | "cli", 160 | "restic", 161 | "viewer", 162 | "kvm", 163 | "qemu", 164 | "grafana", 165 | "vss", 166 | ], 167 | long_description=long_description, 168 | long_description_content_type="text/markdown", 169 | python_requires=">=3.6", 170 | scripts=scripts, 171 | entry_points={ 172 | "console_scripts": console_scripts, 173 | }, 174 | # As we do version specific hacks for installed inline copies, make the 175 | # wheel version and platform specific. 176 | # distclass=BinaryDistribution, 177 | ) 178 | -------------------------------------------------------------------------------- /tests/npbackup-cli-test-linux.yaml: -------------------------------------------------------------------------------- 1 | conf_version: 3.0.1 2 | repos: 3 | default: 4 | repo_uri: ./test 5 | repo_group: default_group 6 | backup_opts: 7 | paths: 8 | - ../npbackup/npbackup 9 | tags: [] 10 | minimum_backup_size_error: 400 Kib 11 | exclude_files_larger_than: 0.0 Kib 12 | repo_opts: 13 | repo_password: test 14 | upload_speed: 100.0 Mib 15 | download_speed: 0.0 Kib 16 | retention_policy: {} 17 | prometheus: {} 18 | env: 19 | env_variables: {} 20 | encrypted_env_variables: {} 21 | groups: 22 | default_group: 23 | backup_opts: 24 | paths: [] 25 | source_type: 26 | tags: [] 27 | compression: auto 28 | use_fs_snapshot: false 29 | ignore_cloud_files: true 30 | exclude_caches: true 31 | one_file_system: true 32 | priority: low 33 | excludes_case_ignore: false 34 | exclude_files: 35 | - excludes/generic_excluded_extensions 36 | - excludes/generic_excludes 37 | - excludes/windows_excludes 38 | - excludes/linux_excludes 39 | exclude_patterns: 40 | exclude_files_larger_than: 41 | additional_parameters: 42 | additional_backup_only_parameters: 43 | minimum_backup_size_error: 10 MiB 44 | pre_exec_commands: [] 45 | pre_exec_per_command_timeout: 3600 46 | pre_exec_failure_is_fatal: false 47 | post_exec_commands: [] 48 | post_exec_per_command_timeout: 3600 49 | post_exec_failure_is_fatal: false 50 | post_exec_execute_even_on_backup_error: true 51 | post_backup_housekeeping_percent_chance: 0 52 | post_backup_housekeeping_interval: 0 53 | repo_opts: 54 | repo_password: 55 | repo_password_command: 56 | minimum_backup_age: 1435 57 | upload_speed: 100 Mib 58 | download_speed: 0 Mib 59 | backend_connections: 0 60 | retention_policy: 61 | last: 3 62 | hourly: 72 63 | daily: 30 64 | weekly: 4 65 | monthly: 12 66 | yearly: 3 67 | tags: [] 68 | keep_within: true 69 | ntp_server: 70 | prune_max_unused: 0 B 71 | prune_max_repack_size: 72 | prometheus: 73 | backup_job: ${MACHINE_ID} 74 | group: ${MACHINE_GROUP} 75 | env: 76 | env_variables: {} 77 | encrypted_env_variables: {} 78 | identity: 79 | machine_id: ${HOSTNAME}__Zo6u 80 | machine_group: 81 | global_prometheus: 82 | metrics: false 83 | instance: ${MACHINE_ID} 84 | destination: 85 | http_username: 86 | http_password: 87 | additional_labels: {} 88 | global_options: 89 | auto_upgrade: false 90 | auto_upgrade_percent_chance: 15 91 | auto_upgrade_interval: 0 92 | auto_upgrade_server_url: 93 | auto_upgrade_server_username: 94 | auto_upgrade_server_password: 95 | auto_upgrade_host_identity: ${MACHINE_ID} 96 | auto_upgrade_group: ${MACHINE_GROUP} 97 | -------------------------------------------------------------------------------- /tests/npbackup-cli-test-windows.yaml: -------------------------------------------------------------------------------- 1 | conf_version: 3.0.1 2 | repos: 3 | default: 4 | repo_uri: ./test 5 | repo_group: default_group 6 | backup_opts: 7 | paths: 8 | - ../npbackup/npbackup 9 | tags: [] 10 | minimum_backup_size_error: 400 Kib 11 | exclude_files_larger_than: 0.0 Kib 12 | repo_opts: 13 | repo_password: test 14 | upload_speed: 100.0 Mib 15 | download_speed: 0.0 Kib 16 | retention_policy: {} 17 | prometheus: {} 18 | env: 19 | env_variables: {} 20 | encrypted_env_variables: {} 21 | is_protected: false 22 | groups: 23 | default_group: 24 | backup_opts: 25 | paths: [] 26 | source_type: 27 | tags: [] 28 | compression: auto 29 | use_fs_snapshot: false 30 | ignore_cloud_files: true 31 | exclude_caches: true 32 | one_file_system: true 33 | priority: low 34 | excludes_case_ignore: false 35 | exclude_files: 36 | - excludes/generic_excluded_extensions 37 | - excludes/generic_excludes 38 | - excludes/windows_excludes 39 | - excludes/linux_excludes 40 | exclude_patterns: [] 41 | exclude_files_larger_than: 42 | additional_parameters: 43 | additional_backup_only_parameters: 44 | minimum_backup_size_error: 10 MiB 45 | pre_exec_commands: [] 46 | pre_exec_per_command_timeout: 3600 47 | pre_exec_failure_is_fatal: false 48 | post_exec_commands: [] 49 | post_exec_per_command_timeout: 3600 50 | post_exec_failure_is_fatal: false 51 | post_exec_execute_even_on_backup_error: true 52 | post_backup_housekeeping_percent_chance: 0 53 | post_backup_housekeeping_interval: 0 54 | repo_opts: 55 | repo_password: 56 | repo_password_command: 57 | minimum_backup_age: 1435 58 | upload_speed: 100 Mib 59 | download_speed: 0 Mib 60 | backend_connections: 0 61 | retention_policy: 62 | last: 3 63 | hourly: 72 64 | daily: 30 65 | weekly: 4 66 | monthly: 12 67 | yearly: 3 68 | tags: [] 69 | keep_within: true 70 | ntp_server: 71 | prune_max_unused: 0 B 72 | prune_max_repack_size: 73 | prometheus: 74 | backup_job: ${MACHINE_ID} 75 | group: ${MACHINE_GROUP} 76 | env: 77 | env_variables: {} 78 | encrypted_env_variables: {} 79 | is_protected: false 80 | identity: 81 | machine_id: ${HOSTNAME}__Zo6u 82 | machine_group: 83 | global_prometheus: 84 | metrics: false 85 | instance: ${MACHINE_ID} 86 | destination: 87 | http_username: 88 | http_password: 89 | additional_labels: {} 90 | global_options: 91 | auto_upgrade: false 92 | auto_upgrade_percent_chance: 15 93 | auto_upgrade_interval: 0 94 | auto_upgrade_server_url: 95 | auto_upgrade_server_username: 96 | auto_upgrade_server_password: 97 | auto_upgrade_host_identity: ${MACHINE_ID} 98 | auto_upgrade_group: ${MACHINE_GROUP} 99 | audience: private 100 | -------------------------------------------------------------------------------- /tests/test_restic_metrics.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | __intname__ = "restic_metrics_tests" 6 | __author__ = "Orsiris de Jong" 7 | __copyright__ = "Copyright (C) 2022-2025 NetInvent" 8 | __license__ = "BSD-3-Clause" 9 | __build__ = "2024120301" 10 | __description__ = ( 11 | "Converts restic command line output to a text file node_exporter can scrape" 12 | ) 13 | __compat__ = "python3.6+" 14 | 15 | import sys 16 | import os 17 | from pathlib import Path 18 | import re 19 | import json 20 | import tempfile 21 | from ofunctions.platform import os_arch 22 | from command_runner import command_runner 23 | 24 | try: 25 | from npbackup.restic_metrics import * 26 | except ImportError: # would be ModuleNotFoundError in Python 3+ 27 | # In case we run tests without actually having installed command_runner 28 | sys.path.insert(0, os.path.abspath(os.path.join(__file__, os.pardir, os.pardir))) 29 | from npbackup.restic_metrics import * 30 | from npbackup.core.restic_source_binary import get_restic_internal_binary 31 | from npbackup.path_helper import BASEDIR 32 | 33 | restic_json_outputs = {} 34 | restic_json_outputs[ 35 | "v0.16.2" 36 | ] = """{"message_type":"summary","files_new":5,"files_changed":15,"files_unmodified":6058,"dirs_new":0,"dirs_changed":27,"dirs_unmodified":866,"data_blobs":17,"tree_blobs":28,"data_added":281097,"total_files_processed":6078,"total_bytes_processed":122342158,"total_duration":1.2836983,"snapshot_id":"360333437921660a5228a9c1b65a2d97381f0bc135499c6e851acb0ab84b0b0a"} 37 | """ 38 | 39 | restic_str_outputs = {} 40 | # log file from restic v0.16.2 41 | restic_str_outputs[ 42 | "v0.16.2" 43 | ] = """repository 962d5924 opened (version 2, compression level auto) 44 | using parent snapshot 325a2fa1 45 | [0:00] 100.00% 4 / 4 index files loaded 46 | 47 | Files: 216 new, 21 changed, 5836 unmodified 48 | Dirs: 29 new, 47 changed, 817 unmodified 49 | Added to the repository: 4.425 MiB (1.431 MiB stored) 50 | 51 | processed 6073 files, 116.657 MiB in 0:03 52 | snapshot b28b0901 saved 53 | """ 54 | 55 | # log file from restic v0.14.0 56 | restic_str_outputs[ 57 | "v0.14.0" 58 | ] = """using parent snapshot df60db01 59 | 60 | Files: 1584 new, 269 changed, 235933 unmodified 61 | Dirs: 258 new, 714 changed, 37066 unmodified 62 | Added to the repo: 493.649 MiB 63 | 64 | processed 237786 files, 85.487 GiB in 11:12" 65 | """ 66 | 67 | # log file form restic v0.9.4 68 | restic_str_outputs[ 69 | "v0.9.4" 70 | ] = """ 71 | Files: 9 new, 32 changed, 110340 unmodified 72 | Dirs: 0 new, 2 changed, 0 unmodified 73 | Added to the repo: 196.568 MiB 74 | processed 110381 files, 107.331 GiB in 0:36 75 | """ 76 | 77 | # restic_metrics_v1 prometheus output 78 | expected_results_V1 = [ 79 | r'restic_repo_files{instance="test",backup_job="some_nas",state="new"} (\d+)', 80 | r'restic_repo_files{instance="test",backup_job="some_nas",state="changed"} (\d+)', 81 | r'restic_repo_files{instance="test",backup_job="some_nas",state="unmodified"} (\d+)', 82 | r'restic_repo_dirs{instance="test",backup_job="some_nas",state="new"} (\d+)', 83 | r'restic_repo_dirs{instance="test",backup_job="some_nas",state="changed"} (\d+)', 84 | r'restic_repo_dirs{instance="test",backup_job="some_nas",state="unmodified"} (\d+)', 85 | r'restic_repo_files{instance="test",backup_job="some_nas",state="total"} (\d+)', 86 | r'restic_repo_size_bytes{instance="test",backup_job="some_nas",state="total"} (\d+)', 87 | r'restic_backup_duration_seconds{instance="test",backup_job="some_nas",action="backup"} (\d+)', 88 | ] 89 | 90 | # restic_metrics_v2 prometheus output 91 | expected_results_V2 = [ 92 | r'restic_files{instance="test",backup_job="some_nas",state="new",action="backup"} (\d+)', 93 | r'restic_files{instance="test",backup_job="some_nas",state="changed",action="backup"} (\d+)', 94 | r'restic_files{instance="test",backup_job="some_nas",state="unmodified",action="backup"} (\d+)', 95 | r'restic_dirs{instance="test",backup_job="some_nas",state="new",action="backup"} (\d+)', 96 | r'restic_dirs{instance="test",backup_job="some_nas",state="changed",action="backup"} (\d+)', 97 | r'restic_dirs{instance="test",backup_job="some_nas",state="unmodified",action="backup"} (\d+)', 98 | r'restic_files{instance="test",backup_job="some_nas",state="total",action="backup"} (\d+)', 99 | r'restic_snasphot_size_bytes{instance="test",backup_job="some_nas",action="backup",type="processed"} (\d+)', 100 | r'restic_total_duration_seconds{instance="test",backup_job="some_nas",action="backup"} (\d+)', 101 | ] 102 | 103 | 104 | def running_on_github_actions(): 105 | """ 106 | This is set in github actions workflow with 107 | env: 108 | RUNNING_ON_GITHUB_ACTIONS: true 109 | """ 110 | return os.environ.get("RUNNING_ON_GITHUB_ACTIONS", "False").lower() == "true" 111 | 112 | 113 | def test_restic_str_output_2_metrics(): 114 | instance = "test" 115 | backup_job = "some_nas" 116 | labels = 'instance="{}",backup_job="{}"'.format(instance, backup_job) 117 | for version, output in restic_str_outputs.items(): 118 | print(f"Testing V1 parser restic str output from version {version}") 119 | errors, prom_metrics = restic_output_2_metrics(True, output, labels) 120 | assert errors is False 121 | # print(f"Parsed result:\n{prom_metrics}") 122 | for expected_result in expected_results_V1: 123 | match_found = False 124 | # print("Searching for {}".format(expected_result)) 125 | for metric in prom_metrics: 126 | result = re.match(expected_result, metric) 127 | if result: 128 | match_found = True 129 | break 130 | assert match_found is True, "No match found for {}".format(expected_result) 131 | 132 | 133 | def test_restic_str_output_to_json(): 134 | labels = {"instance": "test", "backup_job": "some_nas"} 135 | for version, output in restic_str_outputs.items(): 136 | print(f"Testing V2 parser restic str output from version {version}") 137 | json_metrics = restic_str_output_to_json(True, output) 138 | assert json_metrics["errors"] == False 139 | # print(json_metrics) 140 | _, prom_metrics, _ = restic_json_to_prometheus(True, json_metrics, labels) 141 | 142 | # print(f"Parsed result:\n{prom_metrics}") 143 | for expected_result in expected_results_V2: 144 | match_found = False 145 | # print("Searching for {}".format(expected_result)) 146 | for metric in prom_metrics: 147 | result = re.match(expected_result, metric) 148 | if result: 149 | match_found = True 150 | break 151 | assert match_found is True, "No match found for {}".format(expected_result) 152 | 153 | 154 | def test_restic_json_output(): 155 | labels = {"instance": "test", "backup_job": "some_nas"} 156 | for version, json_output in restic_json_outputs.items(): 157 | print(f"Testing V2 direct restic --json output from version {version}") 158 | restic_json = json.loads(json_output) 159 | _, prom_metrics, _ = restic_json_to_prometheus(True, restic_json, labels) 160 | # print(f"Parsed result:\n{prom_metrics}") 161 | for expected_result in expected_results_V2: 162 | match_found = False 163 | # print("Searching for {}".format(expected_result)) 164 | for metric in prom_metrics: 165 | result = re.match(expected_result, metric) 166 | if result: 167 | match_found = True 168 | break 169 | assert match_found is True, "No match found for {}".format(expected_result) 170 | 171 | 172 | def test_real_restic_output(): 173 | # We rely on the binaries downloaded in npbackup_tests here 174 | labels = {"instance": "test", "backup_job": "some_nas"} 175 | restic_binary = get_restic_internal_binary(os_arch()) 176 | print(f"Testing real restic output, Running with restic {restic_binary}") 177 | assert restic_binary is not None, "No restic binary found" 178 | 179 | for api_arg in ["", " --json"]: 180 | # Setup repo and run a quick backup 181 | repo_path = Path(tempfile.mkdtemp(prefix="npbackup_restic_tests_")) 182 | 183 | os.environ["RESTIC_REPOSITORY"] = str(repo_path) 184 | os.environ["RESTIC_PASSWORD"] = "TEST" 185 | 186 | exit_code, output = command_runner( 187 | f"{restic_binary} init --repository-version 2", live_output=True 188 | ) 189 | assert exit_code == 0, "Cannot init repo" 190 | # Just backup current npbackup project 191 | 192 | cmd = f"{restic_binary} backup {BASEDIR} {api_arg}" 193 | exit_code, output = command_runner(cmd, timeout=600, live_output=True) 194 | print("cmd", cmd) 195 | print("OUTPUT", output) 196 | try: 197 | repo_path.unlink() 198 | except OSError as exc: 199 | print(f"CANNOT REMOVE test repo: {exc}") 200 | assert exit_code == 0, "Failed to run restic" 201 | if not api_arg: 202 | restic_json = restic_str_output_to_json(True, output) 203 | else: 204 | restic_json = output 205 | _, prom_metrics, _ = restic_json_to_prometheus(True, restic_json, labels) 206 | # print(f"Parsed result:\n{prom_metrics}") 207 | for expected_result in expected_results_V2: 208 | match_found = False 209 | print("Searching for {}".format(expected_result)) 210 | for metric in prom_metrics: 211 | result = re.match(expected_result, metric) 212 | if result: 213 | match_found = True 214 | break 215 | assert match_found is True, "No match found for {}".format(expected_result) 216 | 217 | 218 | if __name__ == "__main__": 219 | test_restic_str_output_2_metrics() 220 | test_restic_str_output_to_json() 221 | test_restic_json_output() 222 | test_real_restic_output() 223 | -------------------------------------------------------------------------------- /upgrade_server/requirements.txt: -------------------------------------------------------------------------------- 1 | fastapi 2 | fastapi-offline>=1.5.0 3 | uvicorn[standard] 4 | gunicorn 5 | pydantic 6 | ruamel.yaml 7 | ofunctions.logger_utils>=2.4.0 -------------------------------------------------------------------------------- /upgrade_server/upgrade_server.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __appname__ = "npbackup_upgrade_server" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2023-2025 NetInvent" 9 | __license__ = "GPL-3.0-only" 10 | __build__ = "2025012401" 11 | __version__ = "3.0.0" 12 | 13 | 14 | import sys 15 | import os 16 | import multiprocessing 17 | from argparse import ArgumentParser 18 | from upgrade_server import configuration 19 | from ofunctions.logger_utils import logger_get_logger 20 | import upgrade_server.api 21 | from upgrade_server.__debug__ import _DEBUG 22 | 23 | 24 | if __name__ == "__main__": 25 | _DEV = os.environ.get("_DEV", False) 26 | 27 | parser = ArgumentParser( 28 | prog="{} {} - {}".format(__appname__, __copyright__, __license__), 29 | description="""NPBackup Upgrade server""", 30 | ) 31 | 32 | parser.add_argument( 33 | "--dev", action="store_true", help="Run with uvicorn in devel environment" 34 | ) 35 | 36 | parser.add_argument( 37 | "-c", 38 | "--config-file", 39 | dest="config_file", 40 | type=str, 41 | default=None, 42 | required=False, 43 | help="Path to upgrade_server.conf file", 44 | ) 45 | 46 | parser.add_argument( 47 | "--log-file", 48 | type=str, 49 | default=None, 50 | required=False, 51 | help="Optional path for logfile, overrides config file values", 52 | ) 53 | 54 | args = parser.parse_args() 55 | if args.dev: 56 | _DEV = True 57 | 58 | if args.log_file: 59 | log_file = args.log_file 60 | else: 61 | if os.name == "nt": 62 | log_file = os.path.join(f"{__appname__}.log") 63 | else: 64 | log_file = f"/var/log/{__appname__}.log" 65 | logger = logger_get_logger(log_file, debug=_DEBUG) 66 | 67 | if args.config_file: 68 | config_dict = configuration.load_config(args.config_file) 69 | else: 70 | config_dict = configuration.load_config() 71 | 72 | try: 73 | if not args.log_file: 74 | logger = logger_get_logger( 75 | config_dict["http_server"]["log_file"], debug=_DEBUG 76 | ) 77 | except (AttributeError, KeyError, IndexError, TypeError): 78 | pass 79 | 80 | try: 81 | listen = config_dict["http_server"]["listen"] 82 | except (TypeError, KeyError): 83 | listen = None 84 | try: 85 | port = config_dict["http_server"]["port"] 86 | except (TypeError, KeyError): 87 | port = None 88 | 89 | logger = logger_get_logger() 90 | # Cannot run gunicorn on Windows 91 | if _DEV or os.name == "nt": 92 | logger.info("Running dev version") 93 | import uvicorn 94 | 95 | server_args = { 96 | "workers": 1, 97 | "log_level": "debug", 98 | "reload": True, 99 | "host": listen if listen else "0.0.0.0", 100 | "port": port if port else 8080, 101 | } 102 | else: 103 | import gunicorn.app.base 104 | 105 | class StandaloneApplication(gunicorn.app.base.BaseApplication): 106 | """ 107 | This class supersedes gunicorn's class in order to load config before launching the app 108 | """ 109 | 110 | def __init__(self, app, options=None): 111 | self.options = options or {} 112 | self.application = app 113 | super().__init__() 114 | 115 | def load_config(self): 116 | config = { 117 | key: value 118 | for key, value in self.options.items() 119 | if key in self.cfg.settings and value is not None 120 | } 121 | for key, value in config.items(): 122 | self.cfg.set(key.lower(), value) 123 | 124 | def load(self): 125 | return self.application 126 | 127 | server_args = { 128 | "workers": (multiprocessing.cpu_count() * 2) + 1, 129 | "bind": f"{listen}:{port}" if listen else "0.0.0.0:8080", 130 | "worker_class": "uvicorn.workers.UvicornWorker", 131 | } 132 | 133 | try: 134 | if _DEV or os.name == "nt": 135 | uvicorn.run("upgrade_server.api:app", **server_args) 136 | else: 137 | StandaloneApplication(upgrade_server.api.app, server_args).run() 138 | except KeyboardInterrupt as exc: 139 | logger.error("Program interrupted by keyoard: {}".format(exc)) 140 | sys.exit(200) 141 | except Exception as exc: 142 | logger.error("Program interrupted by error: {}".format(exc)) 143 | logger.critical("Trace:", exc_info=True) 144 | sys.exit(201) 145 | -------------------------------------------------------------------------------- /upgrade_server/upgrade_server/__debug__.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.__debug__" 7 | __author__ = "Orsiris de Jong" 8 | __site__ = "https://www.netperfect.fr/npbackup" 9 | __description__ = "NetPerfect Backup Client" 10 | __copyright__ = "Copyright (C) 2023-2025 NetInvent" 11 | __build__ = "2024081901" 12 | 13 | 14 | import sys 15 | import os 16 | from typing import Callable 17 | from functools import wraps 18 | from logging import getLogger 19 | import json 20 | 21 | 22 | logger = getLogger() 23 | 24 | 25 | # If set, debugging will be enabled by setting environment variable to __SPECIAL_DEBUG_STRING content 26 | # Else, a simple true or false will suffice 27 | __SPECIAL_DEBUG_STRING = "" 28 | __debug_os_env = os.environ.get("_DEBUG", "False").strip("'\"") 29 | 30 | 31 | if not __SPECIAL_DEBUG_STRING: 32 | if "--debug" in sys.argv: 33 | _DEBUG = True 34 | sys.argv.pop(sys.argv.index("--debug")) 35 | 36 | 37 | if not "_DEBUG" in globals(): 38 | _DEBUG = False 39 | if __SPECIAL_DEBUG_STRING: 40 | if __debug_os_env == __SPECIAL_DEBUG_STRING: 41 | _DEBUG = True 42 | elif __debug_os_env.capitalize() == "True": 43 | _DEBUG = True 44 | 45 | 46 | def catch_exceptions(fn: Callable): 47 | """ 48 | Catch any exception and log it so we don't loose exceptions in thread 49 | """ 50 | 51 | @wraps(fn) 52 | def wrapper(self, *args, **kwargs): 53 | try: 54 | # pylint: disable=E1102 (not-callable) 55 | return fn(self, *args, **kwargs) 56 | except Exception as exc: 57 | # pylint: disable=E1101 (no-member) 58 | operation = fn.__name__ 59 | logger.error(f"General catcher: Function {operation} failed with: {exc}") 60 | logger.error("Trace:", exc_info=True) 61 | return None 62 | 63 | return wrapper 64 | 65 | 66 | def fmt_json(js: dict): 67 | """ 68 | Just a quick and dirty shorthand for pretty print which doesn't require pprint 69 | to be loaded 70 | """ 71 | js = json.dumps(js, indent=4) 72 | return js 73 | -------------------------------------------------------------------------------- /upgrade_server/upgrade_server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netinvent/npbackup/357abb6b765ee689c49802b0391cb4500514f6ed/upgrade_server/upgrade_server/__init__.py -------------------------------------------------------------------------------- /upgrade_server/upgrade_server/configuration.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.upgrade_server.configuration" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2023-2025 NetInvent" 9 | __license__ = "GPL-3.0-only" 10 | __build__ = "2023020601" 11 | 12 | 13 | import os 14 | from ruamel.yaml import YAML 15 | from logging import getLogger 16 | 17 | 18 | ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) 19 | config_file = os.path.join(ROOT_DIR, "upgrade_server.conf") 20 | 21 | logger = getLogger() 22 | 23 | 24 | def load_config(config_file: str = config_file): 25 | """ 26 | Using ruamel.yaml preserves comments and order of yaml files 27 | """ 28 | logger.debug("Using configuration file {}".format(config_file)) 29 | try: 30 | with open(config_file, "r", encoding="utf-8") as file_handle: 31 | # RoundTrip loader is default and preserves comments and ordering 32 | yaml = YAML(typ="rt") 33 | config_dict = yaml.load(file_handle) 34 | return config_dict 35 | except FileNotFoundError: 36 | logger.error("config file %s not found", config_file) 37 | return None 38 | except TypeError: 39 | logger.error("No config file given") 40 | return None 41 | 42 | 43 | def save_config(config_file, config_dict): 44 | with open(config_file, "w", encoding="utf-8") as file_handle: 45 | yaml = YAML(typ="rt") 46 | yaml.dump(config_dict, file_handle) 47 | -------------------------------------------------------------------------------- /upgrade_server/upgrade_server/crud.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.upgrade_server.crud" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2023-2025 NetInvent" 9 | __license__ = "GPL-3.0-only" 10 | __build__ = "2025012401" 11 | 12 | 13 | import os 14 | from typing import Optional, Union, Tuple 15 | from logging import getLogger 16 | import hashlib 17 | from datetime import datetime, timezone 18 | from upgrade_server.models.files import ClientTargetIdentification, FileGet, FileSend 19 | from upgrade_server.models.oper import CurrentVersion 20 | 21 | 22 | logger = getLogger() 23 | 24 | 25 | def sha256sum_data(data): 26 | # type: (bytes) -> str 27 | """ 28 | Returns sha256sum of some data 29 | """ 30 | sha256 = hashlib.sha256() 31 | sha256.update(data) 32 | return sha256.hexdigest() 33 | 34 | 35 | def is_enabled(config_dict) -> bool: 36 | path = os.path.join(config_dict["upgrades"]["data_root"], "DISABLED") 37 | return not os.path.isfile(path) 38 | 39 | 40 | def _get_path_from_target_id( 41 | config_dict, target_id: ClientTargetIdentification 42 | ) -> Tuple[str, str, str]: 43 | """ 44 | Determine specific or generic upgrade path depending on target_id sent by client 45 | 46 | If a specific sub path is found, we'll return that one, otherwise we'll return the default path 47 | 48 | Possible archive names are: 49 | npbackup-{platform}-{arch}-{build_type}-{audience}.{archive_extension} 50 | 51 | Possible upgrade script names are: 52 | npbackup-{platform}-{arch}-{build_type}-{audience}.sh 53 | npbackup-{platform}-{arch}-{build_type}-{audience}.cmd 54 | npbackup-{platform}.sh 55 | npbackup-{platform}.cmd 56 | 57 | """ 58 | if target_id.platform.value == "windows": 59 | archive_extension = "zip" 60 | script_extension = "cmd" 61 | else: 62 | archive_extension = "tar.gz" 63 | script_extension = "sh" 64 | 65 | expected_archive_filename = f"npbackup-{target_id.platform.value}-{target_id.arch.value}-{target_id.build_type.value}-{target_id.audience.value}.{archive_extension}" 66 | expected_script_filename = f"npbackup-{target_id.platform.value}-{target_id.arch.value}-{target_id.build_type.value}-{target_id.audience.value}.{script_extension}" 67 | expected_short_script_filename = ( 68 | f"npbackup-{target_id.platform.value}.{script_extension}" 69 | ) 70 | 71 | base_path = os.path.join( 72 | config_dict["upgrades"]["data_root"], 73 | ) 74 | 75 | for posssible_sub_path in [ 76 | target_id.auto_upgrade_host_identity, 77 | target_id.installed_version, 78 | target_id.group, 79 | ]: 80 | if posssible_sub_path: 81 | possibile_sub_path = os.path.join(base_path, posssible_sub_path) 82 | if os.path.isdir(possibile_sub_path): 83 | logger.info(f"Found specific upgrade path in {possibile_sub_path}") 84 | base_path = possibile_sub_path 85 | break 86 | 87 | archive_path = os.path.join(base_path, expected_archive_filename) 88 | script_path = os.path.join(base_path, expected_script_filename) 89 | short_script_path = os.path.join(base_path, expected_short_script_filename) 90 | 91 | version_file_path = os.path.join(base_path, "VERSION") 92 | 93 | return version_file_path, archive_path, script_path, short_script_path 94 | 95 | 96 | def store_host_info(destination: str, host_id: dict) -> None: 97 | try: 98 | data = ( 99 | datetime.now(timezone.utc).isoformat() 100 | + "," 101 | + ",".join([str(value) if value else "" for value in host_id.values()]) 102 | + "\n" 103 | ) 104 | with open(destination, "a", encoding="utf-8") as fpw: 105 | fpw.write(data) 106 | except OSError as exc: 107 | logger.error("Cannot write statistics file") 108 | logger.error("Trace:", exc_info=True) 109 | 110 | 111 | def get_current_version( 112 | config_dict: dict, 113 | target_id: ClientTargetIdentification, 114 | ) -> Optional[CurrentVersion]: 115 | try: 116 | version_filename, _, _, _ = _get_path_from_target_id(config_dict, target_id) 117 | logger.info(f"Searching for version in {version_filename}") 118 | if os.path.isfile(version_filename): 119 | with open(version_filename, "r", encoding="utf-8") as fh: 120 | ver = fh.readline().strip() 121 | return CurrentVersion(version=ver) 122 | except OSError as exc: 123 | logger.error(f"Cannot get current version: {exc}") 124 | logger.error("Trace:", exc_info=True) 125 | except Exception as exc: 126 | logger.error(f"Version seems to be bogus in VERSION file: {exc}") 127 | logger.error("Trace:", exc_info=True) 128 | 129 | 130 | def get_file( 131 | config_dict: dict, file: FileGet, content: bool = False 132 | ) -> Optional[Union[FileSend, bytes, bool]]: 133 | 134 | _, archive_path, script_path, short_script_path = _get_path_from_target_id( 135 | config_dict, file 136 | ) 137 | 138 | unknown_artefact = FileSend( 139 | artefact=file.artefact.value, 140 | arch=file.arch.value, 141 | platform=file.platform.value, 142 | build_type=file.build_type.value, 143 | audience=file.audience.value, 144 | sha256sum=None, 145 | filename=None, 146 | file_length=0, 147 | ) 148 | 149 | if file.artefact.value == "archive": 150 | artefact_paths = [archive_path] 151 | elif file.artefact.value == "script": 152 | artefact_paths = [script_path, short_script_path] 153 | else: 154 | logger.error(f"Unknown artefact type {file.artefact.value}") 155 | return unknown_artefact 156 | 157 | artefact_found = False 158 | for artefact_path in artefact_paths: 159 | logger.info( 160 | f"Searching for file {'info' if not content else 'content'} in {artefact_path}" 161 | ) 162 | if not os.path.isfile(artefact_path): 163 | logger.info(f"No {file.artefact.value} file found in {artefact_path}") 164 | else: 165 | artefact_found = True 166 | break 167 | if not artefact_found: 168 | if content: 169 | return False 170 | return unknown_artefact 171 | 172 | logger.info(f"Serving {file.artefact.value} file in {artefact_path}") 173 | with open(artefact_path, "rb") as fh: 174 | file_content_bytes = fh.read() 175 | if content: 176 | return file_content_bytes 177 | length = len(file_content_bytes) 178 | sha256 = sha256sum_data(file_content_bytes) 179 | file_send = FileSend( 180 | artefact=file.artefact.value, 181 | arch=file.arch.value, 182 | platform=file.platform.value, 183 | build_type=file.build_type.value, 184 | audience=file.audience.value, 185 | sha256sum=sha256, 186 | filename=os.path.basename(artefact_path), 187 | file_length=length, 188 | ) 189 | return file_send 190 | -------------------------------------------------------------------------------- /upgrade_server/upgrade_server/models/files.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.upgrade_server.models.files" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2023-2025 NetInvent" 9 | __license__ = "GPL-3.0-only" 10 | __build__ = "2025012401" 11 | 12 | 13 | from typing import Optional 14 | from enum import Enum 15 | from pydantic import BaseModel, constr 16 | 17 | 18 | class Artefact(Enum): 19 | script = "script" 20 | archive = "archive" 21 | 22 | 23 | class Platform(Enum): 24 | windows = "windows" 25 | linux = "linux" 26 | 27 | 28 | class Arch(Enum): 29 | x86 = "x86" 30 | x64 = "x64" 31 | 32 | 33 | class BuildType(Enum): 34 | gui = "gui" 35 | cli = "cli" 36 | 37 | 38 | class Audience(Enum): 39 | public = "public" 40 | private = "private" 41 | 42 | 43 | class ClientTargetIdentification(BaseModel): 44 | arch: Arch 45 | platform: Platform 46 | build_type: BuildType 47 | audience: Audience 48 | auto_upgrade_host_identity: Optional[str] = None 49 | installed_version: Optional[str] = None 50 | group: Optional[str] = None 51 | 52 | 53 | class FileGet(ClientTargetIdentification): 54 | artefact: Artefact 55 | 56 | 57 | class FileSend(ClientTargetIdentification): 58 | artefact: Artefact 59 | sha256sum: Optional[constr(min_length=64, max_length=64)] = None 60 | filename: Optional[str] = None 61 | file_length: int 62 | -------------------------------------------------------------------------------- /upgrade_server/upgrade_server/models/oper.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # This file is part of npbackup 5 | 6 | __intname__ = "npbackup.upgrade_server.models.oper" 7 | __author__ = "Orsiris de Jong" 8 | __copyright__ = "Copyright (C) 2023-2025 NetInvent" 9 | __license__ = "GPL-3.0-only" 10 | __build__ = "202303101" 11 | 12 | 13 | from pydantic import BaseModel 14 | 15 | 16 | class CurrentVersion(BaseModel): 17 | version: str 18 | --------------------------------------------------------------------------------