├── .dockerignore ├── .editorconfig ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── feature_request.md │ └── support_request.md ├── dependabot.yml └── workflows │ ├── deploy-docs.yml │ ├── golangci-lint.yml │ ├── release.yml │ ├── test.yml │ └── unit.yml ├── .golangci.yml ├── Dockerfile ├── LICENSE ├── README.md ├── cmd └── backup │ ├── archive.go │ ├── command.go │ ├── config.go │ ├── config_provider.go │ ├── config_provider_test.go │ ├── copy_archive.go │ ├── create_archive.go │ ├── encrypt_archive.go │ ├── exec.go │ ├── hooks.go │ ├── lock.go │ ├── main.go │ ├── notifications.go │ ├── notifications.tmpl │ ├── profile.go │ ├── prune_backups.go │ ├── run_script.go │ ├── script.go │ ├── stats.go │ ├── stop_restart.go │ ├── stop_restart_test.go │ ├── testdata │ ├── braces.env │ ├── comments.env │ ├── default.env │ └── expansion.env │ └── util.go ├── docs ├── .gitignore ├── Gemfile ├── Gemfile.lock ├── README.md ├── _config.yml ├── _sass │ └── custom │ │ └── custom.scss ├── how-tos │ ├── automatically-prune-old-backups.md │ ├── define-different-retention-schedules.md │ ├── encrypt-backups-using-gpg.md │ ├── encrypt-backups.md │ ├── handle-file-uploads-using-third-party-tools.md │ ├── index.md │ ├── manual-trigger.md │ ├── replace-deprecated-backup-from-snapshot.md │ ├── replace-deprecated-backup-stop-container-label.md │ ├── replace-deprecated-exec-labels.md │ ├── restore-volumes-from-backup.md │ ├── run-custom-commands.md │ ├── run-multiple-schedules.md │ ├── set-container-timezone.md │ ├── set-up-dropbox.md │ ├── set-up-notifications.md │ ├── stop-containers-during-backup.md │ ├── update-deprecated-email-config.md │ ├── use-as-non-root.md │ ├── use-custom-docker-host.md │ ├── use-rootless-docker.md │ └── use-with-docker-swarm.md ├── index.md ├── recipes │ └── index.md └── reference │ └── index.md ├── go.mod ├── go.sum ├── internal ├── errwrap │ └── wrap.go └── storage │ ├── azure │ └── azure.go │ ├── dropbox │ └── dropbox.go │ ├── local │ └── local.go │ ├── s3 │ └── s3.go │ ├── ssh │ └── ssh.go │ ├── storage.go │ └── webdav │ └── webdav.go └── test ├── Dockerfile ├── README.md ├── age-passphrase ├── docker-compose.yml └── run.sh ├── age-publickey ├── .gitignore ├── docker-compose.yml └── run.sh ├── azure ├── docker-compose.yml └── run.sh ├── certs ├── docker-compose.yml ├── run.sh └── san.cnf ├── cli └── run.sh ├── collision ├── docker-compose.yml └── run.sh ├── commands ├── docker-compose.yml └── run.sh ├── confd ├── 01backup.env ├── 02backup.env ├── 03never.env ├── docker-compose.yml └── run.sh ├── dropbox ├── .gitignore ├── docker-compose.yml ├── oauth2_config.json ├── run.sh └── user_v2.yaml ├── extend ├── Dockerfile ├── docker-compose.yml └── run.sh ├── gpg-asym ├── docker-compose.yml └── run.sh ├── gpg ├── docker-compose.yml └── run.sh ├── ignore ├── docker-compose.yml ├── run.sh └── sources │ ├── me.txt │ └── skip.me ├── local ├── docker-compose.yml └── run.sh ├── lock ├── docker-compose.yml └── run.sh ├── nonroot ├── 01conf.env ├── docker-compose.yml └── run.sh ├── notifications ├── docker-compose.yml ├── notifications.tmpl └── run.sh ├── ownership ├── docker-compose.yml └── run.sh ├── pgzip └── run.sh ├── proxy ├── docker-compose.swarm.yml ├── docker-compose.yml └── run.sh ├── pruning ├── docker-compose.yml └── run.sh ├── s3 ├── docker-compose.yml └── run.sh ├── secrets ├── docker-compose.yml └── run.sh ├── services ├── docker-compose.yml └── run.sh ├── ssh ├── docker-compose.yml └── run.sh ├── swarm ├── docker-compose.yml └── run.sh ├── tar ├── docker-compose.yml └── run.sh ├── test.sh ├── user ├── docker-compose.yml └── run.sh ├── util.sh ├── webdav ├── docker-compose.yml └── run.sh └── zstd └── run.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | test 2 | .github 3 | .circleci 4 | docs 5 | .editorconfig 6 | LICENSE 7 | README.md 8 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: http://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Unix-style newlines with a newline ending every file 7 | [*] 8 | end_of_line = lf 9 | insert_final_newline = true 10 | trim_trailing_whitespace = true 11 | indent_style = space 12 | indent_size = 2 13 | 14 | [*.md] 15 | trim_trailing_whitespace = false 16 | 17 | [*.go] 18 | indent_style = tab 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | 14 | 15 | **To Reproduce** 16 | Steps to reproduce the behavior: 17 | 1. ... 18 | 2. ... 19 | 3. ... 20 | 21 | **Expected behavior** 22 | 25 | 26 | **Version (please complete the following information):** 27 | - Image Version: 28 | - Docker Version: 29 | - Docker Compose Version (if applicable): 30 | 31 | **Additional context** 32 | 35 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | 14 | 15 | **Describe the solution you'd like** 16 | 19 | 20 | **Describe alternatives you've considered** 21 | 24 | 25 | **Additional context** 26 | 29 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/support_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Support request 3 | about: Ask for help 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **What are you trying to do?** 11 | 14 | 15 | **What is your current configuration?** 16 | 19 | 20 | **Log output** 21 | 24 | 25 | **Additional context** 26 | 29 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: docker 4 | directory: / 5 | schedule: 6 | interval: weekly 7 | - package-ecosystem: gomod 8 | directory: / 9 | schedule: 10 | interval: weekly 11 | -------------------------------------------------------------------------------- /.github/workflows/deploy-docs.yml: -------------------------------------------------------------------------------- 1 | name: Deploy Documenation site to GitHub Pages 2 | 3 | on: 4 | push: 5 | branches: ['main'] 6 | paths: 7 | - 'docs/**' 8 | - '.github/workflows/deploy-docs.yml' 9 | workflow_dispatch: 10 | 11 | permissions: 12 | contents: read 13 | pages: write 14 | id-token: write 15 | 16 | concurrency: 17 | group: 'pages' 18 | cancel-in-progress: true 19 | 20 | jobs: 21 | build: 22 | runs-on: ubuntu-latest 23 | steps: 24 | - name: Checkout 25 | uses: actions/checkout@v4 26 | - name: Setup Ruby 27 | uses: ruby/setup-ruby@v1 28 | with: 29 | ruby-version: '3.2' 30 | bundler-cache: true 31 | cache-version: 0 32 | working-directory: docs 33 | - name: Setup Pages 34 | id: pages 35 | uses: actions/configure-pages@v2 36 | - name: Build with Jekyll 37 | working-directory: docs 38 | run: bundle exec jekyll build --baseurl "${{ steps.pages.outputs.base_path }}" 39 | env: 40 | JEKYLL_ENV: production 41 | - name: Upload artifact 42 | uses: actions/upload-pages-artifact@v3 43 | with: 44 | path: 'docs/_site/' 45 | 46 | deploy: 47 | environment: 48 | name: github-pages 49 | url: ${{ steps.deployment.outputs.page_url }} 50 | runs-on: ubuntu-latest 51 | needs: build 52 | steps: 53 | - name: Deploy to GitHub Pages 54 | id: deployment 55 | uses: actions/deploy-pages@v4 56 | -------------------------------------------------------------------------------- /.github/workflows/golangci-lint.yml: -------------------------------------------------------------------------------- 1 | name: Run Linters 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | 8 | permissions: 9 | contents: read 10 | # Optional: allow read access to pull request. Use with `only-new-issues` option. 11 | pull-requests: read 12 | 13 | jobs: 14 | golangci: 15 | name: lint 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v4 19 | - uses: actions/setup-go@v5 20 | with: 21 | go-version: '1.24' 22 | cache: false 23 | - name: golangci-lint 24 | uses: golangci/golangci-lint-action@v3 25 | with: 26 | # Require: The version of golangci-lint to use. 27 | # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version. 28 | # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit. 29 | version: v1.64 30 | 31 | # Optional: working directory, useful for monorepos 32 | # working-directory: somedir 33 | 34 | # Optional: golangci-lint command line arguments. 35 | # 36 | # Note: By default, the `.golangci.yml` file should be at the root of the repository. 37 | # The location of the configuration file can be changed by using `--config=` 38 | # args: --timeout=30m --config=/my/path/.golangci.yml --issues-exit-code=0 39 | 40 | # Optional: show only new issues if it's a pull request. The default value is `false`. 41 | # only-new-issues: true 42 | 43 | # Optional: if set to true, then all caching functionality will be completely disabled, 44 | # takes precedence over all other caching options. 45 | # skip-cache: true 46 | 47 | # Optional: if set to true, then the action won't cache or restore ~/go/pkg. 48 | # skip-pkg-cache: true 49 | 50 | # Optional: if set to true, then the action won't cache or restore ~/.cache/go-build. 51 | # skip-build-cache: true 52 | 53 | # Optional: The mode to install golangci-lint. It can be 'binary' or 'goinstall'. 54 | # install-mode: "goinstall" 55 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release Docker Image 2 | 3 | on: 4 | push: 5 | tags: v** 6 | 7 | jobs: 8 | push_to_registries: 9 | name: Push Docker image to multiple registries 10 | runs-on: ubuntu-latest 11 | permissions: 12 | packages: write 13 | contents: read 14 | steps: 15 | - name: Check out the repo 16 | uses: actions/checkout@v4 17 | 18 | - name: set Environment Variables 19 | id: env 20 | run: | 21 | echo "NOW=$(date +'%F %Z %T')" >> $GITHUB_ENV 22 | 23 | - name: Docker meta 24 | id: meta 25 | uses: docker/metadata-action@v5 26 | with: 27 | # list of Docker images to use as base name for tags 28 | images: | 29 | offen/docker-volume-backup 30 | ghcr.io/offen/docker-volume-backup 31 | # define global behaviour for tags 32 | flavor: | 33 | latest=false 34 | # specify one tag which never gets set, to prevent the tag-attribute being empty, as it will fallback to a default 35 | tags: | 36 | # output v2.42.1-alpha.1 (incl. pre-releases) 37 | type=semver,pattern=v{{version}},enable=false 38 | labels: | 39 | org.opencontainers.image.title=${{github.event.repository.name}} 40 | org.opencontainers.image.description=Backup Docker volumes locally or to any S3, WebDAV, Azure Blob Storage, Dropbox or SSH compatible storage 41 | org.opencontainers.image.vendor=${{github.repository_owner}} 42 | org.opencontainers.image.licenses=MPL-2.0 43 | org.opencontainers.image.version=${{github.ref_name}} 44 | org.opencontainers.image.created=${{ env.NOW }} 45 | org.opencontainers.image.source=${{github.server_url}}/${{github.repository}} 46 | org.opencontainers.image.revision=${{github.sha}} 47 | org.opencontainers.image.url=https://offen.github.io/docker-volume-backup/ 48 | org.opencontainers.image.documentation=https://offen.github.io/docker-volume-backup/ 49 | 50 | - name: Set up QEMU 51 | uses: docker/setup-qemu-action@v2 52 | 53 | - name: Set up Docker Buildx 54 | uses: docker/setup-buildx-action@v2 55 | 56 | - name: Log in to Docker Hub 57 | uses: docker/login-action@v2 58 | with: 59 | username: ${{ secrets.DOCKER_USERNAME }} 60 | password: ${{ secrets.DOCKER_PASSWORD }} 61 | 62 | - name: Log in to GHCR 63 | uses: docker/login-action@v2 64 | with: 65 | registry: ghcr.io 66 | username: ${{ github.actor }} 67 | password: ${{ secrets.GITHUB_TOKEN }} 68 | 69 | - name: Extract Docker tags 70 | id: tags 71 | run: | 72 | version_tag="${{github.ref_name}}" 73 | tags=($version_tag) 74 | if [[ "$version_tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then 75 | # prerelease tags like `v2.0.0-alpha.1` should not be released as `latest` nor `v2` 76 | tags+=("latest") 77 | tags+=($(echo "$version_tag" | cut -d. -f1)) 78 | fi 79 | releases="" 80 | for tag in "${tags[@]}"; do 81 | releases="${releases:+$releases,}offen/docker-volume-backup:$tag,ghcr.io/offen/docker-volume-backup:$tag" 82 | done 83 | echo "releases=$releases" >> "$GITHUB_OUTPUT" 84 | 85 | - name: Build and push Docker images 86 | uses: docker/build-push-action@v5 87 | with: 88 | context: . 89 | push: true 90 | platforms: linux/amd64,linux/arm64,linux/arm/v7 91 | tags: ${{ steps.tags.outputs.releases }} 92 | labels: ${{ steps.meta.outputs.labels }} 93 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Run Integration Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-22.04 12 | steps: 13 | - uses: actions/checkout@v4 14 | 15 | - name: Set up Docker Buildx 16 | uses: docker/setup-buildx-action@v2 17 | 18 | - name: Run Tests 19 | working-directory: ./test 20 | run: | 21 | BUILD_IMAGE=1 ./test.sh 22 | -------------------------------------------------------------------------------- /.github/workflows/unit.yml: -------------------------------------------------------------------------------- 1 | name: Run Unit Tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-22.04 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: Setup Go 15 | uses: actions/setup-go@v4 16 | with: 17 | go-version: '1.24.x' 18 | - name: Install dependencies 19 | run: go mod download 20 | - name: Test with the Go CLI 21 | run: go test -v ./... 22 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters: 2 | # Enable specific linter 3 | # https://golangci-lint.run/usage/linters/#enabled-by-default 4 | enable: 5 | - staticcheck 6 | - govet 7 | output: 8 | formats: 9 | - format: colored-line-number 10 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2022 - offen.software 2 | # SPDX-License-Identifier: MPL-2.0 3 | 4 | FROM golang:1.24-alpine AS builder 5 | 6 | WORKDIR /app 7 | COPY . . 8 | RUN go mod download 9 | WORKDIR /app/cmd/backup 10 | RUN go build -o backup . 11 | 12 | FROM alpine:3.21 13 | 14 | WORKDIR /root 15 | 16 | RUN apk add --no-cache ca-certificates && \ 17 | chmod a+rw /var/lock 18 | 19 | COPY --from=builder /app/cmd/backup/backup /usr/bin/backup 20 | 21 | ENTRYPOINT ["/usr/bin/backup", "-foreground"] 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | offen.software logo 3 | 4 | 5 | # docker-volume-backup 6 | 7 | Backup Docker volumes locally or to any S3, WebDAV, Azure Blob Storage, Dropbox or SSH compatible storage. 8 | 9 | The [offen/docker-volume-backup](https://hub.docker.com/r/offen/docker-volume-backup) Docker image can be used as a lightweight (below 15MB) companion container to an existing Docker setup. 10 | It handles __recurring or one-off backups of Docker volumes__ to a __local directory__, __any S3, WebDAV, Azure Blob Storage, Dropbox or SSH compatible storage (or any combination thereof) and rotates away old backups__ if configured. It also supports __encrypting your backups using GPG__ and __sending notifications for (failed) backup runs__. 11 | 12 | Documentation is found at 13 | - [Quickstart](https://offen.github.io/docker-volume-backup) 14 | - [Configuration Reference](https://offen.github.io/docker-volume-backup/reference/) 15 | - [How Tos](https://offen.github.io/docker-volume-backup/how-tos/) 16 | - [Recipes](https://offen.github.io/docker-volume-backup/recipes/) 17 | 18 | --- 19 | 20 | ## Quickstart 21 | 22 | ### Recurring backups in a compose setup 23 | 24 | Add a `backup` service to your compose setup and mount the volumes you would like to see backed up: 25 | 26 | ```yml 27 | version: '3' 28 | 29 | services: 30 | volume-consumer: 31 | build: 32 | context: ./my-app 33 | volumes: 34 | - data:/var/my-app 35 | labels: 36 | # This means the container will be stopped during backup to ensure 37 | # backup integrity. You can omit this label if stopping during backup 38 | # not required. 39 | - docker-volume-backup.stop-during-backup=true 40 | 41 | backup: 42 | # In production, it is advised to lock your image tag to a proper 43 | # release version instead of using `latest`. 44 | # Check https://github.com/offen/docker-volume-backup/releases 45 | # for a list of available releases. 46 | image: offen/docker-volume-backup:latest 47 | restart: always 48 | env_file: ./backup.env # see below for configuration reference 49 | volumes: 50 | - data:/backup/my-app-backup:ro 51 | # Mounting the Docker socket allows the script to stop and restart 52 | # the container during backup. You can omit this if you don't want 53 | # to stop the container. In case you need to proxy the socket, you can 54 | # also provide a location by setting `DOCKER_HOST` in the container 55 | - /var/run/docker.sock:/var/run/docker.sock:ro 56 | # If you mount a local directory or volume to `/archive` a local 57 | # copy of the backup will be stored there. You can override the 58 | # location inside of the container by setting `BACKUP_ARCHIVE`. 59 | # You can omit this if you do not want to keep local backups. 60 | - /path/to/local_backups:/archive 61 | volumes: 62 | data: 63 | ``` 64 | 65 | ### One-off backups using Docker CLI 66 | 67 | To run a one time backup, mount the volume you would like to see backed up into a container and run the `backup` command: 68 | 69 | ```console 70 | docker run --rm \ 71 | -v data:/backup/data \ 72 | --env AWS_ACCESS_KEY_ID="" \ 73 | --env AWS_SECRET_ACCESS_KEY="" \ 74 | --env AWS_S3_BUCKET_NAME="" \ 75 | --entrypoint backup \ 76 | offen/docker-volume-backup:v2 77 | ``` 78 | 79 | Alternatively, pass a `--env-file` in order to use a full config as described [in the docs](https://offen.github.io/docker-volume-backup/reference/). 80 | 81 | ### Looking for help? 82 | 83 | In case your are looking for help or guidance on how to incorporate docker-volume-backup into your existing setup, consider [becoming a sponsor](https://github.com/sponsors/offen?frequency=one-time) and book a one hour consulting session. 84 | 85 | --- 86 | 87 | Copyright © 2024 offen.software and contributors. 88 | Distributed under the MPL-2.0 License. 89 | -------------------------------------------------------------------------------- /cmd/backup/archive.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | // Portions of this file are taken from package `targz`, Copyright (c) 2014 Fredrik Wallgren 5 | // Licensed under the MIT License: https://github.com/walle/targz/blob/57fe4206da5abf7dd3901b4af3891ec2f08c7b08/LICENSE 6 | 7 | package main 8 | 9 | import ( 10 | "archive/tar" 11 | "fmt" 12 | "io" 13 | "os" 14 | "path" 15 | "path/filepath" 16 | "runtime" 17 | "strings" 18 | 19 | "github.com/klauspost/compress/zstd" 20 | "github.com/klauspost/pgzip" 21 | "github.com/offen/docker-volume-backup/internal/errwrap" 22 | ) 23 | 24 | func createArchive(files []string, inputFilePath, outputFilePath string, compression string, compressionConcurrency int) error { 25 | _, outputFilePath, err := makeAbsolute(stripTrailingSlashes(inputFilePath), outputFilePath) 26 | if err != nil { 27 | return errwrap.Wrap(err, "error transposing given file paths") 28 | } 29 | if err := os.MkdirAll(filepath.Dir(outputFilePath), 0755); err != nil { 30 | return errwrap.Wrap(err, "error creating output file path") 31 | } 32 | 33 | if err := compress(files, outputFilePath, compression, compressionConcurrency); err != nil { 34 | return errwrap.Wrap(err, "error creating archive") 35 | } 36 | 37 | return nil 38 | } 39 | 40 | func stripTrailingSlashes(path string) string { 41 | if len(path) > 0 && path[len(path)-1] == '/' { 42 | path = path[0 : len(path)-1] 43 | } 44 | 45 | return path 46 | } 47 | 48 | func makeAbsolute(inputFilePath, outputFilePath string) (string, string, error) { 49 | inputFilePath, err := filepath.Abs(inputFilePath) 50 | if err == nil { 51 | outputFilePath, err = filepath.Abs(outputFilePath) 52 | } 53 | 54 | return inputFilePath, outputFilePath, err 55 | } 56 | 57 | func compress(paths []string, outFilePath, algo string, concurrency int) error { 58 | file, err := os.Create(outFilePath) 59 | if err != nil { 60 | return errwrap.Wrap(err, "error creating out file") 61 | } 62 | 63 | prefix := path.Dir(outFilePath) 64 | compressWriter, err := getCompressionWriter(file, algo, concurrency) 65 | if err != nil { 66 | return errwrap.Wrap(err, "error getting compression writer") 67 | } 68 | tarWriter := tar.NewWriter(compressWriter) 69 | 70 | for _, p := range paths { 71 | if err := writeTarball(p, tarWriter, prefix); err != nil { 72 | return errwrap.Wrap(err, fmt.Sprintf("error writing %s to archive", p)) 73 | } 74 | } 75 | 76 | err = tarWriter.Close() 77 | if err != nil { 78 | return errwrap.Wrap(err, "error closing tar writer") 79 | } 80 | 81 | err = compressWriter.Close() 82 | if err != nil { 83 | return errwrap.Wrap(err, "error closing compression writer") 84 | } 85 | 86 | err = file.Close() 87 | if err != nil { 88 | return errwrap.Wrap(err, "error closing file") 89 | } 90 | 91 | return nil 92 | } 93 | 94 | func getCompressionWriter(file *os.File, algo string, concurrency int) (io.WriteCloser, error) { 95 | switch algo { 96 | case "none": 97 | return &passThroughWriteCloser{file}, nil 98 | case "gz": 99 | w, err := pgzip.NewWriterLevel(file, 5) 100 | if err != nil { 101 | return nil, errwrap.Wrap(err, "gzip error") 102 | } 103 | 104 | if concurrency == 0 { 105 | concurrency = runtime.GOMAXPROCS(0) 106 | } 107 | 108 | if err := w.SetConcurrency(1<<20, concurrency); err != nil { 109 | return nil, errwrap.Wrap(err, "error setting concurrency") 110 | } 111 | 112 | return w, nil 113 | case "zst": 114 | compressWriter, err := zstd.NewWriter(file) 115 | if err != nil { 116 | return nil, errwrap.Wrap(err, "zstd error") 117 | } 118 | return compressWriter, nil 119 | default: 120 | return nil, errwrap.Wrap(nil, fmt.Sprintf("unsupported compression algorithm: %s", algo)) 121 | } 122 | } 123 | 124 | func writeTarball(path string, tarWriter *tar.Writer, prefix string) error { 125 | fileInfo, err := os.Lstat(path) 126 | if err != nil { 127 | return errwrap.Wrap(err, fmt.Sprintf("error getting file info for %s", path)) 128 | } 129 | 130 | if fileInfo.Mode()&os.ModeSocket == os.ModeSocket { 131 | return nil 132 | } 133 | 134 | var link string 135 | if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink { 136 | var err error 137 | if link, err = os.Readlink(path); err != nil { 138 | return errwrap.Wrap(err, fmt.Sprintf("error resolving symlink %s", path)) 139 | } 140 | } 141 | 142 | header, err := tar.FileInfoHeader(fileInfo, link) 143 | if err != nil { 144 | return errwrap.Wrap(err, "error getting file info header") 145 | } 146 | header.Name = strings.TrimPrefix(path, prefix) 147 | 148 | err = tarWriter.WriteHeader(header) 149 | if err != nil { 150 | return errwrap.Wrap(err, "error writing file info header") 151 | } 152 | 153 | if !fileInfo.Mode().IsRegular() { 154 | return nil 155 | } 156 | 157 | file, err := os.Open(path) 158 | if err != nil { 159 | return errwrap.Wrap(err, fmt.Sprintf("error opening %s", path)) 160 | } 161 | defer file.Close() 162 | 163 | _, err = io.Copy(tarWriter, file) 164 | if err != nil { 165 | return errwrap.Wrap(err, fmt.Sprintf("error copying %s to tar writer", path)) 166 | } 167 | 168 | return nil 169 | } 170 | 171 | type passThroughWriteCloser struct { 172 | target io.WriteCloser 173 | } 174 | 175 | func (p *passThroughWriteCloser) Write(b []byte) (int, error) { 176 | return p.target.Write(b) 177 | } 178 | 179 | func (p *passThroughWriteCloser) Close() error { 180 | return nil 181 | } 182 | -------------------------------------------------------------------------------- /cmd/backup/command.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "fmt" 8 | "log/slog" 9 | "os" 10 | "os/signal" 11 | "syscall" 12 | 13 | "github.com/offen/docker-volume-backup/internal/errwrap" 14 | "github.com/robfig/cron/v3" 15 | ) 16 | 17 | type command struct { 18 | logger *slog.Logger 19 | schedules []cron.EntryID 20 | cr *cron.Cron 21 | reload chan struct{} 22 | } 23 | 24 | func newCommand() *command { 25 | return &command{ 26 | logger: slog.New(slog.NewTextHandler(os.Stdout, nil)), 27 | } 28 | } 29 | 30 | // runAsCommand executes a backup run for each configuration that is available 31 | // and then returns 32 | func (c *command) runAsCommand() error { 33 | configurations, err := sourceConfiguration(configStrategyEnv) 34 | if err != nil { 35 | return errwrap.Wrap(err, "error loading env vars") 36 | } 37 | 38 | for _, config := range configurations { 39 | if err := runScript(config); err != nil { 40 | return errwrap.Wrap(err, "error running script") 41 | } 42 | } 43 | 44 | return nil 45 | } 46 | 47 | type foregroundOpts struct { 48 | profileCronExpression string 49 | } 50 | 51 | // runInForeground starts the program as a long running process, scheduling 52 | // a job for each configuration that is available. 53 | func (c *command) runInForeground(opts foregroundOpts) error { 54 | c.cr = cron.New( 55 | cron.WithParser( 56 | cron.NewParser( 57 | cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor, 58 | ), 59 | ), 60 | ) 61 | 62 | if err := c.schedule(configStrategyConfd); err != nil { 63 | return errwrap.Wrap(err, "error scheduling") 64 | } 65 | 66 | if opts.profileCronExpression != "" { 67 | if _, err := c.cr.AddFunc(opts.profileCronExpression, c.profile); err != nil { 68 | return errwrap.Wrap(err, "error adding profiling job") 69 | } 70 | } 71 | 72 | var quit = make(chan os.Signal, 1) 73 | c.reload = make(chan struct{}, 1) 74 | signal.Notify(quit, syscall.SIGTERM, syscall.SIGINT) 75 | c.cr.Start() 76 | 77 | for { 78 | select { 79 | case <-quit: 80 | ctx := c.cr.Stop() 81 | <-ctx.Done() 82 | return nil 83 | case <-c.reload: 84 | if err := c.schedule(configStrategyConfd); err != nil { 85 | return errwrap.Wrap(err, "error reloading configuration") 86 | } 87 | } 88 | } 89 | } 90 | 91 | // schedule wipes all existing schedules and enqueues all schedules available 92 | // using the given configuration strategy 93 | func (c *command) schedule(strategy configStrategy) error { 94 | for _, id := range c.schedules { 95 | c.cr.Remove(id) 96 | } 97 | 98 | configurations, err := sourceConfiguration(strategy) 99 | if err != nil { 100 | return errwrap.Wrap(err, "error sourcing configuration") 101 | } 102 | 103 | for _, cfg := range configurations { 104 | config := cfg 105 | id, err := c.cr.AddFunc(config.BackupCronExpression, func() { 106 | c.logger.Info( 107 | fmt.Sprintf( 108 | "Now running script on schedule %s", 109 | config.BackupCronExpression, 110 | ), 111 | ) 112 | 113 | if err := runScript(config); err != nil { 114 | c.logger.Error( 115 | fmt.Sprintf( 116 | "Unexpected error running schedule %s: %v", 117 | config.BackupCronExpression, 118 | errwrap.Unwrap(err), 119 | ), 120 | "error", 121 | err, 122 | ) 123 | } 124 | }) 125 | 126 | if err != nil { 127 | return errwrap.Wrap(err, fmt.Sprintf("error adding schedule %s", config.BackupCronExpression)) 128 | } 129 | c.logger.Info(fmt.Sprintf("Successfully scheduled backup %s with expression %s", config.source, config.BackupCronExpression)) 130 | if ok := checkCronSchedule(config.BackupCronExpression); !ok { 131 | c.logger.Warn( 132 | fmt.Sprintf("Scheduled cron expression %s will never run, is this intentional?", config.BackupCronExpression), 133 | ) 134 | } 135 | c.schedules = append(c.schedules, id) 136 | } 137 | 138 | return nil 139 | } 140 | 141 | // must exits the program when passed an error. It should be the only 142 | // place where the application exits forcefully. 143 | func (c *command) must(err error) { 144 | if err != nil { 145 | c.logger.Error( 146 | fmt.Sprintf("Fatal error running command: %v", errwrap.Unwrap(err)), 147 | "error", 148 | err, 149 | ) 150 | os.Exit(1) 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /cmd/backup/config_provider.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "bufio" 8 | "fmt" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | 13 | "github.com/joho/godotenv" 14 | "github.com/offen/docker-volume-backup/internal/errwrap" 15 | "github.com/offen/envconfig" 16 | shell "mvdan.cc/sh/v3/shell" 17 | ) 18 | 19 | type configStrategy string 20 | 21 | const ( 22 | configStrategyEnv configStrategy = "env" 23 | configStrategyConfd configStrategy = "confd" 24 | ) 25 | 26 | // sourceConfiguration returns a list of config objects using the given 27 | // strategy. It should be the single entrypoint for retrieving configuration 28 | // for all consumers. 29 | func sourceConfiguration(strategy configStrategy) ([]*Config, error) { 30 | switch strategy { 31 | case configStrategyEnv: 32 | c, err := loadConfigFromEnvVars() 33 | return []*Config{c}, err 34 | case configStrategyConfd: 35 | cs, err := loadConfigsFromEnvFiles("/etc/dockervolumebackup/conf.d") 36 | if err != nil { 37 | if os.IsNotExist(err) { 38 | return sourceConfiguration(configStrategyEnv) 39 | } 40 | return nil, errwrap.Wrap(err, "error loading config files") 41 | } 42 | return cs, nil 43 | default: 44 | return nil, errwrap.Wrap(nil, fmt.Sprintf("received unknown config strategy: %v", strategy)) 45 | } 46 | } 47 | 48 | // envProxy is a function that mimics os.LookupEnv but can read values from any other source 49 | type envProxy func(string) (string, bool) 50 | 51 | // loadConfig creates a config object using the given lookup function 52 | func loadConfig(lookup envProxy) (*Config, error) { 53 | envconfig.Lookup = func(key string) (string, bool) { 54 | value, okValue := lookup(key) 55 | location, okFile := lookup(key + "_FILE") 56 | 57 | switch { 58 | case okValue && !okFile: // only value 59 | return value, true 60 | case !okValue && okFile: // only file 61 | contents, err := os.ReadFile(location) 62 | if err != nil { 63 | return "", false 64 | } 65 | return string(contents), true 66 | case okValue && okFile: // both 67 | return "", false 68 | default: // neither, ignore 69 | return "", false 70 | } 71 | } 72 | 73 | var c = &Config{} 74 | if err := envconfig.Process("", c); err != nil { 75 | return nil, errwrap.Wrap(err, "failed to process configuration values") 76 | } 77 | 78 | return c, nil 79 | } 80 | 81 | func loadConfigFromEnvVars() (*Config, error) { 82 | c, err := loadConfig(os.LookupEnv) 83 | if err != nil { 84 | return nil, errwrap.Wrap(err, "error loading config from environment") 85 | } 86 | c.source = "from environment" 87 | return c, nil 88 | } 89 | 90 | func loadConfigsFromEnvFiles(directory string) ([]*Config, error) { 91 | items, err := os.ReadDir(directory) 92 | if err != nil { 93 | if os.IsNotExist(err) { 94 | return nil, err 95 | } 96 | return nil, errwrap.Wrap(err, "failed to read files from env directory") 97 | } 98 | 99 | configs := []*Config{} 100 | for _, item := range items { 101 | if item.IsDir() { 102 | continue 103 | } 104 | p := filepath.Join(directory, item.Name()) 105 | envFile, err := source(p) 106 | if err != nil { 107 | return nil, errwrap.Wrap(err, fmt.Sprintf("error reading config file %s", p)) 108 | } 109 | lookup := func(key string) (string, bool) { 110 | val, ok := envFile[key] 111 | if ok { 112 | return val, ok 113 | } 114 | return os.LookupEnv(key) 115 | } 116 | c, err := loadConfig(lookup) 117 | if err != nil { 118 | return nil, errwrap.Wrap(err, fmt.Sprintf("error loading config from file %s", p)) 119 | } 120 | c.source = item.Name() 121 | c.additionalEnvVars = envFile 122 | configs = append(configs, c) 123 | } 124 | 125 | return configs, nil 126 | } 127 | 128 | // source tries to mimic the pre v2.37.0 behavior of calling 129 | // `set +a; source $path; set -a` and returns the env vars as a map 130 | func source(path string) (map[string]string, error) { 131 | f, err := os.Open(path) 132 | if err != nil { 133 | return nil, errwrap.Wrap(err, fmt.Sprintf("error opening %s", path)) 134 | } 135 | 136 | result := map[string]string{} 137 | scanner := bufio.NewScanner(f) 138 | for scanner.Scan() { 139 | line := scanner.Text() 140 | line = strings.TrimSpace(line) 141 | if strings.HasPrefix(line, "#") { 142 | continue 143 | } 144 | withExpansion, err := shell.Expand(line, nil) 145 | if err != nil { 146 | return nil, errwrap.Wrap(err, "error expanding env") 147 | } 148 | m, err := godotenv.Unmarshal(withExpansion) 149 | if err != nil { 150 | return nil, errwrap.Wrap(err, fmt.Sprintf("error sourcing %s", path)) 151 | } 152 | for key, value := range m { 153 | currentValue, currentOk := os.LookupEnv(key) 154 | defer func() { 155 | if currentOk { 156 | os.Setenv(key, currentValue) 157 | return 158 | } 159 | os.Unsetenv(key) 160 | }() 161 | result[key] = value 162 | os.Setenv(key, value) 163 | } 164 | } 165 | return result, nil 166 | } 167 | -------------------------------------------------------------------------------- /cmd/backup/config_provider_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | func TestSource(t *testing.T) { 10 | tests := []struct { 11 | name string 12 | input string 13 | expectError bool 14 | expectedOutput map[string]string 15 | }{ 16 | { 17 | "default", 18 | "testdata/default.env", 19 | false, 20 | map[string]string{ 21 | "FOO": "bar", 22 | "BAZ": "qux", 23 | }, 24 | }, 25 | { 26 | "not found", 27 | "testdata/nope.env", 28 | true, 29 | nil, 30 | }, 31 | { 32 | "braces", 33 | "testdata/braces.env", 34 | false, 35 | map[string]string{ 36 | "FOO": "qux", 37 | "BAR": "xxx", 38 | "BAZ": "", 39 | }, 40 | }, 41 | { 42 | "expansion", 43 | "testdata/expansion.env", 44 | false, 45 | map[string]string{ 46 | "BAR": "xxx", 47 | "FOO": "xxx", 48 | "BAZ": "xxx", 49 | "QUX": "yyy", 50 | }, 51 | }, 52 | { 53 | "comments", 54 | "testdata/comments.env", 55 | false, 56 | map[string]string{ 57 | "BAR": "xxx", 58 | "BAZ": "yyy", 59 | }, 60 | }, 61 | } 62 | 63 | os.Setenv("QUX", "yyy") 64 | defer os.Unsetenv("QUX") 65 | 66 | for _, test := range tests { 67 | t.Run(test.name, func(t *testing.T) { 68 | result, err := source(test.input) 69 | if (err != nil) != test.expectError { 70 | t.Errorf("Unexpected error value %v", err) 71 | } 72 | if !reflect.DeepEqual(test.expectedOutput, result) { 73 | t.Errorf("Expected %v, got %v", test.expectedOutput, result) 74 | } 75 | }) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /cmd/backup/copy_archive.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "os" 8 | "path" 9 | 10 | "github.com/offen/docker-volume-backup/internal/errwrap" 11 | "golang.org/x/sync/errgroup" 12 | ) 13 | 14 | // copyArchive makes sure the backup file is copied to both local and remote locations 15 | // as per the given configuration. 16 | func (s *script) copyArchive() error { 17 | _, name := path.Split(s.file) 18 | if stat, err := os.Stat(s.file); err != nil { 19 | return errwrap.Wrap(err, "unable to stat backup file") 20 | } else { 21 | size := stat.Size() 22 | s.stats.BackupFile = BackupFileStats{ 23 | Size: uint64(size), 24 | Name: name, 25 | FullPath: s.file, 26 | } 27 | } 28 | 29 | eg := errgroup.Group{} 30 | for _, backend := range s.storages { 31 | b := backend 32 | eg.Go(func() error { 33 | return b.Copy(s.file) 34 | }) 35 | } 36 | if err := eg.Wait(); err != nil { 37 | return errwrap.Wrap(err, "error copying archive") 38 | } 39 | 40 | return nil 41 | } 42 | -------------------------------------------------------------------------------- /cmd/backup/create_archive.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "fmt" 8 | "io/fs" 9 | "path/filepath" 10 | 11 | "github.com/offen/docker-volume-backup/internal/errwrap" 12 | "github.com/otiai10/copy" 13 | ) 14 | 15 | // createArchive creates a tar archive of the configured backup location and 16 | // saves it to disk. 17 | func (s *script) createArchive() error { 18 | backupSources := s.c.BackupSources 19 | 20 | if s.c.BackupFromSnapshot { 21 | s.logger.Warn( 22 | "Using BACKUP_FROM_SNAPSHOT has been deprecated and will be removed in the next major version.", 23 | ) 24 | s.logger.Warn( 25 | "Please use `archive-pre` and `archive-post` commands to prepare your backup sources. Refer to the documentation for an upgrade guide.", 26 | ) 27 | backupSources = filepath.Join("/tmp", s.c.BackupSources) 28 | // copy before compressing guard against a situation where backup folder's content are still growing. 29 | s.registerHook(hookLevelPlumbing, func(error) error { 30 | if err := remove(backupSources); err != nil { 31 | return errwrap.Wrap(err, "error removing snapshot") 32 | } 33 | s.logger.Info( 34 | fmt.Sprintf("Removed snapshot `%s`.", backupSources), 35 | ) 36 | return nil 37 | }) 38 | if err := copy.Copy(s.c.BackupSources, backupSources, copy.Options{ 39 | PreserveTimes: true, 40 | PreserveOwner: true, 41 | }); err != nil { 42 | return errwrap.Wrap(err, "error creating snapshot") 43 | } 44 | s.logger.Info( 45 | fmt.Sprintf("Created snapshot of `%s` at `%s`.", s.c.BackupSources, backupSources), 46 | ) 47 | } 48 | 49 | tarFile := s.file 50 | s.registerHook(hookLevelPlumbing, func(error) error { 51 | if err := remove(tarFile); err != nil { 52 | return errwrap.Wrap(err, "error removing tar file") 53 | } 54 | s.logger.Info( 55 | fmt.Sprintf("Removed tar file `%s`.", tarFile), 56 | ) 57 | return nil 58 | }) 59 | 60 | backupPath, err := filepath.Abs(stripTrailingSlashes(backupSources)) 61 | if err != nil { 62 | return errwrap.Wrap(err, "error getting absolute path") 63 | } 64 | 65 | var filesEligibleForBackup []string 66 | if err := filepath.WalkDir(backupPath, func(path string, di fs.DirEntry, err error) error { 67 | if err != nil { 68 | return err 69 | } 70 | 71 | if s.c.BackupExcludeRegexp.Re != nil && s.c.BackupExcludeRegexp.Re.MatchString(path) { 72 | return nil 73 | } 74 | filesEligibleForBackup = append(filesEligibleForBackup, path) 75 | return nil 76 | }); err != nil { 77 | return errwrap.Wrap(err, "error walking filesystem tree") 78 | } 79 | 80 | if err := createArchive(filesEligibleForBackup, backupSources, tarFile, s.c.BackupCompression.String(), s.c.GzipParallelism.Int()); err != nil { 81 | return errwrap.Wrap(err, "error compressing backup folder") 82 | } 83 | 84 | s.logger.Info( 85 | fmt.Sprintf("Created backup of `%s` at `%s`.", backupSources, tarFile), 86 | ) 87 | return nil 88 | } 89 | -------------------------------------------------------------------------------- /cmd/backup/hooks.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "errors" 8 | "sort" 9 | 10 | "github.com/offen/docker-volume-backup/internal/errwrap" 11 | ) 12 | 13 | // hook contains a queued action that can be trigger them when the script 14 | // reaches a certain point (e.g. unsuccessful backup) 15 | type hook struct { 16 | level hookLevel 17 | action func(err error) error 18 | } 19 | 20 | type hookLevel int 21 | 22 | const ( 23 | hookLevelPlumbing hookLevel = iota 24 | hookLevelError 25 | hookLevelInfo 26 | ) 27 | 28 | var hookLevels = map[string]hookLevel{ 29 | "info": hookLevelInfo, 30 | "error": hookLevelError, 31 | } 32 | 33 | // registerHook adds the given action at the given level. 34 | func (s *script) registerHook(level hookLevel, action func(err error) error) { 35 | s.hooks = append(s.hooks, hook{level, action}) 36 | } 37 | 38 | // runHooks runs all hooks that have been registered using the 39 | // given levels in the defined ordering. In case executing a hook returns an 40 | // error, the following hooks will still be run before the function returns. 41 | func (s *script) runHooks(err error) error { 42 | sort.SliceStable(s.hooks, func(i, j int) bool { 43 | return s.hooks[i].level < s.hooks[j].level 44 | }) 45 | var actionErrors []error 46 | for _, hook := range s.hooks { 47 | if hook.level > s.hookLevel { 48 | continue 49 | } 50 | if actionErr := hook.action(err); actionErr != nil { 51 | actionErrors = append(actionErrors, errwrap.Wrap(actionErr, "error running hook")) 52 | } 53 | } 54 | if len(actionErrors) != 0 { 55 | return errors.Join(actionErrors...) 56 | } 57 | return nil 58 | } 59 | -------------------------------------------------------------------------------- /cmd/backup/lock.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "fmt" 8 | "time" 9 | 10 | "github.com/gofrs/flock" 11 | "github.com/offen/docker-volume-backup/internal/errwrap" 12 | ) 13 | 14 | // lock opens a lockfile at the given location, keeping it locked until the 15 | // caller invokes the returned release func. In case the lock is currently blocked 16 | // by another execution, it will repeatedly retry until the lock is available 17 | // or the given timeout is exceeded. 18 | func (s *script) lock(lockfile string) (func() error, error) { 19 | start := time.Now() 20 | defer func() { 21 | s.stats.LockedTime = time.Since(start) 22 | }() 23 | 24 | retry := time.NewTicker(5 * time.Second) 25 | defer retry.Stop() 26 | deadline := time.NewTimer(s.c.LockTimeout) 27 | defer deadline.Stop() 28 | 29 | fileLock := flock.New(lockfile) 30 | 31 | for { 32 | acquired, err := fileLock.TryLock() 33 | if err != nil { 34 | return noop, errwrap.Wrap(err, "error trying to lock") 35 | } 36 | if acquired { 37 | if s.encounteredLock { 38 | s.logger.Info("Acquired exclusive lock on subsequent attempt, ready to continue.") 39 | } 40 | return fileLock.Unlock, nil 41 | } 42 | 43 | if !s.encounteredLock { 44 | s.logger.Info( 45 | fmt.Sprintf( 46 | "Exclusive lock was not available on first attempt. Will retry until it becomes available or the timeout of %s is exceeded.", 47 | s.c.LockTimeout, 48 | ), 49 | ) 50 | s.encounteredLock = true 51 | } 52 | 53 | select { 54 | case <-retry.C: 55 | continue 56 | case <-deadline.C: 57 | return noop, errwrap.Wrap(nil, "timed out waiting for lockfile to become available") 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /cmd/backup/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021-2022 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "flag" 8 | ) 9 | 10 | func main() { 11 | foreground := flag.Bool("foreground", false, "run the tool in the foreground") 12 | profile := flag.String("profile", "", "collect runtime metrics and log them periodically on the given cron expression") 13 | flag.Parse() 14 | 15 | c := newCommand() 16 | if *foreground { 17 | opts := foregroundOpts{ 18 | profileCronExpression: *profile, 19 | } 20 | c.must(c.runInForeground(opts)) 21 | } else { 22 | c.must(c.runAsCommand()) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /cmd/backup/notifications.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "bytes" 8 | _ "embed" 9 | "encoding/json" 10 | "errors" 11 | "fmt" 12 | "os" 13 | "text/template" 14 | "time" 15 | 16 | sTypes "github.com/containrrr/shoutrrr/pkg/types" 17 | "github.com/offen/docker-volume-backup/internal/errwrap" 18 | ) 19 | 20 | //go:embed notifications.tmpl 21 | var defaultNotifications string 22 | 23 | // NotificationData data to be passed to the notification templates 24 | type NotificationData struct { 25 | Error error 26 | Config *Config 27 | Stats *Stats 28 | } 29 | 30 | // notify sends a notification using the given title and body templates. 31 | // Automatically creates notification data, adding the given error 32 | func (s *script) notify(titleTemplate string, bodyTemplate string, err error) error { 33 | params := NotificationData{ 34 | Error: err, 35 | Stats: s.stats, 36 | Config: s.c, 37 | } 38 | 39 | titleBuf := &bytes.Buffer{} 40 | if err := s.template.ExecuteTemplate(titleBuf, titleTemplate, params); err != nil { 41 | return errwrap.Wrap(err, fmt.Sprintf("error executing %s template", titleTemplate)) 42 | } 43 | 44 | bodyBuf := &bytes.Buffer{} 45 | if err := s.template.ExecuteTemplate(bodyBuf, bodyTemplate, params); err != nil { 46 | return errwrap.Wrap(err, fmt.Sprintf("error executing %s template", bodyTemplate)) 47 | } 48 | 49 | if err := s.sendNotification(titleBuf.String(), bodyBuf.String()); err != nil { 50 | return errwrap.Wrap(err, "error sending notification") 51 | } 52 | return nil 53 | } 54 | 55 | // notifyFailure sends a notification about a failed backup run 56 | func (s *script) notifyFailure(err error) error { 57 | return s.notify("title_failure", "body_failure", err) 58 | } 59 | 60 | // notifyFailure sends a notification about a successful backup run 61 | func (s *script) notifySuccess() error { 62 | return s.notify("title_success", "body_success", nil) 63 | } 64 | 65 | // sendNotification sends a notification to all configured third party services 66 | func (s *script) sendNotification(title, body string) error { 67 | var errs []error 68 | for _, result := range s.sender.Send(body, &sTypes.Params{"title": title}) { 69 | if result != nil { 70 | errs = append(errs, result) 71 | } 72 | } 73 | if len(errs) != 0 { 74 | return errwrap.Wrap(errors.Join(errs...), "error sending message") 75 | } 76 | return nil 77 | } 78 | 79 | var templateHelpers = template.FuncMap{ 80 | "formatTime": func(t time.Time) string { 81 | return t.Format(time.RFC3339) 82 | }, 83 | "formatBytesDec": func(bytes uint64) string { 84 | return formatBytes(bytes, true) 85 | }, 86 | "formatBytesBin": func(bytes uint64) string { 87 | return formatBytes(bytes, false) 88 | }, 89 | "env": os.Getenv, 90 | "toJson": toJson, 91 | "toPrettyJson": toPrettyJson, 92 | } 93 | 94 | // formatBytes converts an amount of bytes in a human-readable representation 95 | // the decimal parameter specifies if using powers of 1000 (decimal) or powers of 1024 (binary) 96 | func formatBytes(b uint64, decimal bool) string { 97 | unit := uint64(1024) 98 | format := "%.1f %ciB" 99 | if decimal { 100 | unit = uint64(1000) 101 | format = "%.1f %cB" 102 | } 103 | if b < unit { 104 | return fmt.Sprintf("%d B", b) 105 | } 106 | div, exp := unit, 0 107 | for n := b / unit; n >= unit; n /= unit { 108 | div *= unit 109 | exp++ 110 | } 111 | return fmt.Sprintf(format, float64(b)/float64(div), "kMGTPE"[exp]) 112 | } 113 | 114 | func toJson(v interface{}) string { 115 | var bytes []byte 116 | var err error 117 | if bytes, err = json.Marshal(v); err != nil { 118 | return fmt.Sprintf("failed to marshal JSON in notification template: %v", err) 119 | } 120 | return string(bytes) 121 | } 122 | 123 | func toPrettyJson(v interface{}) string { 124 | var bytes []byte 125 | var err error 126 | if bytes, err = json.MarshalIndent(v, "", " "); err != nil { 127 | return fmt.Sprintf("failed to marshal indent JSON in notification template: %v", err) 128 | } 129 | return string(bytes) 130 | } 131 | -------------------------------------------------------------------------------- /cmd/backup/notifications.tmpl: -------------------------------------------------------------------------------- 1 | {{ define "title_failure" -}} 2 | Failure running docker-volume-backup at {{ .Stats.StartTime | formatTime }} 3 | {{- end }} 4 | 5 | 6 | {{ define "body_failure" -}} 7 | Running docker-volume-backup failed with error: {{ .Error }} 8 | 9 | Log output of the failed run was: 10 | 11 | {{ .Stats.LogOutput }} 12 | {{- end }} 13 | 14 | 15 | {{ define "title_success" -}} 16 | Success running docker-volume-backup at {{ .Stats.StartTime | formatTime }} 17 | {{- end }} 18 | 19 | 20 | {{ define "body_success" -}} 21 | Running docker-volume-backup succeeded. 22 | 23 | Log output was: 24 | 25 | {{ .Stats.LogOutput }} 26 | {{- end }} 27 | -------------------------------------------------------------------------------- /cmd/backup/profile.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package main 5 | 6 | import "runtime" 7 | 8 | func (c *command) profile() { 9 | memStats := runtime.MemStats{} 10 | runtime.ReadMemStats(&memStats) 11 | c.logger.Info( 12 | "Collecting runtime information", 13 | "num_goroutines", 14 | runtime.NumGoroutine(), 15 | "memory_heap_alloc", 16 | formatBytes(memStats.HeapAlloc, false), 17 | "memory_heap_inuse", 18 | formatBytes(memStats.HeapInuse, false), 19 | "memory_heap_sys", 20 | formatBytes(memStats.HeapSys, false), 21 | "memory_heap_objects", 22 | memStats.HeapObjects, 23 | ) 24 | } 25 | -------------------------------------------------------------------------------- /cmd/backup/prune_backups.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "fmt" 8 | "slices" 9 | "strings" 10 | "time" 11 | 12 | "github.com/offen/docker-volume-backup/internal/errwrap" 13 | "golang.org/x/sync/errgroup" 14 | ) 15 | 16 | // pruneBackups rotates away backups from local and remote storages using 17 | // the given configuration. In case the given configuration would delete all 18 | // backups, it does nothing instead and logs a warning. 19 | func (s *script) pruneBackups() error { 20 | if s.c.BackupRetentionDays < 0 { 21 | return nil 22 | } 23 | 24 | deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays)).Add(s.c.BackupPruningLeeway) 25 | 26 | eg := errgroup.Group{} 27 | for _, backend := range s.storages { 28 | b := backend 29 | eg.Go(func() error { 30 | if skipPrune(b.Name(), s.c.BackupSkipBackendsFromPrune) { 31 | s.logger.Info( 32 | fmt.Sprintf("Skipping pruning for backend `%s`.", b.Name()), 33 | ) 34 | return nil 35 | } 36 | stats, err := b.Prune(deadline, s.c.BackupPruningPrefix) 37 | if err != nil { 38 | return err 39 | } 40 | s.stats.Lock() 41 | s.stats.Storages[b.Name()] = StorageStats{ 42 | Total: stats.Total, 43 | Pruned: stats.Pruned, 44 | } 45 | s.stats.Unlock() 46 | return nil 47 | }) 48 | } 49 | 50 | if err := eg.Wait(); err != nil { 51 | return errwrap.Wrap(err, "error pruning backups") 52 | } 53 | 54 | return nil 55 | } 56 | 57 | // skipPrune returns true if the given backend name is contained in the 58 | // list of skipped backends. 59 | func skipPrune(name string, skippedBackends []string) bool { 60 | return slices.ContainsFunc( 61 | skippedBackends, 62 | func(b string) bool { 63 | return strings.EqualFold(b, name) // ignore case on both sides 64 | }, 65 | ) 66 | } 67 | -------------------------------------------------------------------------------- /cmd/backup/run_script.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | "runtime/debug" 10 | 11 | "github.com/offen/docker-volume-backup/internal/errwrap" 12 | ) 13 | 14 | // runScript instantiates a new script object and orchestrates a backup run. 15 | // To ensure it runs mutually exclusive a global file lock is acquired before 16 | // it starts running. Any panic within the script will be recovered and returned 17 | // as an error. 18 | func runScript(c *Config) (err error) { 19 | defer func() { 20 | if derr := recover(); derr != nil { 21 | fmt.Printf("%s: %s\n", derr, debug.Stack()) 22 | asErr, ok := derr.(error) 23 | if ok { 24 | err = errwrap.Wrap(asErr, "unexpected panic running script") 25 | } else { 26 | err = errwrap.Wrap(nil, fmt.Sprintf("%v", derr)) 27 | } 28 | } 29 | }() 30 | 31 | s := newScript(c) 32 | 33 | unlock, lockErr := s.lock("/var/lock/dockervolumebackup.lock") 34 | if lockErr != nil { 35 | err = errwrap.Wrap(lockErr, "error acquiring file lock") 36 | return 37 | } 38 | defer func() { 39 | if derr := unlock(); derr != nil { 40 | err = errors.Join(err, errwrap.Wrap(derr, "error releasing file lock")) 41 | } 42 | }() 43 | 44 | unset, err := s.c.applyEnv() 45 | if err != nil { 46 | return errwrap.Wrap(err, "error applying env") 47 | } 48 | defer func() { 49 | if derr := unset(); derr != nil { 50 | err = errors.Join(err, errwrap.Wrap(derr, "error unsetting environment variables")) 51 | } 52 | }() 53 | 54 | if initErr := s.init(); initErr != nil { 55 | err = errwrap.Wrap(initErr, "error instantiating script") 56 | return 57 | } 58 | 59 | return func() (err error) { 60 | scriptErr := func() error { 61 | if err := s.withLabeledCommands(lifecyclePhaseArchive, func() (err error) { 62 | restartContainersAndServices, err := s.stopContainersAndServices() 63 | // The mechanism for restarting containers is not using hooks as it 64 | // should happen as soon as possible (i.e. before uploading backups or 65 | // similar). 66 | defer func() { 67 | if derr := restartContainersAndServices(); derr != nil { 68 | err = errors.Join(err, errwrap.Wrap(derr, "error restarting containers and services")) 69 | } 70 | }() 71 | if err != nil { 72 | return 73 | } 74 | err = s.createArchive() 75 | return 76 | })(); err != nil { 77 | return err 78 | } 79 | 80 | if err := s.withLabeledCommands(lifecyclePhaseProcess, s.encryptArchive)(); err != nil { 81 | return err 82 | } 83 | if err := s.withLabeledCommands(lifecyclePhaseCopy, s.copyArchive)(); err != nil { 84 | return err 85 | } 86 | if err := s.withLabeledCommands(lifecyclePhasePrune, s.pruneBackups)(); err != nil { 87 | return err 88 | } 89 | return nil 90 | }() 91 | 92 | if hookErr := s.runHooks(scriptErr); hookErr != nil { 93 | if scriptErr != nil { 94 | return errwrap.Wrap( 95 | nil, 96 | fmt.Sprintf( 97 | "error %v executing the script followed by %v calling the registered hooks", 98 | scriptErr, 99 | hookErr, 100 | ), 101 | ) 102 | } 103 | return errwrap.Wrap( 104 | hookErr, 105 | "the script ran successfully, but an error occurred calling the registered hooks", 106 | ) 107 | } 108 | if scriptErr != nil { 109 | return errwrap.Wrap(scriptErr, "error running script") 110 | } 111 | return nil 112 | }() 113 | } 114 | -------------------------------------------------------------------------------- /cmd/backup/stats.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "bytes" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | // ContainersStats stats about the docker containers 13 | type ContainersStats struct { 14 | All uint 15 | ToStop uint 16 | Stopped uint 17 | StopErrors uint 18 | } 19 | 20 | // ServicesStats contains info about Swarm services that have been 21 | // operated upon 22 | type ServicesStats struct { 23 | All uint 24 | ToScaleDown uint 25 | ScaledDown uint 26 | ScaleDownErrors uint 27 | } 28 | 29 | // BackupFileStats stats about the created backup file 30 | type BackupFileStats struct { 31 | Name string 32 | FullPath string 33 | Size uint64 34 | } 35 | 36 | // StorageStats stats about the status of an archival directory 37 | type StorageStats struct { 38 | Total uint 39 | Pruned uint 40 | PruneErrors uint 41 | } 42 | 43 | // Stats global stats regarding script execution 44 | type Stats struct { 45 | sync.Mutex 46 | StartTime time.Time 47 | EndTime time.Time 48 | TookTime time.Duration 49 | LockedTime time.Duration 50 | LogOutput *bytes.Buffer 51 | Containers ContainersStats 52 | Services ServicesStats 53 | BackupFile BackupFileStats 54 | Storages map[string]StorageStats 55 | } 56 | -------------------------------------------------------------------------------- /cmd/backup/stop_restart_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "testing" 7 | 8 | "github.com/docker/docker/api/types/swarm" 9 | "github.com/docker/docker/api/types/system" 10 | ) 11 | 12 | type mockInfoClient struct { 13 | result system.Info 14 | err error 15 | } 16 | 17 | func (m *mockInfoClient) Info(context.Context) (system.Info, error) { 18 | return m.result, m.err 19 | } 20 | 21 | func TestIsSwarm(t *testing.T) { 22 | tests := []struct { 23 | name string 24 | client *mockInfoClient 25 | expected bool 26 | expectError bool 27 | }{ 28 | { 29 | "swarm", 30 | &mockInfoClient{ 31 | result: system.Info{ 32 | Swarm: swarm.Info{ 33 | LocalNodeState: swarm.LocalNodeStateActive, 34 | }, 35 | }, 36 | }, 37 | true, 38 | false, 39 | }, 40 | { 41 | "compose", 42 | &mockInfoClient{ 43 | result: system.Info{ 44 | Swarm: swarm.Info{ 45 | LocalNodeState: swarm.LocalNodeStateInactive, 46 | }, 47 | }, 48 | }, 49 | false, 50 | false, 51 | }, 52 | { 53 | "balena", 54 | &mockInfoClient{ 55 | result: system.Info{ 56 | Swarm: swarm.Info{ 57 | LocalNodeState: "", 58 | }, 59 | }, 60 | }, 61 | false, 62 | false, 63 | }, 64 | { 65 | "error", 66 | &mockInfoClient{ 67 | err: errors.New("the dinosaurs escaped"), 68 | }, 69 | false, 70 | true, 71 | }, 72 | } 73 | 74 | for _, test := range tests { 75 | t.Run(test.name, func(t *testing.T) { 76 | result, err := isSwarm(test.client) 77 | if (err != nil) != test.expectError { 78 | t.Errorf("Unexpected error value %v", err) 79 | } 80 | if test.expected != result { 81 | t.Errorf("Expected %v, got %v", test.expected, result) 82 | } 83 | }) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /cmd/backup/testdata/braces.env: -------------------------------------------------------------------------------- 1 | FOO=${bar:-qux} 2 | BAR=xxx 3 | BAZ=$NOPE 4 | -------------------------------------------------------------------------------- /cmd/backup/testdata/comments.env: -------------------------------------------------------------------------------- 1 | # This is a comment about `why` things are here 2 | # FOO="${bar:-qux}" 3 | # e.g. `backup-$HOSTNAME-%Y-%m-%dT%H-%M-%S.tar.gz`. Expansion happens before` 4 | 5 | BAR=xxx 6 | 7 | BAZ=$QUX 8 | -------------------------------------------------------------------------------- /cmd/backup/testdata/default.env: -------------------------------------------------------------------------------- 1 | FOO=bar 2 | BAZ=qux 3 | -------------------------------------------------------------------------------- /cmd/backup/testdata/expansion.env: -------------------------------------------------------------------------------- 1 | BAR=xxx 2 | FOO=${BAR} 3 | BAZ=$BAR 4 | QUX=${QUX} 5 | -------------------------------------------------------------------------------- /cmd/backup/util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package main 5 | 6 | import ( 7 | "bytes" 8 | "fmt" 9 | "io" 10 | "os" 11 | "sync" 12 | "time" 13 | 14 | "github.com/offen/docker-volume-backup/internal/errwrap" 15 | "github.com/robfig/cron/v3" 16 | ) 17 | 18 | var noop = func() error { return nil } 19 | 20 | // remove removes the given file or directory from disk. 21 | func remove(location string) error { 22 | fi, err := os.Lstat(location) 23 | if err != nil { 24 | if os.IsNotExist(err) { 25 | return nil 26 | } 27 | return errwrap.Wrap(err, fmt.Sprintf("error checking for existence of `%s`", location)) 28 | } 29 | if fi.IsDir() { 30 | err = os.RemoveAll(location) 31 | } else { 32 | err = os.Remove(location) 33 | } 34 | if err != nil { 35 | return errwrap.Wrap(err, fmt.Sprintf("error removing `%s", location)) 36 | } 37 | return nil 38 | } 39 | 40 | // buffer takes an io.Writer and returns a wrapped version of the 41 | // writer that writes to both the original target as well as the returned buffer 42 | func buffer(w io.Writer) (io.Writer, *bytes.Buffer) { 43 | buffering := &bufferingWriter{buf: bytes.Buffer{}, writer: w} 44 | return buffering, &buffering.buf 45 | } 46 | 47 | type bufferingWriter struct { 48 | buf bytes.Buffer 49 | writer io.Writer 50 | } 51 | 52 | func (b *bufferingWriter) Write(p []byte) (n int, err error) { 53 | if n, err := b.buf.Write(p); err != nil { 54 | return n, errwrap.Wrap(err, "error writing to buffer") 55 | } 56 | return b.writer.Write(p) 57 | } 58 | 59 | type noopWriteCloser struct { 60 | io.Writer 61 | } 62 | 63 | func (noopWriteCloser) Close() error { 64 | return nil 65 | } 66 | 67 | type handledSwarmService struct { 68 | serviceID string 69 | initialReplicaCount uint64 70 | } 71 | 72 | type concurrentSlice[T any] struct { 73 | val []T 74 | sync.Mutex 75 | } 76 | 77 | func (c *concurrentSlice[T]) append(v T) { 78 | c.Lock() 79 | defer c.Unlock() 80 | c.val = append(c.val, v) 81 | } 82 | 83 | func (c *concurrentSlice[T]) value() []T { 84 | return c.val 85 | } 86 | 87 | // checkCronSchedule detects whether the given cron expression will actually 88 | // ever be executed or not. 89 | func checkCronSchedule(expression string) (ok bool) { 90 | defer func() { 91 | if err := recover(); err != nil { 92 | ok = false 93 | } 94 | }() 95 | sched, err := cron.ParseStandard(expression) 96 | if err != nil { 97 | ok = false 98 | return 99 | } 100 | now := time.Now() 101 | sched.Next(now) // panics when the cron would never run 102 | ok = true 103 | return 104 | } 105 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | _site 2 | .jekyll-cache 3 | -------------------------------------------------------------------------------- /docs/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem "jekyll", "~> 4.3.2" 4 | gem "just-the-docs", "0.6.1" 5 | -------------------------------------------------------------------------------- /docs/Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | addressable (2.8.5) 5 | public_suffix (>= 2.0.2, < 6.0) 6 | colorator (1.1.0) 7 | concurrent-ruby (1.2.2) 8 | em-websocket (0.5.3) 9 | eventmachine (>= 0.12.9) 10 | http_parser.rb (~> 0) 11 | eventmachine (1.2.7) 12 | ffi (1.15.5) 13 | forwardable-extended (2.6.0) 14 | http_parser.rb (0.8.0) 15 | i18n (1.14.1) 16 | concurrent-ruby (~> 1.0) 17 | jekyll (4.3.2) 18 | addressable (~> 2.4) 19 | colorator (~> 1.0) 20 | em-websocket (~> 0.5) 21 | i18n (~> 1.0) 22 | jekyll-sass-converter (>= 2.0, < 4.0) 23 | jekyll-watch (~> 2.0) 24 | kramdown (~> 2.3, >= 2.3.1) 25 | kramdown-parser-gfm (~> 1.0) 26 | liquid (~> 4.0) 27 | mercenary (>= 0.3.6, < 0.5) 28 | pathutil (~> 0.9) 29 | rouge (>= 3.0, < 5.0) 30 | safe_yaml (~> 1.0) 31 | terminal-table (>= 1.8, < 4.0) 32 | webrick (~> 1.7) 33 | jekyll-include-cache (0.2.1) 34 | jekyll (>= 3.7, < 5.0) 35 | jekyll-sass-converter (2.2.0) 36 | sassc (> 2.0.1, < 3.0) 37 | jekyll-seo-tag (2.8.0) 38 | jekyll (>= 3.8, < 5.0) 39 | jekyll-watch (2.2.1) 40 | listen (~> 3.0) 41 | just-the-docs (0.6.1) 42 | jekyll (>= 3.8.5) 43 | jekyll-include-cache 44 | jekyll-seo-tag (>= 2.0) 45 | rake (>= 12.3.1) 46 | kramdown (2.4.0) 47 | rexml 48 | kramdown-parser-gfm (1.1.0) 49 | kramdown (~> 2.0) 50 | liquid (4.0.4) 51 | listen (3.8.0) 52 | rb-fsevent (~> 0.10, >= 0.10.3) 53 | rb-inotify (~> 0.9, >= 0.9.10) 54 | mercenary (0.4.0) 55 | pathutil (0.16.2) 56 | forwardable-extended (~> 2.6) 57 | public_suffix (4.0.7) 58 | rake (13.0.6) 59 | rb-fsevent (0.11.2) 60 | rb-inotify (0.10.1) 61 | ffi (~> 1.0) 62 | rexml (3.3.9) 63 | rouge (3.30.0) 64 | safe_yaml (1.0.5) 65 | sassc (2.4.0) 66 | ffi (~> 1.9) 67 | terminal-table (3.0.2) 68 | unicode-display_width (>= 1.1.1, < 3) 69 | unicode-display_width (2.4.2) 70 | webrick (1.8.1) 71 | 72 | PLATFORMS 73 | ruby 74 | 75 | DEPENDENCIES 76 | jekyll (~> 4.3.2) 77 | just-the-docs (= 0.6.1) 78 | 79 | BUNDLED WITH 80 | 2.1.4 81 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # Documentation site 2 | 3 | This directory contains the sources for the documentation site published at . 4 | 5 | Assuming you have Ruby and [`bundler`][bundler] installed, you can run the site locally using the following commands: 6 | 7 | ``` 8 | bundle install 9 | bundle exec jekyll serve 10 | ``` 11 | 12 | Note that changes in `_config.yml` require a manual restart to take effect. 13 | 14 | [bundler]: https://bundler.io/ 15 | -------------------------------------------------------------------------------- /docs/_config.yml: -------------------------------------------------------------------------------- 1 | title: docker-volume-backup 2 | description: Documentation for the offen/docker-volume-backup Docker image. 3 | theme: just-the-docs 4 | 5 | url: https://offen.github.io/docker-volume-backup/ 6 | 7 | callouts_level: quiet 8 | callouts: 9 | highlight: 10 | color: yellow 11 | important: 12 | title: Important 13 | color: blue 14 | new: 15 | title: New 16 | color: green 17 | note: 18 | title: Note 19 | color: purple 20 | warning: 21 | title: Warning 22 | color: red 23 | 24 | aux_links: 25 | 'GitHub Repository': 26 | - https://github.com/offen/docker-volume-backup 27 | 28 | nav_external_links: 29 | - title: GitHub Repository 30 | url: https://github.com/offen/docker-volume-backup 31 | 32 | footer_content: >- 33 | Copyright © 2024 offen.software and contributors. 34 | Distributed under the MPL-2.0 License.
35 | Something missing, unclear or not working? Open an issue. 36 | -------------------------------------------------------------------------------- /docs/_sass/custom/custom.scss: -------------------------------------------------------------------------------- 1 | .site-title { 2 | font-size: unset !important; 3 | } 4 | 5 | .main-content pre { 6 | font-size: 1.1em; 7 | } 8 | -------------------------------------------------------------------------------- /docs/how-tos/automatically-prune-old-backups.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Automatically prune old backups 3 | layout: default 4 | parent: How Tos 5 | nav_order: 3 6 | --- 7 | 8 | # Automatically prune old backups 9 | 10 | When `BACKUP_RETENTION_DAYS` is configured, the command will check if there are any archives in the remote storage backend(s) or local archive that are older than the given retention value and rotate these backups away. 11 | 12 | {: .note } 13 | Be aware that this mechanism looks at __all files in the target bucket or archive__, which means that other files that are older than the given deadline are deleted as well. 14 | In case you need to use a target that cannot be used exclusively for your backups, you can configure `BACKUP_PRUNING_PREFIX` to limit which files are considered eligible for deletion: 15 | 16 | ```yml 17 | version: '3' 18 | 19 | services: 20 | # ... define other services using the `data` volume here 21 | backup: 22 | image: offen/docker-volume-backup:v2 23 | environment: 24 | BACKUP_FILENAME: backup-%Y-%m-%dT%H-%M-%S.tar.gz 25 | BACKUP_PRUNING_PREFIX: backup- 26 | BACKUP_RETENTION_DAYS: '7' 27 | volumes: 28 | - ${HOME}/backups:/archive 29 | - data:/backup/my-app-backup:ro 30 | - /var/run/docker.sock:/var/run/docker.sock:ro 31 | 32 | volumes: 33 | data: 34 | ``` 35 | -------------------------------------------------------------------------------- /docs/how-tos/define-different-retention-schedules.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Define different retention schedules 3 | layout: default 4 | parent: How Tos 5 | nav_order: 9 6 | --- 7 | 8 | # Define different retention schedules 9 | 10 | If you want to manage backup retention on different schedules, the most straight forward approach is to define a dedicated configuration for retention rule using a different prefix in the `BACKUP_FILENAME` parameter and then run them on different cron schedules. 11 | 12 | For example, if you wanted to keep daily backups for 7 days, weekly backups for a month, and retain monthly backups forever, you could create three configuration files and mount them into `/etc/dockervolumebackup/conf.d`: 13 | 14 | ```ini 15 | # 01daily.conf 16 | BACKUP_FILENAME="daily-backup-%Y-%m-%dT%H-%M-%S.tar.gz" 17 | # run every day at 2am 18 | BACKUP_CRON_EXPRESSION="0 2 * * *" 19 | BACKUP_PRUNING_PREFIX="daily-backup-" 20 | BACKUP_RETENTION_DAYS="7" 21 | ``` 22 | 23 | ```ini 24 | # 02weekly.conf 25 | BACKUP_FILENAME="weekly-backup-%Y-%m-%dT%H-%M-%S.tar.gz" 26 | # run every monday at 3am 27 | BACKUP_CRON_EXPRESSION="0 3 * * 1" 28 | BACKUP_PRUNING_PREFIX="weekly-backup-" 29 | BACKUP_RETENTION_DAYS="31" 30 | ``` 31 | 32 | ```ini 33 | # 03monthly.conf 34 | BACKUP_FILENAME="monthly-backup-%Y-%m-%dT%H-%M-%S.tar.gz" 35 | # run every 1st of a month at 4am 36 | BACKUP_CRON_EXPRESSION="0 4 1 * *" 37 | ``` 38 | 39 | {: .note } 40 | While it's possible to define colliding cron schedules for each of these configurations, you might need to adjust the value for `LOCK_TIMEOUT` in case your backups are large and might take longer than an hour. 41 | -------------------------------------------------------------------------------- /docs/how-tos/encrypt-backups-using-gpg.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Encrypt backups using GPG 3 | layout: default 4 | parent: How Tos 5 | nav_order: 7 6 | nav_exclude: true 7 | --- 8 | 9 | See: [Encrypt Backups](encrypt-backups) 10 | -------------------------------------------------------------------------------- /docs/how-tos/encrypt-backups.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Encrypting backups 3 | layout: default 4 | parent: How Tos 5 | nav_order: 7 6 | --- 7 | 8 | # Encrypting backups 9 | 10 | The image supports encrypting backups using one of two available methods: **GPG** or **[age](https://age-encryption.org/)** 11 | 12 | ## Using GPG encryption 13 | 14 | In case a `GPG_PASSPHRASE` or `GPG_PUBLIC_KEY_RING` environment variable is set, the backup archive will be encrypted using the given key and saved as a `.gpg` file instead. 15 | 16 | Assuming you have `gpg` installed, you can decrypt such a backup using (your OS will prompt for the passphrase before decryption can happen): 17 | 18 | ```console 19 | gpg -o backup.tar.gz -d backup.tar.gz.gpg 20 | ``` 21 | 22 | ## Using age encryption 23 | 24 | age allows backups to be encrypted with either a symmetric key (password) or a public key. One of those options are available for use. 25 | 26 | Given `AGE_PASSPHRASE` being provided, the backup archive will be encrypted with the passphrase and saved as a `.age` file instead. Refer to age documentation for how to properly decrypt. 27 | 28 | Given `AGE_PUBLIC_KEYS` being provided (allowing multiple by separating each public key with `,`), the backup archive will be encrypted with the provided public keys. It will also result in the archive being saved as a `.age` file. 29 | 30 | You can use SSH keys in addition to `age` keys for encryption; `AGE_PUBLIC_KEYS` accepts both. 31 | -------------------------------------------------------------------------------- /docs/how-tos/handle-file-uploads-using-third-party-tools.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Handle file uploads using third party tools 3 | layout: default 4 | parent: How Tos 5 | nav_order: 10 6 | --- 7 | 8 | # Handle file uploads using third party tools 9 | 10 | If you want to use an unsupported storage backend, or want to use a third party (e.g. rsync, rclone) tool for file uploads, you can build a Docker image containing the required binaries off this one, and call through to these in lifecycle hooks. 11 | 12 | For example, if you wanted to use `rsync`, define your Docker image like this: 13 | 14 | ```Dockerfile 15 | FROM offen/docker-volume-backup:v2 16 | 17 | RUN apk add rsync 18 | ``` 19 | 20 | Using this image, you can now omit configuring any of the supported storage backends, and instead define your own mechanism in a `docker-volume-backup.copy-post` label: 21 | 22 | ```yml 23 | version: '3' 24 | 25 | services: 26 | backup: 27 | image: your-custom-image 28 | restart: always 29 | environment: 30 | BACKUP_FILENAME: "daily-backup-%Y-%m-%dT%H-%M-%S.tar.gz" 31 | BACKUP_CRON_EXPRESSION: "0 2 * * *" 32 | labels: 33 | - docker-volume-backup.copy-post=/bin/sh -c 'rsync $$COMMAND_RUNTIME_ARCHIVE_FILEPATH /destination' 34 | volumes: 35 | - app_data:/backup/app_data:ro 36 | - /var/run/docker.sock:/var/run/docker.sock:ro 37 | 38 | # other services defined here ... 39 | volumes: 40 | app_data: 41 | ``` 42 | 43 | {: .note } 44 | Commands will be invoked with the filepath of the tar archive passed as `COMMAND_RUNTIME_BACKUP_FILEPATH`. 45 | -------------------------------------------------------------------------------- /docs/how-tos/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: How Tos 3 | layout: default 4 | nav_order: 3 5 | has_children: true 6 | --- 7 | 8 | ## How Tos 9 | -------------------------------------------------------------------------------- /docs/how-tos/manual-trigger.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Trigger a backup manually 3 | layout: default 4 | parent: How Tos 5 | nav_order: 8 6 | --- 7 | 8 | # Trigger a backup manually 9 | 10 | You can manually trigger a backup run outside of the defined cron schedule by executing the `backup` command inside the container: 11 | 12 | ```console 13 | docker exec backup 14 | ``` 15 | 16 | If the container is configured to run multiple schedules, you can source the respective conf file before invoking the command: 17 | 18 | ```console 19 | docker exec /bin/sh -c 'set -a; source /etc/dockervolumebackup/conf.d/myconf.env; set +a && backup' 20 | ``` 21 | -------------------------------------------------------------------------------- /docs/how-tos/replace-deprecated-backup-from-snapshot.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Replace deprecated BACKUP_FROM_SNAPSHOT usage 3 | layout: default 4 | parent: How Tos 5 | nav_order: 17 6 | --- 7 | 8 | # Replace deprecated `BACKUP_FROM_SNAPSHOT` usage 9 | 10 | Starting with version 2.15.0, the `BACKUP_FROM_SNAPSHOT` feature has been deprecated. 11 | If you need to prepare your sources before the backup is taken, use `archive-pre`, `archive-post` and an intermediate volume: 12 | 13 | ```yml 14 | version: '3' 15 | 16 | services: 17 | my_app: 18 | build: . 19 | volumes: 20 | - data:/var/my_app 21 | - backup:/tmp/backup 22 | labels: 23 | - docker-volume-backup.archive-pre=cp -r /var/my_app /tmp/backup/my-app 24 | - docker-volume-backup.archive-post=rm -rf /tmp/backup/my-app 25 | 26 | backup: 27 | image: offen/docker-volume-backup:v2 28 | environment: 29 | BACKUP_SOURCES: /tmp/backup 30 | volumes: 31 | - backup:/backup:ro 32 | - /var/run/docker.sock:/var/run/docker.sock:ro 33 | 34 | volumes: 35 | data: 36 | backup: 37 | ``` 38 | -------------------------------------------------------------------------------- /docs/how-tos/replace-deprecated-backup-stop-container-label.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Replace deprecated BACKUP_STOP_CONTAINER_LABEL setting 3 | layout: default 4 | parent: How Tos 5 | nav_order: 20 6 | --- 7 | 8 | # Replace deprecated `BACKUP_STOP_CONTAINER_LABEL` setting 9 | 10 | Version `v2.36.0` deprecated the `BACKUP_STOP_CONTAINER_LABEL` setting and renamed it `BACKUP_STOP_DURING_BACKUP_LABEL` which is supposed to signal that this will stop both containers _and_ services. 11 | Migrating is done by renaming the key for your custom value: 12 | 13 | ```diff 14 | env: 15 | - BACKUP_STOP_CONTAINER_LABEL: database 16 | + BACKUP_STOP_DURING_BACKUP_LABEL: database 17 | ``` 18 | 19 | The old key will stay supported until the next major version, but logs a warning each time a backup is taken. 20 | -------------------------------------------------------------------------------- /docs/how-tos/replace-deprecated-exec-labels.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Replace deprecated exec-pre and exec-post labels 3 | layout: default 4 | parent: How Tos 5 | nav_order: 18 6 | --- 7 | 8 | # Replace deprecated `exec-pre` and `exec-post` labels 9 | 10 | Version 2.19.0 introduced the option to run labeled commands at multiple points in time during the backup lifecycle. 11 | In order to be able to use more obvious terminology in the new labels, the existing `exec-pre` and `exec-post` labels have been deprecated. 12 | If you want to emulate the existing behavior, all you need to do is change `exec-pre` to `archive-pre` and `exec-post` to `archive-post`: 13 | 14 | ```diff 15 | labels: 16 | - - docker-volume-backup.exec-pre=cp -r /var/my_app /tmp/backup/my-app 17 | + - docker-volume-backup.archive-pre=cp -r /var/my_app /tmp/backup/my-app 18 | - - docker-volume-backup.exec-post=rm -rf /tmp/backup/my-app 19 | + - docker-volume-backup.archive-post=rm -rf /tmp/backup/my-app 20 | ``` 21 | 22 | The `EXEC_LABEL` setting and the `docker-volume-backup.exec-label` label stay as is. 23 | Check the additional documentation on running commands during the backup lifecycle to find out about further possibilities. 24 | -------------------------------------------------------------------------------- /docs/how-tos/restore-volumes-from-backup.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Restore volumes from a backup 3 | layout: default 4 | parent: How Tos 5 | nav_order: 6 6 | --- 7 | 8 | # Restore volumes from a backup 9 | 10 | In case you need to restore a volume from a backup, the most straight forward procedure to do so would be: 11 | 12 | - Stop the container(s) that are using the volume 13 | - Untar the backup you want to restore 14 | ```console 15 | tar -C /tmp -xvf backup.tar.gz 16 | ``` 17 | - Using a temporary once-off container, mount the volume (the example assumes it's named `data`) and copy over the backup. Make sure you copy the correct path level (this depends on how you mount your volume into the backup container), you might need to strip some leading elements 18 | ```console 19 | docker run -d --name temp_restore_container -v data:/backup_restore alpine 20 | docker cp /tmp/backup/data-backup temp_restore_container:/backup_restore 21 | docker stop temp_restore_container 22 | docker rm temp_restore_container 23 | ``` 24 | - Restart the container(s) that are using the volume 25 | 26 | Depending on your setup and the application(s) you are running, this might involve other steps to be taken still. 27 | 28 | --- 29 | 30 | If you want to rollback an entire volume to an earlier backup snapshot (recommended for database volumes): 31 | 32 | - Trigger a manual backup if necessary (see `Manually triggering a backup`). 33 | - Stop the container(s) that are using the volume. 34 | - If volume was initially created using docker-compose, find out exact volume name using: 35 | ```console 36 | docker volume ls 37 | ``` 38 | - Remove existing volume (the example assumes it's named `data`): 39 | ```console 40 | docker volume rm data 41 | ``` 42 | - Create new volume with the same name and restore a snapshot: 43 | ```console 44 | docker run --rm -it -v data:/backup/my-app-backup -v /path/to/local_backups:/archive:ro alpine tar -xvzf /archive/full_backup_filename.tar.gz 45 | ``` 46 | - Restart the container(s) that are using the volume. 47 | -------------------------------------------------------------------------------- /docs/how-tos/run-custom-commands.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Run custom commands during the backup lifecycle 3 | layout: default 4 | nav_order: 5 5 | parent: How Tos 6 | --- 7 | 8 | # Run custom commands during the backup lifecycle 9 | 10 | In certain scenarios it can be required to run specific commands before and after a backup is taken (e.g. dumping a database). 11 | When mounting the Docker socket into the `docker-volume-backup` container, you can define pre- and post-commands that will be run in the context of the target container (it is also possible to run commands inside the `docker-volume-backup` container itself using this feature). 12 | 13 | {: .important } 14 | In a multi-node Swarm setup, commands can currently only be run on the node the `offen/docker-volume-backup` container is running on. 15 | Labeled containers on other nodes are not visible to the backup command. 16 | 17 | Such commands are defined by specifying the command in a `docker-volume-backup.[step]-[pre|post]` label where `step` can be any of the following phases of a backup lifecycle: 18 | 19 | - `archive` (the tar archive is created) 20 | - `process` (the tar archive is processed, e.g. encrypted - optional) 21 | - `copy` (the tar archive is copied to all configured storages) 22 | - `prune` (existing backups are pruned based on the defined ruleset - optional) 23 | 24 | {: .note } 25 | So that the `docker-volume-backup` container can access the labels on other containers, it is necessary that the docker socket is mounted into 26 | the `docker-volume-backup` container as shown in the Quickstart example. 27 | 28 | Taking a database dump using `mysqldump` would look like this: 29 | 30 | ```yml 31 | version: '3' 32 | 33 | services: 34 | # ... define other services using the `data` volume here 35 | database: 36 | image: mariadb 37 | volumes: 38 | - backup_data:/tmp/backups 39 | labels: 40 | - docker-volume-backup.archive-pre=/bin/sh -c 'mysqldump --all-databases > /backups/dump.sql' 41 | 42 | volumes: 43 | backup_data: 44 | ``` 45 | 46 | {: .note } 47 | Due to Docker limitations, you currently cannot use any kind of redirection in these commands unless you pass the command to `/bin/sh -c` or similar. 48 | I.e. instead of using `echo "ok" > ok.txt` you will need to use `/bin/sh -c 'echo "ok" > ok.txt'`. 49 | 50 | If you have more than one `docker-volume-backup` container (possibly across several docker-compose environments) to backup or you are using 51 | multiple backup schedules, you will need to use `EXEC_LABEL` in the configuration and a `docker-volume-backup.exec-label` label on each 52 | container using custom commands to ensure that the commands are only run by the correct `docker-volume-backup` instance. 53 | 54 | {: .important } 55 | In case you use `EXEC_LABEL` together with configuration mounted from `conf.d` it's important to understand that a distinct `EXEC_LABEL` __should be set in each configuration__. 56 | Else, schedules that do not specify an `EXEC_LABEL` will still trigger commands on all containers with such labels, no matter whether they specify `docker-volume-backup.exec-label` or not. 57 | 58 | ```yml 59 | version: '3' 60 | 61 | services: 62 | database: 63 | image: mariadb 64 | volumes: 65 | - backup_data:/tmp/backups 66 | labels: 67 | - docker-volume-backup.archive-pre=/bin/sh -c 'mysqldump --all-databases > /tmp/volume/dump.sql' 68 | - docker-volume-backup.exec-label=database 69 | 70 | backup: 71 | image: offen/docker-volume-backup:v2 72 | environment: 73 | EXEC_LABEL: database 74 | volumes: 75 | - data:/backup/dump:ro 76 | - /var/run/docker.sock:/var/run/docker.sock:ro 77 | 78 | volumes: 79 | backup_data: 80 | ``` 81 | 82 | 83 | The backup procedure is guaranteed to wait for all `pre` or `post` commands to finish before proceeding. 84 | However, there are no guarantees about the order in which they are run, which could also happen concurrently. 85 | 86 | By default the backup command is executed by the user provided by the container's image. 87 | It is possible to specify a custom user that is used to run commands in dedicated labels with the format `docker-volume-backup.[step]-[pre|post].user`: 88 | 89 | ```yml 90 | version: '3' 91 | 92 | services: 93 | gitea: 94 | image: gitea/gitea 95 | volumes: 96 | - backup_data:/tmp 97 | labels: 98 | - docker-volume-backup.archive-pre.user=git 99 | - docker-volume-backup.archive-pre=/bin/bash -c 'cd /tmp; /usr/local/bin/gitea dump -c /data/gitea/conf/app.ini -R -f dump.zip' 100 | ``` 101 | 102 | Make sure the user exists and is present in `passwd` inside the target container. 103 | -------------------------------------------------------------------------------- /docs/how-tos/run-multiple-schedules.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Run multiple backup schedules in the same container 3 | layout: default 4 | parent: How Tos 5 | nav_order: 11 6 | --- 7 | 8 | # Run multiple backup schedules in the same container 9 | 10 | Multiple backup schedules with different configuration can be configured by mounting an arbitrary number of configuration files (using the `.env` format) into `/etc/dockervolumebackup/conf.d`: 11 | 12 | ```yml 13 | version: '3' 14 | 15 | services: 16 | # ... define other services using the `data` volume here 17 | backup: 18 | image: offen/docker-volume-backup:v2 19 | volumes: 20 | - data:/backup/my-app-backup:ro 21 | - /var/run/docker.sock:/var/run/docker.sock:ro 22 | - ./configuration:/etc/dockervolumebackup/conf.d 23 | 24 | volumes: 25 | data: 26 | ``` 27 | 28 | A separate cronjob will be created for each config file. 29 | If a configuration value is set both in the global environment as well as in the config file, the config file will take precedence. 30 | The `backup` command expects to run on an exclusive lock, so in case you provide the same or overlapping schedules in your cron expressions, the runs will still be executed serially, one after the other. 31 | The exact order of schedules that use the same cron expression is not specified. 32 | In case you need your schedules to overlap, you need to create a dedicated container for each schedule instead. 33 | When changing the configuration, you currently need to manually restart the container for the changes to take effect. 34 | 35 | Set `BACKUP_SOURCES` for each config file to control which subset of volume mounts gets backed up: 36 | 37 | ```yml 38 | # With a volume configuration like this: 39 | volumes: 40 | - /var/run/docker.sock:/var/run/docker.sock:ro 41 | - ./configuration:/etc/dockervolumebackup/conf.d 42 | - app1_data:/backup/app1_data:ro 43 | - app2_data:/backup/app2_data:ro 44 | ``` 45 | 46 | ```ini 47 | # In the 1st config file: 48 | BACKUP_SOURCES=/backup/app1_data 49 | 50 | # In the 2nd config file: 51 | BACKUP_SOURCES=/backup/app2_data 52 | ``` 53 | -------------------------------------------------------------------------------- /docs/how-tos/set-container-timezone.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Set the timezone the container runs in 3 | layout: default 4 | parent: How Tos 5 | nav_order: 8 6 | --- 7 | 8 | # Set the timezone the container runs in 9 | 10 | By default a container based on this image will run in the UTC timezone. 11 | As the image is designed to be as small as possible, additional timezone data is not included. 12 | In case you want to run your cron rules in your local timezone (respecting DST and similar), you can mount your Docker host's `/etc/timezone` and `/etc/localtime` in read-only mode: 13 | 14 | ```yml 15 | version: '3' 16 | 17 | services: 18 | backup: 19 | image: offen/docker-volume-backup:v2 20 | volumes: 21 | - data:/backup/my-app-backup:ro 22 | - /etc/timezone:/etc/timezone:ro 23 | - /etc/localtime:/etc/localtime:ro 24 | 25 | volumes: 26 | data: 27 | ``` 28 | -------------------------------------------------------------------------------- /docs/how-tos/set-up-dropbox.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Set up Dropbox storage backend 3 | layout: default 4 | parent: How Tos 5 | nav_order: 12 6 | --- 7 | 8 | # Set up Dropbox storage backend 9 | 10 | ## Acquiring authentication tokens 11 | 12 | 1. Create a new Dropbox App in the [App Console](https://www.dropbox.com/developers/apps) 13 | 2. Open your new Dropbox App and set `DROPBOX_APP_KEY` and `DROPBOX_APP_SECRET` in your environment (e.g. docker-compose.yml) accordingly 14 | 3. Click on `Permissions` in your app and make sure, that the following permissions are cranted (or more): 15 | - `files.metadata.write` 16 | - `files.metadata.read` 17 | - `files.content.write` 18 | - `files.content.read` 19 | 4. Replace APPKEY in `https://www.dropbox.com/oauth2/authorize?client_id=APPKEY&token_access_type=offline&response_type=code` with the app key from step 2 20 | 5. Visit the URL and confirm the access of your app. This gives you an `auth code` -> save it somewhere! 21 | 6. Replace AUTHCODE, APPKEY, APPSECRET accordingly and perform the request: 22 | ``` 23 | curl https://api.dropbox.com/oauth2/token \ 24 | -d code=AUTHCODE \ 25 | -d grant_type=authorization_code \ 26 | -d client_id=APPKEY \ 27 | -d client_secret=APPSECRET 28 | ``` 29 | 7. Execute the request. You will get a JSON formatted reply. Use the value of the `refresh_token` for the last environment variable `DROPBOX_REFRESH_TOKEN` 30 | 8. You should now have `DROPBOX_APP_KEY`, `DROPBOX_APP_SECRET` and `DROPBOX_REFRESH_TOKEN` set. These don't expire. 31 | 32 | Note: Using the "Generated access token" in the app console is not supported, as it is only very short lived and therefore not suitable for an automatic backup solution. The refresh token handles this automatically - the setup procedure above is only needed once. 33 | 34 | ## Other parameters 35 | 36 | Important: If you chose `App folder` access during the creation of your Dropbox app in step 1 above, `DROPBOX_REMOTE_PATH` will be a relative path under the App folder! 37 | (_For example, DROPBOX_REMOTE_PATH=/somedir means the backup file will be uploaded to /Apps/myapp/somedir_) 38 | On the other hand if you chose `Full Dropbox` access, the value for `DROPBOX_REMOTE_PATH` will represent an absolute path inside your Dropbox storage area. 39 | (_Still considering the same example above, the backup file will be uploaded to /somedir in your Dropbox root_) 40 | -------------------------------------------------------------------------------- /docs/how-tos/stop-containers-during-backup.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Stop containers during backup 3 | layout: default 4 | parent: How Tos 5 | nav_order: 1 6 | --- 7 | 8 | # Stop containers during backup 9 | 10 | {: .note } 11 | In case you are running Docker in Swarm mode, [dedicated documentation](./use-with-docker-swarm.html) on service and container restart applies. 12 | 13 | In many cases, it will be desirable to stop the services that are consuming the volume you want to backup in order to ensure data integrity. 14 | This image can automatically stop and restart containers and services. 15 | By default, any container that is labeled `docker-volume-backup.stop-during-backup=true` will be stopped before the backup is being taken and restarted once it has finished. 16 | 17 | In case you need more fine grained control about which containers should be stopped (e.g. when backing up multiple volumes on different schedules), you can set the `BACKUP_STOP_DURING_BACKUP_LABEL` environment variable and then use the same value for labeling: 18 | 19 | ```yml 20 | version: '3' 21 | 22 | services: 23 | app: 24 | # definition for app ... 25 | labels: 26 | - docker-volume-backup.stop-during-backup=service1 27 | 28 | backup: 29 | image: offen/docker-volume-backup:v2 30 | environment: 31 | BACKUP_STOP_DURING_BACKUP_LABEL: service1 32 | volumes: 33 | - data:/backup/my-app-backup:ro 34 | - /var/run/docker.sock:/var/run/docker.sock:ro 35 | 36 | volumes: 37 | data: 38 | ``` 39 | -------------------------------------------------------------------------------- /docs/how-tos/update-deprecated-email-config.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Update deprecated email configuration 3 | layout: default 4 | parent: How Tos 5 | nav_order: 19 6 | --- 7 | 8 | # Update deprecated email configuration 9 | 10 | Starting with version 2.6.0, configuring email notifications using `EMAIL_*` keys has been deprecated. 11 | Instead of providing multiple values using multiple keys, you can now provide a single URL for `NOTIFICATION_URLS`. 12 | 13 | Before: 14 | ```ini 15 | EMAIL_NOTIFICATION_RECIPIENT="you@example.com" 16 | EMAIL_NOTIFICATION_SENDER="no-reply@example.com" 17 | EMAIL_SMTP_HOST="posteo.de" 18 | EMAIL_SMTP_PASSWORD="secret" 19 | EMAIL_SMTP_USERNAME="me" 20 | EMAIL_SMTP_PORT="587" 21 | ``` 22 | 23 | After: 24 | ```ini 25 | NOTIFICATION_URLS=smtp://me:secret@posteo.de:587/?fromAddress=no-reply@example.com&toAddresses=you@example.com 26 | ``` 27 | -------------------------------------------------------------------------------- /docs/how-tos/use-as-non-root.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Use the image as a non-root user 3 | layout: default 4 | parent: How Tos 5 | nav_order: 16 6 | --- 7 | 8 | # Use the image as a non-root user 9 | 10 | {: .important } 11 | Running as a non-root user limits interaction with the Docker Daemon. 12 | If you want to stop and restart containers and services during backup, and the host's Docker daemon is running as root, you will also need to run this tool as root. 13 | 14 | By default, this image executes backups using the `root` user. 15 | In case you prefer to use a different user, you can use Docker's [`user`](https://docs.docker.com/engine/reference/run/#user) option, passing the user and group id: 16 | 17 | ```console 18 | docker run --rm \ 19 | -v data:/backup/data \ 20 | --env AWS_ACCESS_KEY_ID="" \ 21 | --env AWS_SECRET_ACCESS_KEY="" \ 22 | --env AWS_S3_BUCKET_NAME="" \ 23 | --entrypoint backup \ 24 | --user 1000:1000 \ 25 | offen/docker-volume-backup:v2 26 | ``` 27 | 28 | or in a compose file: 29 | 30 | ```yml 31 | services: 32 | backup: 33 | image: offen/docker-volume-backup:v2 34 | user: 1000:1000 35 | # further configuration omitted ... 36 | ``` 37 | -------------------------------------------------------------------------------- /docs/how-tos/use-custom-docker-host.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Use a custom Docker host 3 | layout: default 4 | parent: How Tos 5 | nav_order: 14 6 | --- 7 | 8 | # Use a custom Docker host 9 | 10 | If you are interfacing with Docker via TCP, set `DOCKER_HOST` to the correct URL. 11 | 12 | ```ini 13 | DOCKER_HOST=tcp://docker_socket_proxy:2375 14 | ``` 15 | 16 | If you do this as you seek to restrict access to the Docker socket, this tool is potentially calling the following Docker APIs: 17 | 18 | | API | When | 19 | |-|-| 20 | | `Info` | always | 21 | | `ContainerExecCreate` | running commands from `exec-labels` | 22 | | `ContainerExecAttach` | running commands from `exec-labels` | 23 | | `ContainerExecInspect` | running commands from `exec-labels` | 24 | | `ContainerList` | always | 25 | `ServiceList` | Docker engine is running in Swarm mode | 26 | | `ServiceInspect` | Docker engine is running in Swarm mode | 27 | | `ServiceUpdate` | Docker engine is running in Swarm mode and `stop-during-backup` is used | 28 | | `ConatinerStop` | `stop-during-backup` labels are applied to containers | 29 | | `ContainerStart` | `stop-during-backup` labels are applied to container | 30 | 31 | --- 32 | 33 | In case you are using [`docker-socket-proxy`][proxy], this means following permissions are required: 34 | 35 | | Permission | When | 36 | |-|-| 37 | | INFO | always required | 38 | | CONTAINERS | always required | 39 | | POST | required when using `stop-during-backup` or `exec` labels | 40 | | EXEC | required when using `exec`-labeled commands | 41 | | SERVICES | required when Docker Engine is running in Swarm mode | 42 | | NODES | required when labeling services `stop-during-backup` | 43 | | TASKS | required when labeling services `stop-during-backup` | 44 | 45 | [proxy]: https://github.com/Tecnativa/docker-socket-proxy 46 | -------------------------------------------------------------------------------- /docs/how-tos/use-rootless-docker.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Use with rootless Docker 3 | layout: default 4 | parent: How Tos 5 | nav_order: 15 6 | --- 7 | 8 | # Use with rootless Docker 9 | 10 | It's also possible to use this image with a [rootless Docker installation][rootless-docker]. 11 | Instead of mounting `/var/run/docker.sock`, mount the user-specific socket into the container: 12 | 13 | ```yml 14 | services: 15 | backup: 16 | image: offen/docker-volume-backup:v2 17 | # ... configuration omitted 18 | volumes: 19 | - backup:/backup:ro 20 | - /run/user/1000/docker.sock:/var/run/docker.sock:ro 21 | ``` 22 | 23 | [rootless-docker]: https://docs.docker.com/engine/security/rootless/ 24 | -------------------------------------------------------------------------------- /docs/how-tos/use-with-docker-swarm.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Use with Docker Swarm 3 | layout: default 4 | parent: How Tos 5 | nav_order: 13 6 | --- 7 | 8 | # Use with Docker Swarm 9 | 10 | {: .note } 11 | The mechanisms described in this page __do only apply when Docker is running in [Swarm mode][swarm]__. 12 | 13 | [swarm]: https://docs.docker.com/engine/swarm/ 14 | 15 | ## Stopping containers during backup 16 | 17 | Stopping and restarting containers during backup creation when running Docker in Swarm mode is supported in two ways. 18 | 19 | {: .important } 20 | Make sure you label your services and containers using only one of the describe approaches. 21 | In case the script encounters a container that is labeled and has a parent service that is also labeled, it will exit early. 22 | 23 | ### Scaling services down to zero before scaling back up 24 | 25 | When labeling a service in the `deploy` section, the following strategy for stopping and restarting will be used: 26 | 27 | - The service is scaled down to zero replicas 28 | - The backup is created 29 | - The service is scaled back up to the previous number of replicas 30 | 31 | {: .note } 32 | This approach will only work for services that are deployed in __replicated mode__. 33 | 34 | Such a service definition could look like: 35 | 36 | ```yml 37 | services: 38 | app: 39 | image: myorg/myimage:latest 40 | deploy: 41 | labels: 42 | - docker-volume-backup.stop-during-backup=true 43 | replicas: 2 44 | ``` 45 | 46 | ### Stopping the containers 47 | 48 | This approach bypasses the services and stops containers directly, creates the backup and restarts the containers again. 49 | As Docker Swarm would usually try to instantly restart containers that are manually stopped, this approach only works when using the `on-failure` restart policy. 50 | A restart policy of `always` is not compatible with this approach. 51 | 52 | Such a service definition could look like: 53 | 54 | ```yml 55 | services: 56 | app: 57 | image: myapp/myimage:latest 58 | labels: 59 | - docker-volume-backup.stop-during-backup=true 60 | deploy: 61 | replicas: 2 62 | restart_policy: 63 | condition: on-failure 64 | ``` 65 | 66 | --- 67 | 68 | ## Memory limit considerations 69 | 70 | When running in Swarm mode, it's also advised to set a hard memory limit on your service (~25MB should be enough in most cases, but if you backup large files above half a gigabyte or similar, you might have to raise this in case the backup exits with `Killed`): 71 | 72 | ```yml 73 | services: 74 | backup: 75 | image: offen/docker-volume-backup:v2 76 | deployment: 77 | resources: 78 | limits: 79 | memory: 25M 80 | ``` 81 | 82 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Home 3 | layout: home 4 | nav_order: 1 5 | --- 6 | 7 | # offen/docker-volume-backup 8 | {:.no_toc} 9 | 10 | Backup Docker volumes locally or to any S3, WebDAV, Azure Blob Storage, Dropbox or SSH compatible storage. 11 | {: .fs-6 .fw-300 } 12 | 13 | --- 14 | 15 | The [offen/docker-volume-backup](https://hub.docker.com/r/offen/docker-volume-backup) Docker image can be used as a lightweight (below 15MB) companion container to an existing Docker setup. 16 | It handles __recurring or one-off backups of Docker volumes__ to a __local directory__, __any S3, WebDAV, Azure Blob Storage, Dropbox or SSH compatible storage (or any combination thereof) and rotates away old backups__ if configured. It also supports __encrypting your backups using GPG__ and __sending notifications for (failed) backup runs__. 17 | 18 | {: .note } 19 | Code and documentation for `v1` versions are found on [this branch][v1-branch]. 20 | 21 | [v1-branch]: https://github.com/offen/docker-volume-backup/tree/v1 22 | 23 | --- 24 | 25 | 1. TOC 26 | {:toc} 27 | 28 | ## Quickstart 29 | 30 | ### Recurring backups in a compose setup 31 | 32 | Add a `backup` service to your compose setup and mount the volumes you would like to see backed up: 33 | 34 | ```yml 35 | version: '3' 36 | 37 | services: 38 | volume-consumer: 39 | build: 40 | context: ./my-app 41 | volumes: 42 | - data:/var/my-app 43 | labels: 44 | # This means the container will be stopped during backup to ensure 45 | # backup integrity. You can omit this label if stopping during backup 46 | # not required. 47 | - docker-volume-backup.stop-during-backup=true 48 | 49 | backup: 50 | # In production, it is advised to lock your image tag to a proper 51 | # release version instead of using `latest`. 52 | # Check https://github.com/offen/docker-volume-backup/releases 53 | # for a list of available releases. 54 | image: offen/docker-volume-backup:latest 55 | restart: always 56 | env_file: ./backup.env # see below for configuration reference 57 | volumes: 58 | - data:/backup/my-app-backup:ro 59 | # Mounting the Docker socket allows the script to stop and restart 60 | # the container during backup and to access the container labels to 61 | # specify custom commands. You can omit this if you don't want to 62 | # stop the container or run custom commands. In case you need to 63 | # proxy the socket, you can also provide a location by setting 64 | # `DOCKER_HOST` in the container 65 | - /var/run/docker.sock:/var/run/docker.sock:ro 66 | # If you mount a local directory or volume to `/archive` a local 67 | # copy of the backup will be stored there. You can override the 68 | # location inside of the container by setting `BACKUP_ARCHIVE`. 69 | # You can omit this if you do not want to keep local backups. 70 | - /path/to/local_backups:/archive 71 | volumes: 72 | data: 73 | ``` 74 | 75 | ### One-off backups using Docker CLI 76 | 77 | To run a one time backup, mount the volume you would like to see backed up into a container and run the `backup` command: 78 | 79 | ```console 80 | docker run --rm \ 81 | -v data:/backup/data \ 82 | --env AWS_ACCESS_KEY_ID="" \ 83 | --env AWS_SECRET_ACCESS_KEY="" \ 84 | --env AWS_S3_BUCKET_NAME="" \ 85 | --entrypoint backup \ 86 | offen/docker-volume-backup:v2 87 | ``` 88 | 89 | Alternatively, pass a `--env-file` in order to use a full config as described below. 90 | 91 | ## Available image registries 92 | 93 | This Docker image is published to both Docker Hub and the GitHub container registry. 94 | Depending on your preferences and needs, you can reference both `offen/docker-volume-backup` as well as `ghcr.io/offen/docker-volume-backup`: 95 | 96 | ``` 97 | docker pull offen/docker-volume-backup:v2 98 | docker pull ghcr.io/offen/docker-volume-backup:v2 99 | ``` 100 | 101 | Documentation references Docker Hub, but all examples will work using ghcr.io just as well. 102 | 103 | ## Supported Engines 104 | 105 | This tool is developed and tested against the Docker CE engine exclusively. 106 | While it may work against different implementations (e.g. Balena Engine), there are no guarantees about support for non-Docker engines. 107 | 108 | ## Differences to `jareware/docker-volume-backup` 109 | 110 | This image is heavily inspired by `jareware/docker-volume-backup`. We decided to publish this image as a simpler and more lightweight alternative because of the following requirements: 111 | 112 | - The original image is based on `ubuntu` and requires additional tools, making it heavy. 113 | This version is roughly 1/25 in compressed size (it's ~15MB). 114 | - The original image uses a shell script, when this version is written in Go. 115 | - The original image proposed to handle backup rotation through AWS S3 lifecycle policies. 116 | This image adds the option to rotate away old backups through the same command so this functionality can also be offered for non-AWS storage backends like MinIO. 117 | Local copies of backups can also be pruned once they reach a certain age. 118 | - InfluxDB specific functionality from the original image was removed. 119 | - `arm64` and `arm/v7` architectures are supported. 120 | - Docker in Swarm mode is supported. 121 | - Notifications on finished backups are supported. 122 | - IAM authentication through instance profiles is supported. 123 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/offen/docker-volume-backup 2 | 3 | go 1.24 4 | 5 | require ( 6 | filippo.io/age v1.2.1 7 | github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.0 8 | github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 9 | github.com/containrrr/shoutrrr v0.8.0 10 | github.com/cosiner/argv v0.1.0 11 | github.com/docker/cli v28.1.1+incompatible 12 | github.com/docker/docker v27.1.1+incompatible 13 | github.com/gofrs/flock v0.12.1 14 | github.com/joho/godotenv v1.5.1 15 | github.com/klauspost/compress v1.18.0 16 | github.com/leekchan/timeutil v0.0.0-20150802142658-28917288c48d 17 | github.com/minio/minio-go/v7 v7.0.92 18 | github.com/offen/envconfig v1.5.0 19 | github.com/otiai10/copy v1.14.1 20 | github.com/pkg/sftp v1.13.9 21 | github.com/robfig/cron/v3 v3.0.1 22 | github.com/studio-b12/gowebdav v0.10.0 23 | golang.org/x/crypto v0.38.0 24 | golang.org/x/oauth2 v0.30.0 25 | golang.org/x/sync v0.14.0 26 | mvdan.cc/sh/v3 v3.11.0 27 | ) 28 | 29 | require ( 30 | filippo.io/edwards25519 v1.1.0 // indirect 31 | github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect 32 | github.com/cloudflare/circl v1.6.0 // indirect 33 | github.com/containerd/log v0.1.0 // indirect 34 | github.com/distribution/reference v0.6.0 // indirect 35 | github.com/felixge/httpsnoop v1.0.4 // indirect 36 | github.com/go-ini/ini v1.67.0 // indirect 37 | github.com/go-logr/logr v1.4.1 // indirect 38 | github.com/go-logr/stdr v1.2.2 // indirect 39 | github.com/goccy/go-json v0.10.5 // indirect 40 | github.com/golang-jwt/jwt/v5 v5.2.2 // indirect 41 | github.com/golang/protobuf v1.5.4 // indirect 42 | github.com/minio/crc64nvme v1.0.1 // indirect 43 | github.com/moby/docker-image-spec v1.3.1 // indirect 44 | github.com/otiai10/mint v1.6.3 // indirect 45 | github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect 46 | github.com/tinylib/msgp v1.3.0 // indirect 47 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.51.0 // indirect 48 | go.opentelemetry.io/otel v1.26.0 // indirect 49 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 // indirect 50 | go.opentelemetry.io/otel/metric v1.26.0 // indirect 51 | go.opentelemetry.io/otel/sdk v1.26.0 // indirect 52 | go.opentelemetry.io/otel/trace v1.26.0 // indirect 53 | golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect 54 | google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de // indirect 55 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect 56 | ) 57 | 58 | require ( 59 | github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 // indirect 60 | github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect 61 | github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect 62 | github.com/Microsoft/go-winio v0.5.2 // indirect 63 | github.com/ProtonMail/go-crypto v1.3.0 64 | github.com/docker/go-connections v0.4.0 // indirect 65 | github.com/docker/go-units v0.4.0 // indirect 66 | github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 67 | github.com/dustin/go-humanize v1.0.1 // indirect 68 | github.com/fatih/color v1.17.0 // indirect 69 | github.com/gogo/protobuf v1.3.2 // indirect 70 | github.com/google/uuid v1.6.0 // indirect 71 | github.com/klauspost/cpuid/v2 v2.2.10 // indirect 72 | github.com/klauspost/pgzip v1.2.6 73 | github.com/kr/fs v0.1.0 // indirect 74 | github.com/kylelemons/godebug v1.1.0 // indirect 75 | github.com/mattn/go-colorable v0.1.13 // indirect 76 | github.com/mattn/go-isatty v0.0.20 // indirect 77 | github.com/minio/md5-simd v1.1.2 // indirect 78 | github.com/moby/term v0.0.0-20200312100748-672ec06f55cd // indirect 79 | github.com/morikuni/aec v1.0.0 // indirect 80 | github.com/opencontainers/go-digest v1.0.0 // indirect 81 | github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect 82 | github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect 83 | github.com/pkg/errors v0.9.1 // indirect 84 | github.com/rs/xid v1.6.0 // indirect 85 | github.com/sirupsen/logrus v1.9.3 // indirect 86 | golang.org/x/net v0.40.0 // indirect 87 | golang.org/x/sys v0.33.0 // indirect 88 | golang.org/x/text v0.25.0 // indirect 89 | gotest.tools/v3 v3.0.3 // indirect 90 | ) 91 | -------------------------------------------------------------------------------- /internal/errwrap/wrap.go: -------------------------------------------------------------------------------- 1 | // Copyright 2024 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package errwrap 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | "runtime" 10 | "strings" 11 | ) 12 | 13 | // Wrap wraps the given error using the given message while prepending 14 | // the name of the calling function, creating a poor man's stack trace 15 | func Wrap(err error, msg string) error { 16 | pc := make([]uintptr, 15) 17 | n := runtime.Callers(2, pc) 18 | frames := runtime.CallersFrames(pc[:n]) 19 | frame, _ := frames.Next() 20 | // strip full import paths and just use the package name 21 | chunks := strings.Split(frame.Function, "/") 22 | withCaller := fmt.Sprintf("%s: %s", chunks[len(chunks)-1], msg) 23 | if err == nil { 24 | return errors.New(withCaller) 25 | } 26 | return fmt.Errorf("%s: %w", withCaller, err) 27 | } 28 | 29 | // Unwrap receives an error and returns the last error in the chain of 30 | // wrapped errors 31 | func Unwrap(err error) error { 32 | if err == nil { 33 | return nil 34 | } 35 | for { 36 | u := errors.Unwrap(err) 37 | if u == nil { 38 | break 39 | } 40 | err = u 41 | } 42 | return err 43 | } 44 | -------------------------------------------------------------------------------- /internal/storage/azure/azure.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package azure 5 | 6 | import ( 7 | "bytes" 8 | "context" 9 | "errors" 10 | "fmt" 11 | "os" 12 | "path" 13 | "path/filepath" 14 | "strings" 15 | "sync" 16 | "text/template" 17 | "time" 18 | 19 | "github.com/Azure/azure-sdk-for-go/sdk/azidentity" 20 | "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" 21 | "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" 22 | "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" 23 | "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" 24 | "github.com/offen/docker-volume-backup/internal/errwrap" 25 | "github.com/offen/docker-volume-backup/internal/storage" 26 | ) 27 | 28 | type azureBlobStorage struct { 29 | *storage.StorageBackend 30 | client *azblob.Client 31 | uploadStreamOptions *blockblob.UploadStreamOptions 32 | containerName string 33 | } 34 | 35 | // Config contains values that define the configuration of an Azure Blob Storage. 36 | type Config struct { 37 | AccountName string 38 | ContainerName string 39 | PrimaryAccountKey string 40 | ConnectionString string 41 | Endpoint string 42 | RemotePath string 43 | AccessTier string 44 | } 45 | 46 | // NewStorageBackend creates and initializes a new Azure Blob Storage backend. 47 | func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) { 48 | if opts.PrimaryAccountKey != "" && opts.ConnectionString != "" { 49 | return nil, errwrap.Wrap(nil, "using primary account key and connection string are mutually exclusive") 50 | } 51 | 52 | endpointTemplate, err := template.New("endpoint").Parse(opts.Endpoint) 53 | if err != nil { 54 | return nil, errwrap.Wrap(err, "error parsing endpoint template") 55 | } 56 | var ep bytes.Buffer 57 | if err := endpointTemplate.Execute(&ep, opts); err != nil { 58 | return nil, errwrap.Wrap(err, "error executing endpoint template") 59 | } 60 | normalizedEndpoint := fmt.Sprintf("%s/", strings.TrimSuffix(ep.String(), "/")) 61 | 62 | var client *azblob.Client 63 | if opts.PrimaryAccountKey != "" { 64 | cred, err := azblob.NewSharedKeyCredential(opts.AccountName, opts.PrimaryAccountKey) 65 | if err != nil { 66 | return nil, errwrap.Wrap(err, "error creating shared key Azure credential") 67 | } 68 | 69 | client, err = azblob.NewClientWithSharedKeyCredential(normalizedEndpoint, cred, nil) 70 | if err != nil { 71 | return nil, errwrap.Wrap(err, "error creating azure client from primary account key") 72 | } 73 | } else if opts.ConnectionString != "" { 74 | client, err = azblob.NewClientFromConnectionString(opts.ConnectionString, nil) 75 | if err != nil { 76 | return nil, errwrap.Wrap(err, "error creating azure client from connection string") 77 | } 78 | } else { 79 | cred, err := azidentity.NewManagedIdentityCredential(nil) 80 | if err != nil { 81 | return nil, errwrap.Wrap(err, "error creating managed identity credential") 82 | } 83 | client, err = azblob.NewClient(normalizedEndpoint, cred, nil) 84 | if err != nil { 85 | return nil, errwrap.Wrap(err, "error creating azure client from managed identity") 86 | } 87 | } 88 | 89 | var uploadStreamOptions *blockblob.UploadStreamOptions 90 | if opts.AccessTier != "" { 91 | var found bool 92 | for _, t := range blob.PossibleAccessTierValues() { 93 | if string(t) == opts.AccessTier { 94 | found = true 95 | uploadStreamOptions = &blockblob.UploadStreamOptions{ 96 | AccessTier: &t, 97 | } 98 | } 99 | } 100 | if !found { 101 | return nil, errwrap.Wrap(nil, fmt.Sprintf("%s is not a possible access tier value", opts.AccessTier)) 102 | } 103 | } 104 | 105 | storage := azureBlobStorage{ 106 | client: client, 107 | uploadStreamOptions: uploadStreamOptions, 108 | containerName: opts.ContainerName, 109 | StorageBackend: &storage.StorageBackend{ 110 | DestinationPath: opts.RemotePath, 111 | Log: logFunc, 112 | }, 113 | } 114 | return &storage, nil 115 | } 116 | 117 | // Name returns the name of the storage backend 118 | func (b *azureBlobStorage) Name() string { 119 | return "Azure" 120 | } 121 | 122 | // Copy copies the given file to the storage backend. 123 | func (b *azureBlobStorage) Copy(file string) error { 124 | fileReader, err := os.Open(file) 125 | if err != nil { 126 | return errwrap.Wrap(err, fmt.Sprintf("error opening file %s", file)) 127 | } 128 | 129 | _, err = b.client.UploadStream( 130 | context.Background(), 131 | b.containerName, 132 | path.Join(b.DestinationPath, filepath.Base(file)), 133 | fileReader, 134 | b.uploadStreamOptions, 135 | ) 136 | if err != nil { 137 | return errwrap.Wrap(err, fmt.Sprintf("error uploading file %s", file)) 138 | } 139 | return nil 140 | } 141 | 142 | // Prune rotates away backups according to the configuration and provided 143 | // deadline for the Azure Blob storage backend. 144 | func (b *azureBlobStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) { 145 | lookupPrefix := path.Join(b.DestinationPath, pruningPrefix) 146 | pager := b.client.NewListBlobsFlatPager(b.containerName, &container.ListBlobsFlatOptions{ 147 | Prefix: &lookupPrefix, 148 | }) 149 | var matches []string 150 | var totalCount uint 151 | for pager.More() { 152 | resp, err := pager.NextPage(context.Background()) 153 | if err != nil { 154 | return nil, errwrap.Wrap(err, "error paging over blobs") 155 | } 156 | for _, v := range resp.Segment.BlobItems { 157 | totalCount++ 158 | if v.Properties.LastModified.Before(deadline) { 159 | matches = append(matches, *v.Name) 160 | } 161 | } 162 | } 163 | 164 | stats := &storage.PruneStats{ 165 | Total: totalCount, 166 | Pruned: uint(len(matches)), 167 | } 168 | 169 | pruneErr := b.DoPrune(b.Name(), len(matches), int(totalCount), deadline, func() error { 170 | wg := sync.WaitGroup{} 171 | wg.Add(len(matches)) 172 | var errs []error 173 | 174 | for _, match := range matches { 175 | name := match 176 | go func() { 177 | _, err := b.client.DeleteBlob(context.Background(), b.containerName, name, nil) 178 | if err != nil { 179 | errs = append(errs, err) 180 | } 181 | wg.Done() 182 | }() 183 | } 184 | wg.Wait() 185 | if len(errs) != 0 { 186 | return errors.Join(errs...) 187 | } 188 | return nil 189 | }) 190 | 191 | return stats, pruneErr 192 | } 193 | -------------------------------------------------------------------------------- /internal/storage/local/local.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package local 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | "io" 10 | "os" 11 | "path" 12 | "path/filepath" 13 | "time" 14 | 15 | "github.com/offen/docker-volume-backup/internal/errwrap" 16 | "github.com/offen/docker-volume-backup/internal/storage" 17 | ) 18 | 19 | type localStorage struct { 20 | *storage.StorageBackend 21 | latestSymlink string 22 | } 23 | 24 | // Config allows configuration of a local storage backend. 25 | type Config struct { 26 | ArchivePath string 27 | LatestSymlink string 28 | } 29 | 30 | // NewStorageBackend creates and initializes a new local storage backend. 31 | func NewStorageBackend(opts Config, logFunc storage.Log) storage.Backend { 32 | return &localStorage{ 33 | StorageBackend: &storage.StorageBackend{ 34 | DestinationPath: opts.ArchivePath, 35 | Log: logFunc, 36 | }, 37 | latestSymlink: opts.LatestSymlink, 38 | } 39 | } 40 | 41 | // Name return the name of the storage backend 42 | func (b *localStorage) Name() string { 43 | return "Local" 44 | } 45 | 46 | // Copy copies the given file to the local storage backend. 47 | func (b *localStorage) Copy(file string) error { 48 | _, name := path.Split(file) 49 | 50 | if err := copyFile(file, path.Join(b.DestinationPath, name)); err != nil { 51 | return errwrap.Wrap(err, "error copying file to archive") 52 | } 53 | b.Log(storage.LogLevelInfo, b.Name(), "Stored copy of backup `%s` in `%s`.", file, b.DestinationPath) 54 | 55 | if b.latestSymlink != "" { 56 | symlink := path.Join(b.DestinationPath, b.latestSymlink) 57 | if _, err := os.Lstat(symlink); err == nil { 58 | os.Remove(symlink) 59 | } 60 | if err := os.Symlink(name, symlink); err != nil { 61 | return errwrap.Wrap(err, "error creating latest symlink") 62 | } 63 | b.Log(storage.LogLevelInfo, b.Name(), "Created/Updated symlink `%s` for latest backup.", b.latestSymlink) 64 | } 65 | 66 | return nil 67 | } 68 | 69 | // Prune rotates away backups according to the configuration and provided deadline for the local storage backend. 70 | func (b *localStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) { 71 | globPattern := path.Join( 72 | b.DestinationPath, 73 | fmt.Sprintf("%s*", pruningPrefix), 74 | ) 75 | globMatches, err := filepath.Glob(globPattern) 76 | if err != nil { 77 | return nil, errwrap.Wrap( 78 | err, 79 | fmt.Sprintf( 80 | "error looking up matching files using pattern %s", 81 | globPattern, 82 | ), 83 | ) 84 | } 85 | 86 | var candidates []string 87 | for _, candidate := range globMatches { 88 | fi, err := os.Lstat(candidate) 89 | if err != nil { 90 | return nil, errwrap.Wrap( 91 | err, 92 | fmt.Sprintf( 93 | "error calling Lstat on file %s", 94 | candidate, 95 | ), 96 | ) 97 | } 98 | 99 | if !fi.IsDir() && fi.Mode()&os.ModeSymlink != os.ModeSymlink { 100 | candidates = append(candidates, candidate) 101 | } 102 | } 103 | 104 | var matches []string 105 | for _, candidate := range candidates { 106 | fi, err := os.Stat(candidate) 107 | if err != nil { 108 | return nil, errwrap.Wrap( 109 | err, 110 | fmt.Sprintf( 111 | "error calling stat on file %s", 112 | candidate, 113 | ), 114 | ) 115 | } 116 | if fi.ModTime().Before(deadline) { 117 | matches = append(matches, candidate) 118 | } 119 | } 120 | 121 | stats := &storage.PruneStats{ 122 | Total: uint(len(candidates)), 123 | Pruned: uint(len(matches)), 124 | } 125 | 126 | pruneErr := b.DoPrune(b.Name(), len(matches), len(candidates), deadline, func() error { 127 | var removeErrors []error 128 | for _, match := range matches { 129 | if err := os.Remove(match); err != nil { 130 | removeErrors = append(removeErrors, err) 131 | } 132 | } 133 | if len(removeErrors) != 0 { 134 | return errwrap.Wrap( 135 | errors.Join(removeErrors...), 136 | fmt.Sprintf( 137 | "%d error(s) deleting files", 138 | len(removeErrors), 139 | ), 140 | ) 141 | } 142 | return nil 143 | }) 144 | 145 | return stats, pruneErr 146 | } 147 | 148 | // copy creates a copy of the file located at `dst` at `src`. 149 | func copyFile(src, dst string) error { 150 | in, err := os.Open(src) 151 | if err != nil { 152 | return err 153 | } 154 | defer in.Close() 155 | 156 | out, err := os.Create(dst) 157 | if err != nil { 158 | return err 159 | } 160 | 161 | _, err = io.Copy(out, in) 162 | if err != nil { 163 | out.Close() 164 | return err 165 | } 166 | return out.Close() 167 | } 168 | -------------------------------------------------------------------------------- /internal/storage/s3/s3.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package s3 5 | 6 | import ( 7 | "context" 8 | "crypto/x509" 9 | "errors" 10 | "fmt" 11 | "os" 12 | "path" 13 | "time" 14 | 15 | "github.com/minio/minio-go/v7" 16 | "github.com/minio/minio-go/v7/pkg/credentials" 17 | "github.com/offen/docker-volume-backup/internal/errwrap" 18 | "github.com/offen/docker-volume-backup/internal/storage" 19 | ) 20 | 21 | type s3Storage struct { 22 | *storage.StorageBackend 23 | client *minio.Client 24 | bucket string 25 | storageClass string 26 | partSize int64 27 | } 28 | 29 | // Config contains values that define the configuration of a S3 backend. 30 | type Config struct { 31 | Endpoint string 32 | AccessKeyID string 33 | SecretAccessKey string 34 | IamRoleEndpoint string 35 | EndpointProto string 36 | EndpointInsecure bool 37 | RemotePath string 38 | BucketName string 39 | StorageClass string 40 | PartSize int64 41 | CACert *x509.Certificate 42 | } 43 | 44 | // NewStorageBackend creates and initializes a new S3/Minio storage backend. 45 | func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) { 46 | var creds *credentials.Credentials 47 | if opts.AccessKeyID != "" && opts.SecretAccessKey != "" { 48 | creds = credentials.NewStaticV4( 49 | opts.AccessKeyID, 50 | opts.SecretAccessKey, 51 | "", 52 | ) 53 | } else if opts.IamRoleEndpoint != "" { 54 | creds = credentials.NewIAM(opts.IamRoleEndpoint) 55 | } else { 56 | return nil, errwrap.Wrap(nil, "AWS_S3_BUCKET_NAME is defined, but no credentials were provided") 57 | } 58 | 59 | options := minio.Options{ 60 | Creds: creds, 61 | Secure: opts.EndpointProto == "https", 62 | } 63 | 64 | transport, err := minio.DefaultTransport(true) 65 | if err != nil { 66 | return nil, errwrap.Wrap(err, "failed to create default minio transport") 67 | } 68 | 69 | if opts.EndpointInsecure { 70 | if !options.Secure { 71 | return nil, errwrap.Wrap(nil, "AWS_ENDPOINT_INSECURE = true is only meaningful for https") 72 | } 73 | transport.TLSClientConfig.InsecureSkipVerify = true 74 | } else if opts.CACert != nil { 75 | if transport.TLSClientConfig.RootCAs == nil { 76 | transport.TLSClientConfig.RootCAs = x509.NewCertPool() 77 | } 78 | transport.TLSClientConfig.RootCAs.AddCert(opts.CACert) 79 | } 80 | options.Transport = transport 81 | 82 | mc, err := minio.New(opts.Endpoint, &options) 83 | if err != nil { 84 | return nil, errwrap.Wrap(err, "error setting up minio client") 85 | } 86 | 87 | return &s3Storage{ 88 | StorageBackend: &storage.StorageBackend{ 89 | DestinationPath: opts.RemotePath, 90 | Log: logFunc, 91 | }, 92 | client: mc, 93 | bucket: opts.BucketName, 94 | storageClass: opts.StorageClass, 95 | partSize: opts.PartSize, 96 | }, nil 97 | } 98 | 99 | // Name returns the name of the storage backend 100 | func (v *s3Storage) Name() string { 101 | return "S3" 102 | } 103 | 104 | // Copy copies the given file to the S3/Minio storage backend. 105 | func (b *s3Storage) Copy(file string) error { 106 | _, name := path.Split(file) 107 | putObjectOptions := minio.PutObjectOptions{ 108 | ContentType: "application/tar+gzip", 109 | StorageClass: b.storageClass, 110 | } 111 | 112 | if b.partSize > 0 { 113 | srcFileInfo, err := os.Stat(file) 114 | if err != nil { 115 | return errwrap.Wrap(err, "error reading the local file") 116 | } 117 | 118 | _, partSize, _, err := minio.OptimalPartInfo(srcFileInfo.Size(), uint64(b.partSize*1024*1024)) 119 | if err != nil { 120 | return errwrap.Wrap(err, "error computing the optimal s3 part size") 121 | } 122 | 123 | putObjectOptions.PartSize = uint64(partSize) 124 | } 125 | 126 | if _, err := b.client.FPutObject(context.Background(), b.bucket, path.Join(b.DestinationPath, name), file, putObjectOptions); err != nil { 127 | if errResp := minio.ToErrorResponse(err); errResp.Message != "" { 128 | return errwrap.Wrap( 129 | nil, 130 | fmt.Sprintf( 131 | "error uploading backup to remote storage: [Message]: '%s', [Code]: %s, [StatusCode]: %d", 132 | errResp.Message, 133 | errResp.Code, 134 | errResp.StatusCode, 135 | ), 136 | ) 137 | } 138 | return errwrap.Wrap(err, "error uploading backup to remote storage") 139 | } 140 | 141 | b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup `%s` to bucket `%s`.", file, b.bucket) 142 | 143 | return nil 144 | } 145 | 146 | // Prune rotates away backups according to the configuration and provided deadline for the S3/Minio storage backend. 147 | func (b *s3Storage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) { 148 | candidates := b.client.ListObjects(context.Background(), b.bucket, minio.ListObjectsOptions{ 149 | Prefix: path.Join(b.DestinationPath, pruningPrefix), 150 | Recursive: true, 151 | }) 152 | 153 | var matches []minio.ObjectInfo 154 | var lenCandidates int 155 | for candidate := range candidates { 156 | lenCandidates++ 157 | if candidate.Err != nil { 158 | return nil, errwrap.Wrap( 159 | candidate.Err, 160 | "error looking up candidates from remote storage", 161 | ) 162 | } 163 | if candidate.LastModified.Before(deadline) { 164 | matches = append(matches, candidate) 165 | } 166 | } 167 | 168 | stats := &storage.PruneStats{ 169 | Total: uint(lenCandidates), 170 | Pruned: uint(len(matches)), 171 | } 172 | 173 | pruneErr := b.DoPrune(b.Name(), len(matches), lenCandidates, deadline, func() error { 174 | objectsCh := make(chan minio.ObjectInfo) 175 | go func() { 176 | for _, match := range matches { 177 | objectsCh <- match 178 | } 179 | close(objectsCh) 180 | }() 181 | errChan := b.client.RemoveObjects(context.Background(), b.bucket, objectsCh, minio.RemoveObjectsOptions{}) 182 | var removeErrors []error 183 | for result := range errChan { 184 | if result.Err != nil { 185 | removeErrors = append(removeErrors, result.Err) 186 | } 187 | } 188 | if len(removeErrors) != 0 { 189 | return errors.Join(removeErrors...) 190 | } 191 | return nil 192 | }) 193 | 194 | return stats, pruneErr 195 | } 196 | -------------------------------------------------------------------------------- /internal/storage/ssh/ssh.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package ssh 5 | 6 | import ( 7 | "fmt" 8 | "io" 9 | "os" 10 | "path" 11 | "strings" 12 | "time" 13 | 14 | "github.com/offen/docker-volume-backup/internal/errwrap" 15 | "github.com/offen/docker-volume-backup/internal/storage" 16 | "github.com/pkg/sftp" 17 | "golang.org/x/crypto/ssh" 18 | ) 19 | 20 | type sshStorage struct { 21 | *storage.StorageBackend 22 | client *ssh.Client 23 | sftpClient *sftp.Client 24 | hostName string 25 | } 26 | 27 | // Config allows to configure a SSH backend. 28 | type Config struct { 29 | HostName string 30 | Port string 31 | User string 32 | Password string 33 | IdentityFile string 34 | IdentityPassphrase string 35 | RemotePath string 36 | } 37 | 38 | // NewStorageBackend creates and initializes a new SSH storage backend. 39 | func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) { 40 | var authMethods []ssh.AuthMethod 41 | 42 | if opts.Password != "" { 43 | authMethods = append(authMethods, ssh.Password(opts.Password)) 44 | } 45 | 46 | if _, err := os.Stat(opts.IdentityFile); err == nil { 47 | key, err := os.ReadFile(opts.IdentityFile) 48 | if err != nil { 49 | return nil, errwrap.Wrap(nil, "error reading the private key") 50 | } 51 | 52 | var signer ssh.Signer 53 | if opts.IdentityPassphrase != "" { 54 | signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(opts.IdentityPassphrase)) 55 | if err != nil { 56 | return nil, errwrap.Wrap(nil, "error parsing the encrypted private key") 57 | } 58 | authMethods = append(authMethods, ssh.PublicKeys(signer)) 59 | } else { 60 | signer, err = ssh.ParsePrivateKey(key) 61 | if err != nil { 62 | return nil, errwrap.Wrap(nil, "error parsing the private key") 63 | } 64 | authMethods = append(authMethods, ssh.PublicKeys(signer)) 65 | } 66 | } 67 | 68 | sshClientConfig := &ssh.ClientConfig{ 69 | User: opts.User, 70 | Auth: authMethods, 71 | HostKeyCallback: ssh.InsecureIgnoreHostKey(), 72 | } 73 | sshClient, err := ssh.Dial("tcp", fmt.Sprintf("%s:%s", opts.HostName, opts.Port), sshClientConfig) 74 | 75 | if err != nil { 76 | return nil, errwrap.Wrap(err, "error creating ssh client") 77 | } 78 | _, _, err = sshClient.SendRequest("keepalive", false, nil) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | sftpClient, err := sftp.NewClient(sshClient, 84 | sftp.UseConcurrentReads(true), 85 | sftp.UseConcurrentWrites(true), 86 | sftp.MaxConcurrentRequestsPerFile(64), 87 | ) 88 | if err != nil { 89 | return nil, errwrap.Wrap(err, "error creating sftp client") 90 | } 91 | 92 | return &sshStorage{ 93 | StorageBackend: &storage.StorageBackend{ 94 | DestinationPath: opts.RemotePath, 95 | Log: logFunc, 96 | }, 97 | client: sshClient, 98 | sftpClient: sftpClient, 99 | hostName: opts.HostName, 100 | }, nil 101 | } 102 | 103 | // Name returns the name of the storage backend 104 | func (b *sshStorage) Name() string { 105 | return "SSH" 106 | } 107 | 108 | // Copy copies the given file to the SSH storage backend. 109 | func (b *sshStorage) Copy(file string) error { 110 | source, err := os.Open(file) 111 | _, name := path.Split(file) 112 | if err != nil { 113 | return errwrap.Wrap(err, " error reading the file to be uploaded") 114 | } 115 | defer source.Close() 116 | 117 | destination, err := b.sftpClient.Create(path.Join(b.DestinationPath, name)) 118 | if err != nil { 119 | return errwrap.Wrap(err, "error creating file") 120 | } 121 | defer destination.Close() 122 | 123 | chunk := make([]byte, 1e9) 124 | for { 125 | num, err := source.Read(chunk) 126 | if err == io.EOF { 127 | tot, err := destination.Write(chunk[:num]) 128 | if err != nil { 129 | return errwrap.Wrap(err, "error uploading the file") 130 | } 131 | 132 | if tot != len(chunk[:num]) { 133 | return errwrap.Wrap(nil, "failed to write stream") 134 | } 135 | 136 | break 137 | } 138 | 139 | if err != nil { 140 | return errwrap.Wrap(err, "error uploading the file") 141 | } 142 | 143 | tot, err := destination.Write(chunk[:num]) 144 | if err != nil { 145 | return errwrap.Wrap(err, "error uploading the file") 146 | } 147 | 148 | if tot != len(chunk[:num]) { 149 | return errwrap.Wrap(nil, "failed to write stream") 150 | } 151 | } 152 | 153 | b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup `%s` to '%s' at path '%s'.", file, b.hostName, b.DestinationPath) 154 | 155 | return nil 156 | } 157 | 158 | // Prune rotates away backups according to the configuration and provided deadline for the SSH storage backend. 159 | func (b *sshStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) { 160 | candidates, err := b.sftpClient.ReadDir(b.DestinationPath) 161 | if err != nil { 162 | return nil, errwrap.Wrap(err, "error reading directory") 163 | } 164 | 165 | var matches []string 166 | var numCandidates int 167 | for _, candidate := range candidates { 168 | if candidate.IsDir() || !strings.HasPrefix(candidate.Name(), pruningPrefix) { 169 | continue 170 | } 171 | 172 | numCandidates++ 173 | if candidate.ModTime().Before(deadline) { 174 | matches = append(matches, candidate.Name()) 175 | } 176 | } 177 | 178 | stats := &storage.PruneStats{ 179 | Total: uint(numCandidates), 180 | Pruned: uint(len(matches)), 181 | } 182 | 183 | pruneErr := b.DoPrune(b.Name(), len(matches), numCandidates, deadline, func() error { 184 | for _, match := range matches { 185 | p := path.Join(b.DestinationPath, match) 186 | if err := b.sftpClient.Remove(p); err != nil { 187 | return errwrap.Wrap(err, fmt.Sprintf("error removing file %s", p)) 188 | } 189 | } 190 | return nil 191 | }) 192 | 193 | return stats, pruneErr 194 | } 195 | -------------------------------------------------------------------------------- /internal/storage/storage.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package storage 5 | 6 | import ( 7 | "time" 8 | 9 | "github.com/offen/docker-volume-backup/internal/errwrap" 10 | ) 11 | 12 | // Backend is an interface for defining functions which all storage providers support. 13 | type Backend interface { 14 | Copy(file string) error 15 | Prune(deadline time.Time, pruningPrefix string) (*PruneStats, error) 16 | Name() string 17 | } 18 | 19 | // StorageBackend is a generic type of storage. Everything here are common properties of all storage types. 20 | type StorageBackend struct { 21 | DestinationPath string 22 | Log Log 23 | } 24 | 25 | type LogLevel int 26 | 27 | const ( 28 | LogLevelInfo LogLevel = iota 29 | LogLevelWarning 30 | ) 31 | 32 | type Log func(logType LogLevel, context string, msg string, params ...any) 33 | 34 | // PruneStats is a wrapper struct for returning stats after pruning 35 | type PruneStats struct { 36 | Total uint 37 | Pruned uint 38 | } 39 | 40 | // DoPrune holds general control flow that applies to any kind of storage. 41 | // Callers can pass in a thunk that performs the actual deletion of files. 42 | func (b *StorageBackend) DoPrune(context string, lenMatches, lenCandidates int, deadline time.Time, doRemoveFiles func() error) error { 43 | if lenMatches != 0 && lenMatches != lenCandidates { 44 | if err := doRemoveFiles(); err != nil { 45 | return err 46 | } 47 | 48 | formattedDeadline, err := deadline.Local().MarshalText() 49 | if err != nil { 50 | return errwrap.Wrap(err, "error marshaling deadline") 51 | } 52 | b.Log(LogLevelInfo, context, 53 | "Pruned %d out of %d backups as they were older than the given deadline of %s.", 54 | lenMatches, 55 | lenCandidates, 56 | string(formattedDeadline), 57 | ) 58 | } else if lenMatches != 0 && lenMatches == lenCandidates { 59 | b.Log(LogLevelWarning, context, "The current configuration would delete all %d existing backups.", lenMatches) 60 | b.Log(LogLevelWarning, context, "Refusing to do so, please check your configuration.") 61 | } else { 62 | b.Log(LogLevelInfo, context, "None of %d existing backups were pruned.", lenCandidates) 63 | } 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /internal/storage/webdav/webdav.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 - offen.software 2 | // SPDX-License-Identifier: MPL-2.0 3 | 4 | package webdav 5 | 6 | import ( 7 | "fmt" 8 | "io/fs" 9 | "net/http" 10 | "os" 11 | "path" 12 | "strings" 13 | "time" 14 | 15 | "github.com/offen/docker-volume-backup/internal/errwrap" 16 | "github.com/offen/docker-volume-backup/internal/storage" 17 | "github.com/studio-b12/gowebdav" 18 | ) 19 | 20 | type webDavStorage struct { 21 | *storage.StorageBackend 22 | client *gowebdav.Client 23 | url string 24 | } 25 | 26 | // Config allows to configure a WebDAV storage backend. 27 | type Config struct { 28 | URL string 29 | RemotePath string 30 | Username string 31 | Password string 32 | URLInsecure bool 33 | } 34 | 35 | // NewStorageBackend creates and initializes a new WebDav storage backend. 36 | func NewStorageBackend(opts Config, logFunc storage.Log) (storage.Backend, error) { 37 | if opts.Username == "" || opts.Password == "" { 38 | return nil, errwrap.Wrap(nil, "WEBDAV_URL is defined, but no credentials were provided") 39 | } else { 40 | webdavClient := gowebdav.NewClient(opts.URL, opts.Username, opts.Password) 41 | 42 | if opts.URLInsecure { 43 | defaultTransport, ok := http.DefaultTransport.(*http.Transport) 44 | if !ok { 45 | return nil, errwrap.Wrap(nil, "unexpected error when asserting type for http.DefaultTransport") 46 | } 47 | webdavTransport := defaultTransport.Clone() 48 | webdavTransport.TLSClientConfig.InsecureSkipVerify = opts.URLInsecure 49 | webdavClient.SetTransport(webdavTransport) 50 | } 51 | 52 | return &webDavStorage{ 53 | StorageBackend: &storage.StorageBackend{ 54 | DestinationPath: opts.RemotePath, 55 | Log: logFunc, 56 | }, 57 | client: webdavClient, 58 | }, nil 59 | } 60 | } 61 | 62 | // Name returns the name of the storage backend 63 | func (b *webDavStorage) Name() string { 64 | return "WebDAV" 65 | } 66 | 67 | // Copy copies the given file to the WebDav storage backend. 68 | func (b *webDavStorage) Copy(file string) error { 69 | _, name := path.Split(file) 70 | if err := b.client.MkdirAll(b.DestinationPath, 0644); err != nil { 71 | return errwrap.Wrap(err, fmt.Sprintf("error creating directory '%s' on server", b.DestinationPath)) 72 | } 73 | 74 | r, err := os.Open(file) 75 | if err != nil { 76 | return errwrap.Wrap(err, "error opening the file to be uploaded") 77 | } 78 | 79 | if err := b.client.WriteStream(path.Join(b.DestinationPath, name), r, 0644); err != nil { 80 | return errwrap.Wrap(err, "error uploading the file") 81 | } 82 | b.Log(storage.LogLevelInfo, b.Name(), "Uploaded a copy of backup '%s' to '%s' at path '%s'.", file, b.url, b.DestinationPath) 83 | 84 | return nil 85 | } 86 | 87 | // Prune rotates away backups according to the configuration and provided deadline for the WebDav storage backend. 88 | func (b *webDavStorage) Prune(deadline time.Time, pruningPrefix string) (*storage.PruneStats, error) { 89 | candidates, err := b.client.ReadDir(b.DestinationPath) 90 | if err != nil { 91 | return nil, errwrap.Wrap(err, "error looking up candidates from remote storage") 92 | } 93 | 94 | var matches []fs.FileInfo 95 | var numCandidates int 96 | for _, candidate := range candidates { 97 | if candidate.IsDir() || !strings.HasPrefix(candidate.Name(), pruningPrefix) { 98 | continue 99 | } 100 | numCandidates++ 101 | if candidate.ModTime().Before(deadline) { 102 | matches = append(matches, candidate) 103 | } 104 | } 105 | 106 | stats := &storage.PruneStats{ 107 | Total: uint(numCandidates), 108 | Pruned: uint(len(matches)), 109 | } 110 | 111 | pruneErr := b.DoPrune(b.Name(), len(matches), numCandidates, deadline, func() error { 112 | for _, match := range matches { 113 | if err := b.client.Remove(path.Join(b.DestinationPath, match.Name())); err != nil { 114 | return errwrap.Wrap(err, "error removing file") 115 | } 116 | } 117 | return nil 118 | }) 119 | return stats, pruneErr 120 | } 121 | -------------------------------------------------------------------------------- /test/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker:27-dind 2 | 3 | RUN apk add \ 4 | age \ 5 | coreutils \ 6 | curl \ 7 | expect \ 8 | gpg \ 9 | gpg-agent \ 10 | jq \ 11 | moreutils \ 12 | tar \ 13 | zstd \ 14 | --no-cache 15 | 16 | WORKDIR /code/test 17 | -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | # Integration Tests 2 | 3 | ## Running tests 4 | 5 | The main entry point for running tests is the `./test.sh` script. 6 | It can be used to run the entire test suite, or just a single test case. 7 | 8 | ### Run all tests 9 | 10 | ```sh 11 | ./test.sh 12 | ``` 13 | 14 | ### Run a single test case 15 | 16 | ```sh 17 | ./test.sh 18 | ``` 19 | 20 | ### Configuring a test run 21 | 22 | In addition to the match pattern, which can be given as the first positional argument, certain behavior can be changed by setting environment variables: 23 | 24 | #### `BUILD_IMAGE` 25 | 26 | When set, the test script will build an up-to-date `docker-volume-backup` image from the current state of your source tree, and run the tests against it. 27 | 28 | ```sh 29 | BUILD_IMAGE=1 ./test.sh 30 | ``` 31 | 32 | The default behavior is not to build an image, and instead look for a version on your host system. 33 | 34 | #### `IMAGE_TAG` 35 | 36 | Setting this value lets you run tests against different existing images, so you can compare behavior: 37 | 38 | ```sh 39 | IMAGE_TAG=v2.30.0 ./test.sh 40 | ``` 41 | 42 | #### `NO_IMAGE_CACHE` 43 | 44 | When set, images from remote registries will not be cached and shared between sandbox containers. 45 | 46 | ```sh 47 | NO_IMAGE_CACHE=1 ./test.sh 48 | ``` 49 | 50 | By default, two local images are created that persist the image data and provide it to containers at runtime. 51 | 52 | ## Understanding the test setup 53 | 54 | The test setup runs each test case in an isolated Docker container, which itself is running an otherwise unused Docker daemon. 55 | This means, tests can rely on noone else using that daemon, making expectations about the number of running containers and so forth. 56 | As the sandbox container is also expected to be torn down post test, the scripts do not need to do any clean up or similar. 57 | 58 | ## Anatomy of a test case 59 | 60 | The `test.sh` script looks for an exectuable file called `run.sh` in each directory. 61 | When found, it is executed and signals success by returning a 0 exit code. 62 | Any other exit code is considered a failure and will halt execution of further tests. 63 | 64 | There is an `util.sh` file containing a few commonly used helpers which can be used by putting the following prelude to a new test case: 65 | 66 | ```sh 67 | cd "$(dirname "$0")" 68 | . ../util.sh 69 | current_test=$(basename $(pwd)) 70 | ``` 71 | -------------------------------------------------------------------------------- /test/age-passphrase/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | backup: 3 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 4 | restart: always 5 | environment: 6 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 7 | BACKUP_FILENAME: test.tar.gz 8 | BACKUP_LATEST_SYMLINK: test-latest.tar.gz.age 9 | BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} 10 | AGE_PASSPHRASE: "Dance.0Tonight.Go.Typical" 11 | volumes: 12 | - ${LOCAL_DIR:-./local}:/archive 13 | - app_data:/backup/app_data:ro 14 | - /var/run/docker.sock:/var/run/docker.sock:ro 15 | 16 | offen: 17 | image: offen/offen:latest 18 | labels: 19 | - docker-volume-backup.stop-during-backup=true 20 | volumes: 21 | - app_data:/var/opt/offen 22 | 23 | volumes: 24 | app_data: 25 | -------------------------------------------------------------------------------- /test/age-passphrase/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | . ../util.sh 7 | current_test=$(basename "$(pwd)") 8 | 9 | export LOCAL_DIR="$(mktemp -d)" 10 | 11 | docker compose up -d --quiet-pull 12 | sleep 5 13 | 14 | docker compose exec backup backup 15 | 16 | expect_running_containers "2" 17 | 18 | TMP_DIR=$(mktemp -d) 19 | 20 | # complex usage of expect(1) due to age not have a way to programmatically 21 | # provide the passphrase 22 | expect -i <"$LOCAL_DIR/pk-a.txt" 12 | PK_A="$(grep -E 'public key' <"$LOCAL_DIR/pk-a.txt" | cut -d: -f2 | xargs)" 13 | age-keygen >"$LOCAL_DIR/pk-b.txt" 14 | PK_B="$(grep -E 'public key' <"$LOCAL_DIR/pk-b.txt" | cut -d: -f2 | xargs)" 15 | 16 | ssh-keygen -t ed25519 -m pem -f "$LOCAL_DIR/id_ed25519" -C "docker-volume-backup@local" 17 | PK_C="$(cat $LOCAL_DIR/id_ed25519.pub)" 18 | 19 | export BACKUP_AGE_PUBLIC_KEYS="$PK_A,$PK_B,$PK_C" 20 | 21 | docker compose up -d --quiet-pull 22 | sleep 5 23 | 24 | docker compose exec backup backup 25 | 26 | expect_running_containers "2" 27 | 28 | do_decrypt() { 29 | TMP_DIR=$(mktemp -d) 30 | age --decrypt -i "$1" -o "$LOCAL_DIR/decrypted.tar.gz" "$LOCAL_DIR/test.tar.gz.age" 31 | tar -xf "$LOCAL_DIR/decrypted.tar.gz" -C "$TMP_DIR" 32 | 33 | if [ ! -f "$TMP_DIR/backup/app_data/offen.db" ]; then 34 | fail "Could not find expected file in untared archive." 35 | fi 36 | rm -vf "$LOCAL_DIR/decrypted.tar.gz" 37 | 38 | pass "Found relevant files in decrypted and untared local backup." 39 | 40 | if [ ! -L "$LOCAL_DIR/test-latest.tar.gz.age" ]; then 41 | fail "Could not find local symlink to latest encrypted backup." 42 | fi 43 | } 44 | 45 | do_decrypt "$LOCAL_DIR/pk-a.txt" 46 | do_decrypt "$LOCAL_DIR/pk-b.txt" 47 | do_decrypt "$LOCAL_DIR/id_ed25519" 48 | -------------------------------------------------------------------------------- /test/azure/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | storage: 3 | image: mcr.microsoft.com/azure-storage/azurite:3.34.0 4 | volumes: 5 | - ${DATA_DIR:-./data}:/data 6 | command: azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --location /data 7 | healthcheck: 8 | test: nc 127.0.0.1 10000 -z 9 | interval: 1s 10 | retries: 30 11 | 12 | az_cli: 13 | image: mcr.microsoft.com/azure-cli:2.71.0 14 | volumes: 15 | - ${LOCAL_DIR:-./local}:/dump 16 | command: 17 | - /bin/sh 18 | - -c 19 | - | 20 | az storage container create --name test-container 21 | depends_on: 22 | storage: 23 | condition: service_healthy 24 | environment: 25 | AZURE_STORAGE_CONNECTION_STRING: DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://storage:10000/devstoreaccount1; 26 | 27 | backup: 28 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 29 | hostname: hostnametoken 30 | restart: always 31 | environment: 32 | AZURE_STORAGE_ACCOUNT_NAME: devstoreaccount1 33 | AZURE_STORAGE_PRIMARY_ACCOUNT_KEY: Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== 34 | AZURE_STORAGE_CONTAINER_NAME: test-container 35 | AZURE_STORAGE_ENDPOINT: http://storage:10000/{{ .AccountName }}/ 36 | AZURE_STORAGE_PATH: 'path/to/backup' 37 | AZURE_STORAGE_ACCESS_TIER: Hot 38 | BACKUP_FILENAME: test.tar.gz 39 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 40 | BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} 41 | BACKUP_PRUNING_LEEWAY: 5s 42 | BACKUP_PRUNING_PREFIX: test 43 | volumes: 44 | - app_data:/backup/app_data:ro 45 | - /var/run/docker.sock:/var/run/docker.sock:ro 46 | 47 | offen: 48 | image: offen/offen:latest 49 | labels: 50 | - docker-volume-backup.stop-during-backup=true 51 | volumes: 52 | - app_data:/var/opt/offen 53 | 54 | volumes: 55 | app_data: 56 | -------------------------------------------------------------------------------- /test/azure/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export LOCAL_DIR=$(mktemp -d) 10 | export TMP_DIR=$(mktemp -d) 11 | export DATA_DIR=$(mktemp -d) 12 | 13 | download_az () { 14 | docker compose run --rm az_cli \ 15 | az storage blob download -f /dump/$1.tar.gz -c test-container -n path/to/backup/$1.tar.gz 16 | } 17 | 18 | docker compose up -d --quiet-pull 19 | 20 | sleep 5 21 | 22 | docker compose exec backup backup 23 | 24 | sleep 5 25 | 26 | expect_running_containers "3" 27 | 28 | download_az "test" 29 | 30 | tar -xvf "$LOCAL_DIR/test.tar.gz" -C $TMP_DIR 31 | 32 | if [ ! -f "$TMP_DIR/backup/app_data/offen.db" ]; then 33 | fail "Could not find expeced file in untared backup" 34 | fi 35 | 36 | pass "Found relevant files in untared remote backups." 37 | rm "$LOCAL_DIR/test.tar.gz" 38 | 39 | # The second part of this test checks if backups get deleted when the retention 40 | # is set to 0 days (which it should not as it would mean all backups get deleted) 41 | BACKUP_RETENTION_DAYS="0" docker compose up -d 42 | sleep 5 43 | 44 | docker compose exec backup backup 45 | 46 | download_az "test" 47 | if [ ! -f "$LOCAL_DIR/test.tar.gz" ]; then 48 | fail "Remote backup was deleted" 49 | fi 50 | pass "Remote backups have not been deleted." 51 | 52 | # The third part of this test checks if old backups get deleted when the retention 53 | # is set to 7 days (which it should) 54 | 55 | BACKUP_RETENTION_DAYS="7" docker compose up -d 56 | sleep 5 57 | 58 | info "Create first backup with no prune" 59 | docker compose exec backup backup 60 | 61 | docker compose run --rm az_cli \ 62 | az storage blob upload -f /dump/test.tar.gz -c test-container -n path/to/backup/test-old.tar.gz 63 | 64 | docker compose down 65 | rm "$LOCAL_DIR/test.tar.gz" 66 | 67 | back_date="$(date "+%Y-%m-%dT%H:%M:%S%z" -d "14 days ago" | rev | cut -c 3- | rev):00" 68 | jq --arg back_date "$back_date" '(.collections[] | select(.name=="$BLOBS_COLLECTION$") | .data[] | select(.name=="path/to/backup/test-old.tar.gz") | .properties.creationTime = $back_date)' "$DATA_DIR/__azurite_db_blob__.json" | sponge "$DATA_DIR/__azurite_db_blob__.json" 69 | 70 | docker compose up -d 71 | sleep 5 72 | 73 | info "Create second backup and prune" 74 | docker compose exec backup backup 75 | 76 | info "Download first backup which should be pruned" 77 | download_az "test-old" || true 78 | if [ -f "$LOCAL_DIR/test-old.tar.gz" ]; then 79 | fail "Backdated file was not deleted" 80 | fi 81 | download_az "test" || true 82 | if [ ! -f "$LOCAL_DIR/test.tar.gz" ]; then 83 | fail "Recent file was not found" 84 | fi 85 | 86 | pass "Old remote backup has been pruned, new one is still present." 87 | -------------------------------------------------------------------------------- /test/certs/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | minio: 3 | hostname: minio.local 4 | image: minio/minio:RELEASE.2020-08-04T23-10-51Z 5 | environment: 6 | MINIO_ROOT_USER: test 7 | MINIO_ROOT_PASSWORD: test 8 | MINIO_ACCESS_KEY: test 9 | MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ 10 | entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server --certs-dir "/certs" --address ":443" /data' 11 | volumes: 12 | - minio_backup_data:/data 13 | - ${CERT_DIR:-.}/minio.crt:/certs/public.crt 14 | - ${CERT_DIR:-.}/minio.key:/certs/private.key 15 | 16 | backup: 17 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 18 | depends_on: 19 | - minio 20 | restart: always 21 | environment: 22 | BACKUP_FILENAME: test.tar.gz 23 | AWS_ACCESS_KEY_ID: test 24 | AWS_SECRET_ACCESS_KEY: GMusLtUmILge2by+z890kQ 25 | AWS_ENDPOINT: minio.local:443 26 | AWS_ENDPOINT_CA_CERT: /root/minio-rootCA.crt 27 | AWS_S3_BUCKET_NAME: backup 28 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 29 | BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} 30 | BACKUP_PRUNING_LEEWAY: 5s 31 | volumes: 32 | - app_data:/backup/app_data:ro 33 | - /var/run/docker.sock:/var/run/docker.sock:ro 34 | - ${CERT_DIR:-.}/rootCA.crt:/root/minio-rootCA.crt 35 | 36 | offen: 37 | image: offen/offen:latest 38 | labels: 39 | - docker-volume-backup.stop-during-backup=true 40 | volumes: 41 | - app_data:/var/opt/offen 42 | 43 | volumes: 44 | minio_backup_data: 45 | name: minio_backup_data 46 | app_data: 47 | -------------------------------------------------------------------------------- /test/certs/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export CERT_DIR=$(mktemp -d) 10 | 11 | openssl genrsa -des3 -passout pass:test -out "$CERT_DIR/rootCA.key" 4096 12 | openssl req -passin pass:test \ 13 | -subj "/C=DE/ST=BE/O=IntegrationTest, Inc." \ 14 | -x509 -new -key "$CERT_DIR/rootCA.key" -sha256 -days 1 -out "$CERT_DIR/rootCA.crt" 15 | 16 | openssl genrsa -out "$CERT_DIR/minio.key" 4096 17 | openssl req -new -sha256 -key "$CERT_DIR/minio.key" \ 18 | -subj "/C=DE/ST=BE/O=IntegrationTest, Inc./CN=minio" \ 19 | -out "$CERT_DIR/minio.csr" 20 | 21 | openssl x509 -req -passin pass:test \ 22 | -in "$CERT_DIR/minio.csr" \ 23 | -CA "$CERT_DIR/rootCA.crt" -CAkey "$CERT_DIR/rootCA.key" -CAcreateserial \ 24 | -extfile san.cnf \ 25 | -out "$CERT_DIR/minio.crt" -days 1 -sha256 26 | 27 | openssl x509 -in "$CERT_DIR/minio.crt" -noout -text 28 | 29 | docker compose up -d --quiet-pull 30 | sleep 5 31 | 32 | docker compose exec backup backup 33 | 34 | sleep 5 35 | 36 | expect_running_containers "3" 37 | 38 | docker run --rm \ 39 | -v minio_backup_data:/minio_data \ 40 | alpine \ 41 | ash -c 'tar -xvf /minio_data/backup/test.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db' 42 | 43 | pass "Found relevant files in untared remote backups." 44 | -------------------------------------------------------------------------------- /test/certs/san.cnf: -------------------------------------------------------------------------------- 1 | subjectAltName = DNS:minio.local 2 | -------------------------------------------------------------------------------- /test/cli/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd $(dirname $0) 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | docker network create test_network 10 | docker volume create backup_data 11 | docker volume create app_data 12 | # This volume is created to test whether empty directories are handled 13 | # correctly. It is not supposed to hold any data. 14 | docker volume create empty_data 15 | 16 | docker run -d -q \ 17 | --name minio \ 18 | --network test_network \ 19 | --env MINIO_ROOT_USER=test \ 20 | --env MINIO_ROOT_PASSWORD=test \ 21 | --env MINIO_ACCESS_KEY=test \ 22 | --env MINIO_SECRET_KEY=GMusLtUmILge2by+z890kQ \ 23 | -v backup_data:/data \ 24 | minio/minio:RELEASE.2020-08-04T23-10-51Z server /data 25 | 26 | docker exec minio mkdir -p /data/backup 27 | 28 | docker run -d -q \ 29 | --name offen \ 30 | --network test_network \ 31 | -v app_data:/var/opt/offen/ \ 32 | offen/offen:latest 33 | 34 | sleep 10 35 | 36 | docker run --rm -q \ 37 | --network test_network \ 38 | -v app_data:/backup/app_data \ 39 | -v empty_data:/backup/empty_data \ 40 | -v /var/run/docker.sock:/var/run/docker.sock:ro \ 41 | --env AWS_ACCESS_KEY_ID=test \ 42 | --env AWS_SECRET_ACCESS_KEY=GMusLtUmILge2by+z890kQ \ 43 | --env AWS_ENDPOINT=minio:9000 \ 44 | --env AWS_ENDPOINT_PROTO=http \ 45 | --env AWS_S3_BUCKET_NAME=backup \ 46 | --env BACKUP_FILENAME=test.tar.gz \ 47 | --env "BACKUP_FROM_SNAPSHOT=true" \ 48 | --entrypoint backup \ 49 | offen/docker-volume-backup:${TEST_VERSION:-canary} 50 | 51 | docker run --rm -q \ 52 | -v backup_data:/data alpine \ 53 | ash -c 'tar -xvf /data/backup/test.tar.gz && test -f /backup/app_data/offen.db && test -d /backup/empty_data' 54 | 55 | pass "Found relevant files in untared remote backup." 56 | 57 | # This test does not stop containers during backup. This is happening on 58 | # purpose in order to cover this setup as well. 59 | expect_running_containers "2" 60 | 61 | docker rm $(docker stop minio offen) 62 | docker volume rm backup_data app_data 63 | docker network rm test_network 64 | -------------------------------------------------------------------------------- /test/collision/docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2021 - offen.software 2 | # SPDX-License-Identifier: Unlicense 3 | 4 | services: 5 | backup: 6 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 7 | environment: 8 | BACKUP_FILENAME: test.tar.gz 9 | volumes: 10 | - offen_data:/backup/offen_data:ro 11 | - ${LOCAL_DIR:-./local}:/archive 12 | - /var/run/docker.sock:/var/run/docker.sock:ro 13 | 14 | offen: 15 | image: offen/offen:latest 16 | labels: 17 | - docker-volume-backup.stop-during-backup=true 18 | deploy: 19 | labels: 20 | - docker-volume-backup.stop-during-backup=true 21 | replicas: 2 22 | volumes: 23 | - offen_data:/var/opt/offen 24 | 25 | volumes: 26 | offen_data: 27 | -------------------------------------------------------------------------------- /test/collision/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd $(dirname $0) 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export LOCAL_DIR=$(mktemp -d) 10 | 11 | docker swarm init 12 | 13 | docker stack deploy --compose-file=docker-compose.yml test_stack 14 | 15 | while [ -z $(docker ps -q -f name=backup) ]; do 16 | info "Backup container not ready yet. Retrying." 17 | sleep 1 18 | done 19 | 20 | sleep 20 21 | 22 | set +e 23 | docker exec $(docker ps -q -f name=backup) backup 24 | if [ $? = "0" ]; then 25 | fail "Expected script to exit with error code." 26 | fi 27 | 28 | if [ -f "${LOCAL_DIR}/test.tar.gz" ]; then 29 | fail "Found backup file that should not have been created." 30 | fi 31 | 32 | expect_running_containers "3" 33 | 34 | pass "Script did not perform backup as there was a label collision." 35 | -------------------------------------------------------------------------------- /test/commands/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | database: 3 | image: mariadb:10.7 4 | deploy: 5 | restart_policy: 6 | condition: on-failure 7 | environment: 8 | MARIADB_ROOT_PASSWORD: test 9 | MARIADB_DATABASE: backup 10 | labels: 11 | # this is testing the deprecated label on purpose 12 | - docker-volume-backup.exec-pre=/bin/sh -c 'mysqldump -ptest --all-databases > /tmp/volume/dump.sql' 13 | - docker-volume-backup.copy-post=/bin/sh -c 'echo "post" > /tmp/volume/post.txt' 14 | - docker-volume-backup.exec-label=test 15 | volumes: 16 | - app_data:/tmp/volume 17 | 18 | other_database: 19 | image: mariadb:10.7 20 | deploy: 21 | restart_policy: 22 | condition: on-failure 23 | environment: 24 | MARIADB_ROOT_PASSWORD: test 25 | MARIADB_DATABASE: backup 26 | labels: 27 | - docker-volume-backup.archive-pre=touch /tmp/volume/not-relevant.txt 28 | - docker-volume-backup.exec-label=not-relevant 29 | volumes: 30 | - app_data:/tmp/volume 31 | 32 | backup: 33 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 34 | deploy: 35 | restart_policy: 36 | condition: on-failure 37 | environment: 38 | BACKUP_FILENAME: test.tar.gz 39 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 40 | EXEC_LABEL: test 41 | EXEC_FORWARD_OUTPUT: "true" 42 | volumes: 43 | - ${LOCAL_DIR:-./local}:/archive 44 | - app_data:/backup/data:ro 45 | - /var/run/docker.sock:/var/run/docker.sock:ro 46 | 47 | volumes: 48 | app_data: 49 | -------------------------------------------------------------------------------- /test/commands/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd $(dirname $0) 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export LOCAL_DIR=$(mktemp -d) 10 | export TMP_DIR=$(mktemp -d) 11 | 12 | docker compose up -d --quiet-pull 13 | sleep 30 # mariadb likes to take a bit before responding 14 | 15 | docker compose exec backup backup 16 | 17 | tar -xvf "$LOCAL_DIR/test.tar.gz" -C $TMP_DIR 18 | if [ ! -f "$TMP_DIR/backup/data/dump.sql" ]; then 19 | fail "Could not find file written by pre command." 20 | fi 21 | pass "Found expected file." 22 | 23 | if [ -f "$TMP_DIR/backup/data/not-relevant.txt" ]; then 24 | fail "Command ran for container with other label." 25 | fi 26 | pass "Command did not run for container with other label." 27 | 28 | if [ -f "$TMP_DIR/backup/data/post.txt" ]; then 29 | fail "File created in post command was present in backup." 30 | fi 31 | pass "Did not find unexpected file." 32 | 33 | docker compose down --volumes 34 | 35 | info "Running commands test in swarm mode next." 36 | 37 | export LOCAL_DIR=$(mktemp -d) 38 | export TMP_DIR=$(mktemp -d) 39 | 40 | docker swarm init 41 | 42 | docker stack deploy --compose-file=docker-compose.yml test_stack 43 | 44 | while [ -z $(docker ps -q -f name=backup) ]; do 45 | info "Backup container not ready yet. Retrying." 46 | sleep 1 47 | done 48 | 49 | sleep 20 50 | 51 | docker exec $(docker ps -q -f name=backup) backup 52 | 53 | tar -xvf "$LOCAL_DIR/test.tar.gz" -C $TMP_DIR 54 | if [ ! -f "$TMP_DIR/backup/data/dump.sql" ]; then 55 | fail "Could not find file written by pre command." 56 | fi 57 | pass "Found expected file." 58 | 59 | if [ -f "$TMP_DIR/backup/data/post.txt" ]; then 60 | fail "File created in post command was present in backup." 61 | fi 62 | pass "Did not find unexpected file." 63 | -------------------------------------------------------------------------------- /test/confd/01backup.env: -------------------------------------------------------------------------------- 1 | # This is a comment 2 | # NOT=$(docker ps -aq) 3 | # e.g. `backup-$HOSTNAME-%Y-%m-%dT%H-%M-%S.tar.gz`. Expansion happens before` 4 | 5 | NAME="$EXPANSION_VALUE" 6 | BACKUP_CRON_EXPRESSION="*/1 * * * *" 7 | -------------------------------------------------------------------------------- /test/confd/02backup.env: -------------------------------------------------------------------------------- 1 | NAME="other" 2 | BACKUP_CRON_EXPRESSION="*/1 * * * *" 3 | BACKUP_FILENAME="override-$NAME.tar.gz" 4 | -------------------------------------------------------------------------------- /test/confd/03never.env: -------------------------------------------------------------------------------- 1 | NAME="never" 2 | BACKUP_CRON_EXPRESSION="0 0 5 31 2 ?" 3 | -------------------------------------------------------------------------------- /test/confd/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | backup: 3 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 4 | restart: always 5 | environment: 6 | BACKUP_FILENAME: $$NAME.tar.gz 7 | BACKUP_FILENAME_EXPAND: 'true' 8 | EXPANSION_VALUE: conf 9 | volumes: 10 | - ${LOCAL_DIR:-./local}:/archive 11 | - app_data:/backup/app_data:ro 12 | - ./01backup.env:/etc/dockervolumebackup/conf.d/01backup.env 13 | - ./02backup.env:/etc/dockervolumebackup/conf.d/02backup.env 14 | - ./03never.env:/etc/dockervolumebackup/conf.d/03never.env 15 | - /var/run/docker.sock:/var/run/docker.sock:ro 16 | 17 | offen: 18 | image: offen/offen:latest 19 | labels: 20 | - docker-volume-backup.stop-during-backup=true 21 | volumes: 22 | - app_data:/var/opt/offen 23 | 24 | volumes: 25 | app_data: 26 | -------------------------------------------------------------------------------- /test/confd/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd $(dirname $0) 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export LOCAL_DIR=$(mktemp -d) 10 | 11 | docker compose up -d --quiet-pull 12 | 13 | # sleep until a backup is guaranteed to have happened on the 1 minute schedule 14 | sleep 100 15 | 16 | docker compose logs backup 17 | 18 | if [ ! -f "$LOCAL_DIR/conf.tar.gz" ]; then 19 | fail "Config from file was not used." 20 | fi 21 | pass "Config from file was used." 22 | 23 | if [ ! -f "$LOCAL_DIR/override-other.tar.gz" ]; then 24 | fail "Run on same schedule did not succeed." 25 | fi 26 | pass "Run on same schedule succeeded." 27 | 28 | if [ -f "$LOCAL_DIR/never.tar.gz" ]; then 29 | fail "Unexpected file was found." 30 | fi 31 | pass "Unexpected cron did not run." 32 | -------------------------------------------------------------------------------- /test/dropbox/.gitignore: -------------------------------------------------------------------------------- 1 | user_v2_ready.yaml 2 | -------------------------------------------------------------------------------- /test/dropbox/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | openapi_mock: 3 | image: muonsoft/openapi-mock:0.3.9 4 | environment: 5 | OPENAPI_MOCK_USE_EXAMPLES: if_present 6 | OPENAPI_MOCK_SPECIFICATION_URL: '/etc/openapi/user_v2.yaml' 7 | ports: 8 | - 8080:8080 9 | volumes: 10 | - ${SPEC_FILE:-./user_v2.yaml}:/etc/openapi/user_v2.yaml 11 | 12 | oauth2_mock: 13 | image: ghcr.io/navikt/mock-oauth2-server:1.0.0 14 | ports: 15 | - 8090:8090 16 | environment: 17 | PORT: 8090 18 | JSON_CONFIG_PATH: '/etc/oauth2/config.json' 19 | volumes: 20 | - ./oauth2_config.json:/etc/oauth2/config.json 21 | 22 | backup: 23 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 24 | hostname: hostnametoken 25 | depends_on: 26 | - openapi_mock 27 | - oauth2_mock 28 | restart: always 29 | environment: 30 | BACKUP_FILENAME_EXPAND: 'true' 31 | BACKUP_FILENAME: test-$$HOSTNAME.tar.gz 32 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 33 | BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} 34 | BACKUP_PRUNING_LEEWAY: 5s 35 | BACKUP_PRUNING_PREFIX: test 36 | DROPBOX_ENDPOINT: http://openapi_mock:8080 37 | DROPBOX_OAUTH2_ENDPOINT: http://oauth2_mock:8090 38 | DROPBOX_REFRESH_TOKEN: test 39 | DROPBOX_APP_KEY: test 40 | DROPBOX_APP_SECRET: test 41 | DROPBOX_REMOTE_PATH: /test 42 | DROPBOX_CONCURRENCY_LEVEL: 6 43 | volumes: 44 | - app_data:/backup/app_data:ro 45 | - /var/run/docker.sock:/var/run/docker.sock:ro 46 | 47 | offen: 48 | image: offen/offen:latest 49 | labels: 50 | - docker-volume-backup.stop-during-backup=true 51 | volumes: 52 | - app_data:/var/opt/offen 53 | 54 | volumes: 55 | app_data: 56 | -------------------------------------------------------------------------------- /test/dropbox/oauth2_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "interactiveLogin": true, 3 | "httpServer": "NettyWrapper", 4 | "tokenCallbacks": [ 5 | { 6 | "issuerId": "issuer1", 7 | "tokenExpiry": 120, 8 | "requestMappings": [ 9 | { 10 | "requestParam": "scope", 11 | "match": "scope1", 12 | "claims": { 13 | "sub": "subByScope", 14 | "aud": [ 15 | "audByScope" 16 | ] 17 | } 18 | } 19 | ] 20 | }, 21 | { 22 | "issuerId": "issuer2", 23 | "requestMappings": [ 24 | { 25 | "requestParam": "someparam", 26 | "match": "somevalue", 27 | "claims": { 28 | "sub": "subBySomeParam", 29 | "aud": [ 30 | "audBySomeParam" 31 | ] 32 | } 33 | } 34 | ] 35 | } 36 | ] 37 | } -------------------------------------------------------------------------------- /test/dropbox/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export SPEC_FILE=$(mktemp -d)/user_v2.yaml 10 | cp user_v2.yaml $SPEC_FILE 11 | sed -i 's/SERVER_MODIFIED_1/'"$(date "+%Y-%m-%dT%H:%M:%SZ")/g" $SPEC_FILE 12 | sed -i 's/SERVER_MODIFIED_2/'"$(date "+%Y-%m-%dT%H:%M:%SZ" -d "14 days ago")/g" $SPEC_FILE 13 | 14 | docker compose up -d --quiet-pull 15 | sleep 5 16 | 17 | logs=$(docker compose exec -T backup backup) 18 | 19 | sleep 5 20 | 21 | expect_running_containers "4" 22 | 23 | echo "$logs" 24 | if echo "$logs" | grep -q "ERROR"; then 25 | fail "Backup failed, errors reported: $logs" 26 | else 27 | pass "Backup succeeded, no errors reported." 28 | fi 29 | 30 | # The second part of this test checks if backups get deleted when the retention 31 | # is set to 0 days (which it should not as it would mean all backups get deleted) 32 | BACKUP_RETENTION_DAYS="0" docker compose up -d 33 | sleep 5 34 | 35 | logs=$(docker compose exec -T backup backup) 36 | 37 | echo "$logs" 38 | if echo "$logs" | grep -q "Refusing to do so, please check your configuration"; then 39 | pass "Remote backups have not been deleted." 40 | else 41 | fail "Remote backups would have been deleted: $logs" 42 | fi 43 | 44 | # The third part of this test checks if old backups get deleted when the retention 45 | # is set to 7 days (which it should) 46 | BACKUP_RETENTION_DAYS="7" docker compose up -d 47 | sleep 5 48 | 49 | info "Create second backup and prune" 50 | logs=$(docker compose exec -T backup backup) 51 | 52 | echo "$logs" 53 | if echo "$logs" | grep -q "Pruned 1 out of 2 backups as they were older"; then 54 | pass "Old remote backup has been pruned, new one is still present." 55 | elif echo "$logs" | grep -q "ERROR"; then 56 | fail "Pruning failed, errors reported: $logs" 57 | elif echo "$logs" | grep -q "None of 1 existing backups were pruned"; then 58 | fail "Pruning failed, old backup has not been pruned: $logs" 59 | else 60 | fail "Pruning failed, unknown result: $logs" 61 | fi 62 | -------------------------------------------------------------------------------- /test/extend/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG version=canary 2 | FROM offen/docker-volume-backup:$version 3 | 4 | RUN apk add rsync 5 | -------------------------------------------------------------------------------- /test/extend/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | backup: 3 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 4 | restart: always 5 | labels: 6 | - docker-volume-backup.copy-post=/bin/sh -c 'mkdir -p /tmp/unpack && tar -xvf $$COMMAND_RUNTIME_ARCHIVE_FILEPATH -C /tmp/unpack && rsync -r /tmp/unpack/backup/app_data /local' 7 | environment: 8 | BACKUP_FILENAME: test.tar.gz 9 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 10 | EXEC_FORWARD_OUTPUT: "true" 11 | volumes: 12 | - ${LOCAL_DIR:-local}:/local 13 | - app_data:/backup/app_data:ro 14 | - /var/run/docker.sock:/var/run/docker.sock:ro 15 | 16 | offen: 17 | image: offen/offen:latest 18 | labels: 19 | - docker-volume-backup.stop-during-backup=true 20 | volumes: 21 | - app_data:/var/opt/offen 22 | 23 | volumes: 24 | app_data: 25 | -------------------------------------------------------------------------------- /test/extend/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export LOCAL_DIR=$(mktemp -d) 10 | 11 | export BASE_VERSION="${TEST_VERSION:-canary}" 12 | export TEST_VERSION="${TEST_VERSION:-canary}-with-rsync" 13 | 14 | docker build . -t offen/docker-volume-backup:$TEST_VERSION --build-arg version=$BASE_VERSION 15 | 16 | docker compose up -d --quiet-pull 17 | sleep 5 18 | 19 | docker compose exec backup backup 20 | 21 | sleep 5 22 | 23 | expect_running_containers "2" 24 | 25 | if [ ! -f "$LOCAL_DIR/app_data/offen.db" ]; then 26 | fail "Could not find expected file in untared archive." 27 | fi 28 | -------------------------------------------------------------------------------- /test/gpg-asym/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | backup: 3 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 4 | restart: always 5 | environment: 6 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 7 | BACKUP_FILENAME: test.tar.gz 8 | BACKUP_LATEST_SYMLINK: test-latest.tar.gz.gpg 9 | BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} 10 | GPG_PUBLIC_KEY_RING_FILE: /keys/public_key.asc 11 | volumes: 12 | - ${KEY_DIR:-.}/public_key.asc:/keys/public_key.asc 13 | - ${LOCAL_DIR:-./local}:/archive 14 | - app_data:/backup/app_data:ro 15 | - /var/run/docker.sock:/var/run/docker.sock:ro 16 | 17 | offen: 18 | image: offen/offen:latest 19 | labels: 20 | - docker-volume-backup.stop-during-backup=true 21 | volumes: 22 | - app_data:/var/opt/offen 23 | 24 | volumes: 25 | app_data: 26 | -------------------------------------------------------------------------------- /test/gpg-asym/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export LOCAL_DIR=$(mktemp -d) 10 | 11 | export KEY_DIR=$(mktemp -d) 12 | 13 | export PASSPHRASE="test" 14 | 15 | gpg --batch --gen-key < "$LOCAL_DIR/decrypted.tar.gz" 37 | 38 | tar -xf "$LOCAL_DIR/decrypted.tar.gz" -C $TMP_DIR 39 | 40 | if [ ! -f $TMP_DIR/backup/app_data/offen.db ]; then 41 | fail "Could not find expected file in untared archive." 42 | fi 43 | rm "$LOCAL_DIR/decrypted.tar.gz" 44 | 45 | pass "Found relevant files in decrypted and untared local backup." 46 | 47 | if [ ! -L "$LOCAL_DIR/test-latest.tar.gz.gpg" ]; then 48 | fail "Could not find local symlink to latest encrypted backup." 49 | fi 50 | -------------------------------------------------------------------------------- /test/gpg/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | backup: 3 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 4 | restart: always 5 | environment: 6 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 7 | BACKUP_FILENAME: test.tar.gz 8 | BACKUP_LATEST_SYMLINK: test-latest.tar.gz.gpg 9 | BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} 10 | GPG_PASSPHRASE: 1234#$$ecret 11 | volumes: 12 | - ${LOCAL_DIR:-./local}:/archive 13 | - app_data:/backup/app_data:ro 14 | - /var/run/docker.sock:/var/run/docker.sock:ro 15 | 16 | offen: 17 | image: offen/offen:latest 18 | labels: 19 | - docker-volume-backup.stop-during-backup=true 20 | volumes: 21 | - app_data:/var/opt/offen 22 | 23 | volumes: 24 | app_data: 25 | -------------------------------------------------------------------------------- /test/gpg/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export LOCAL_DIR=$(mktemp -d) 10 | 11 | docker compose up -d --quiet-pull 12 | sleep 5 13 | 14 | docker compose exec backup backup 15 | 16 | expect_running_containers "2" 17 | 18 | TMP_DIR=$(mktemp -d) 19 | 20 | echo "1234#\$ecret" | gpg -d --pinentry-mode loopback --yes --passphrase-fd 0 "$LOCAL_DIR/test.tar.gz.gpg" > "$LOCAL_DIR/decrypted.tar.gz" 21 | tar -xf "$LOCAL_DIR/decrypted.tar.gz" -C $TMP_DIR 22 | 23 | if [ ! -f $TMP_DIR/backup/app_data/offen.db ]; then 24 | fail "Could not find expected file in untared archive." 25 | fi 26 | rm "$LOCAL_DIR/decrypted.tar.gz" 27 | 28 | pass "Found relevant files in decrypted and untared local backup." 29 | 30 | if [ ! -L "$LOCAL_DIR/test-latest.tar.gz.gpg" ]; then 31 | fail "Could not find local symlink to latest encrypted backup." 32 | fi 33 | -------------------------------------------------------------------------------- /test/ignore/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | backup: 3 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 4 | deploy: 5 | restart_policy: 6 | condition: on-failure 7 | environment: 8 | BACKUP_FILENAME: test.tar.gz 9 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 10 | BACKUP_EXCLUDE_REGEXP: '\.(me|you)$$' 11 | volumes: 12 | - ${LOCAL_DIR:-./local}:/archive 13 | - ./sources:/backup/data:ro 14 | -------------------------------------------------------------------------------- /test/ignore/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd $(dirname $0) 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export LOCAL_DIR=$(mktemp -d) 10 | 11 | docker compose up -d --quiet-pull 12 | sleep 5 13 | docker compose exec backup backup 14 | 15 | TMP_DIR=$(mktemp -d) 16 | tar --same-owner -xvf "$LOCAL_DIR/test.tar.gz" -C "$TMP_DIR" 17 | 18 | if [ ! -f "$TMP_DIR/backup/data/me.txt" ]; then 19 | fail "Expected file was not found." 20 | fi 21 | pass "Expected file was found." 22 | 23 | if [ -f "$TMP_DIR/backup/data/skip.me" ]; then 24 | fail "Ignored file was found." 25 | fi 26 | pass "Ignored file was not found." 27 | -------------------------------------------------------------------------------- /test/ignore/sources/me.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/offen/docker-volume-backup/b1f49ea3e15d1f925d8b0a8b9ee96b6f0acc52bc/test/ignore/sources/me.txt -------------------------------------------------------------------------------- /test/ignore/sources/skip.me: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/offen/docker-volume-backup/b1f49ea3e15d1f925d8b0a8b9ee96b6f0acc52bc/test/ignore/sources/skip.me -------------------------------------------------------------------------------- /test/local/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | backup: 3 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 4 | hostname: hostnametoken 5 | restart: always 6 | environment: 7 | BACKUP_FILENAME_EXPAND: 'true' 8 | BACKUP_FILENAME: test-$$HOSTNAME.tar.gz 9 | BACKUP_LATEST_SYMLINK: test-$$HOSTNAME.latest.tar.gz.gpg 10 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 11 | BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} 12 | BACKUP_PRUNING_LEEWAY: 5s 13 | BACKUP_PRUNING_PREFIX: test 14 | volumes: 15 | - app_data:/backup/app_data:ro 16 | - /var/run/docker.sock:/var/run/docker.sock:ro 17 | - ${LOCAL_DIR:-./local}:/archive 18 | 19 | offen: 20 | image: offen/offen:latest 21 | labels: 22 | - docker-volume-backup.stop-during-backup=true 23 | volumes: 24 | - app_data:/var/opt/offen 25 | 26 | volumes: 27 | app_data: 28 | -------------------------------------------------------------------------------- /test/local/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export LOCAL_DIR=$(mktemp -d) 10 | 11 | docker compose up -d --quiet-pull 12 | sleep 5 13 | 14 | # A symlink for a known file in the volume is created so the test can check 15 | # whether symlinks are preserved on backup. 16 | docker compose exec offen ln -s /var/opt/offen/offen.db /var/opt/offen/db.link 17 | docker compose exec backup backup 18 | 19 | sleep 5 20 | 21 | expect_running_containers "2" 22 | 23 | tmp_dir=$(mktemp -d) 24 | tar -xvf "$LOCAL_DIR/test-hostnametoken.tar.gz" -C $tmp_dir 25 | if [ ! -f "$tmp_dir/backup/app_data/offen.db" ]; then 26 | fail "Could not find expected file in untared archive." 27 | fi 28 | rm -f "$LOCAL_DIR/test-hostnametoken.tar.gz" 29 | 30 | if [ ! -L "$tmp_dir/backup/app_data/db.link" ]; then 31 | fail "Could not find expected symlink in untared archive." 32 | fi 33 | 34 | pass "Found relevant files in decrypted and untared local backup." 35 | 36 | if [ ! -L "$LOCAL_DIR/test-hostnametoken.latest.tar.gz.gpg" ]; then 37 | fail "Could not find symlink to latest version." 38 | fi 39 | 40 | pass "Found symlink to latest version in local backup." 41 | 42 | # The second part of this test checks if backups get deleted when the retention 43 | # is set to 0 days (which it should not as it would mean all backups get deleted) 44 | BACKUP_RETENTION_DAYS="0" docker compose up -d 45 | sleep 5 46 | 47 | docker compose exec backup backup 48 | 49 | if [ "$(find "$LOCAL_DIR" -type f | wc -l)" != "1" ]; then 50 | fail "Backups should not have been deleted, instead seen: "$(find "$local_dir" -type f)"" 51 | fi 52 | pass "Local backups have not been deleted." 53 | 54 | # The third part of this test checks if old backups get deleted when the retention 55 | # is set to 7 days (which it should) 56 | 57 | BACKUP_RETENTION_DAYS="7" docker compose up -d 58 | sleep 5 59 | 60 | info "Create first backup with no prune" 61 | docker compose exec backup backup 62 | 63 | touch -r "$LOCAL_DIR/test-hostnametoken.tar.gz" -d "14 days ago" "$LOCAL_DIR/test-hostnametoken-old.tar.gz" 64 | 65 | info "Create second backup and prune" 66 | docker compose exec backup backup 67 | 68 | if [ -f "$LOCAL_DIR/test-hostnametoken-old.tar.gz" ]; then 69 | fail "Backdated file has not been deleted." 70 | fi 71 | 72 | if [ ! -f "$LOCAL_DIR/test-hostnametoken.tar.gz" ]; then 73 | fail "Recent file has been deleted." 74 | fi 75 | 76 | pass "Old remote backup has been pruned, new one is still present." 77 | -------------------------------------------------------------------------------- /test/lock/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | backup: 3 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 4 | restart: always 5 | environment: 6 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 7 | BACKUP_RETENTION_DAYS: '7' 8 | volumes: 9 | - app_data:/backup/app_data:ro 10 | - /var/run/docker.sock:/var/run/docker.sock:ro 11 | - ${LOCAL_DIR:-./local}:/archive 12 | 13 | offen: 14 | image: offen/offen:latest 15 | labels: 16 | - docker-volume-backup.stop-during-backup=true 17 | volumes: 18 | - app_data:/var/opt/offen 19 | 20 | volumes: 21 | app_data: 22 | -------------------------------------------------------------------------------- /test/lock/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export LOCAL_DIR=$(mktemp -d) 10 | 11 | docker compose up -d --quiet-pull 12 | sleep 5 13 | 14 | ec=0 15 | 16 | docker compose exec -e BACKUP_RETENTION_DAYS=7 -e BACKUP_FILENAME=test.tar.gz backup backup & \ 17 | { set +e; sleep 0.1; docker compose exec -e BACKUP_FILENAME=test2.tar.gz -e LOCK_TIMEOUT=1s backup backup; ec=$?;} 18 | 19 | if [ "$ec" = "0" ]; then 20 | fail "Subsequent invocation exited 0" 21 | fi 22 | pass "Subsequent invocation did not exit 0" 23 | 24 | sleep 5 25 | 26 | if [ ! -f "${LOCAL_DIR}/test.tar.gz" ]; then 27 | fail "Could not find expected tar file" 28 | fi 29 | pass "Found expected tar file" 30 | 31 | if [ -f "${LOCAL_DIR}/test2.tar.gz" ]; then 32 | fail "Subsequent invocation was expected to fail but created archive" 33 | fi 34 | pass "Subsequent invocation did not create archive" 35 | -------------------------------------------------------------------------------- /test/nonroot/01conf.env: -------------------------------------------------------------------------------- 1 | AWS_ACCESS_KEY_ID="test" 2 | AWS_SECRET_ACCESS_KEY="GMusLtUmILge2by+z890kQ" 3 | AWS_ENDPOINT="minio:9000" 4 | AWS_ENDPOINT_PROTO="http" 5 | AWS_S3_BUCKET_NAME="backup" 6 | BACKUP_CRON_EXPRESSION="0 0 5 31 2 ?" 7 | BACKUP_FILENAME="test.tar.gz" 8 | -------------------------------------------------------------------------------- /test/nonroot/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | minio: 3 | image: minio/minio:RELEASE.2020-08-04T23-10-51Z 4 | environment: 5 | MINIO_ROOT_USER: test 6 | MINIO_ROOT_PASSWORD: test 7 | MINIO_ACCESS_KEY: test 8 | MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ 9 | entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data' 10 | volumes: 11 | - ${LOCAL_DIR:-local}:/data 12 | 13 | backup: 14 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 15 | user: 1000:1000 16 | depends_on: 17 | - minio 18 | restart: always 19 | volumes: 20 | - app_data:/backup/app_data:ro 21 | - ./01conf.env:/etc/dockervolumebackup/conf.d/01conf.env 22 | 23 | offen: 24 | image: offen/offen:latest 25 | labels: 26 | - docker-volume-backup.stop-during-backup=true 27 | volumes: 28 | - app_data:/var/opt/offen 29 | 30 | volumes: 31 | app_data: 32 | -------------------------------------------------------------------------------- /test/nonroot/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export LOCAL_DIR=$(mktemp -d) 10 | 11 | docker compose up -d --quiet-pull 12 | sleep 5 13 | 14 | docker compose logs backup 15 | 16 | # conf.d is used to confirm /etc files are also accessible for non-root users 17 | docker compose exec backup /bin/sh -c 'set -a; source /etc/dockervolumebackup/conf.d/01conf.env; set +a && backup' 18 | 19 | sleep 5 20 | 21 | expect_running_containers "3" 22 | 23 | if [ ! -f "$LOCAL_DIR/backup/test.tar.gz" ]; then 24 | fail "Could not find archive." 25 | fi 26 | pass "Archive was created." 27 | 28 | -------------------------------------------------------------------------------- /test/notifications/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | backup: 3 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 4 | restart: always 5 | environment: 6 | BACKUP_FILENAME: test.tar.gz 7 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 8 | BACKUP_PRUNING_PREFIX: test 9 | NOTIFICATION_LEVEL: info 10 | NOTIFICATION_URLS: ${NOTIFICATION_URLS} 11 | EXTRA_VALUE: extra-value 12 | volumes: 13 | - ${LOCAL_DIR:-./local}:/archive 14 | - app_data:/backup/app_data:ro 15 | - ./notifications.tmpl:/etc/dockervolumebackup/notifications.d/notifications.tmpl 16 | 17 | offen: 18 | image: offen/offen:latest 19 | labels: 20 | - docker-volume-backup.stop-during-backup=true 21 | volumes: 22 | - app_data:/var/opt/offen 23 | 24 | gotify: 25 | image: gotify/server 26 | ports: 27 | - 8080:80 28 | environment: 29 | - GOTIFY_DEFAULTUSER_PASS=custom 30 | volumes: 31 | - gotify_data:/app/data 32 | 33 | volumes: 34 | app_data: 35 | gotify_data: 36 | -------------------------------------------------------------------------------- /test/notifications/notifications.tmpl: -------------------------------------------------------------------------------- 1 | {{ define "title_success" -}} 2 | Successful test run with {{ env "EXTRA_VALUE" }}, yay! 3 | {{- end }} 4 | 5 | {{ define "body_success" -}} 6 | Backing up {{ .Stats.BackupFile.FullPath }} succeeded. 7 | {{- end }} 8 | -------------------------------------------------------------------------------- /test/notifications/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd $(dirname $0) 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export LOCAL_DIR=$(mktemp -d) 10 | 11 | docker compose up -d --quiet-pull 12 | sleep 5 13 | 14 | GOTIFY_TOKEN=$(curl -sSLX POST -H 'Content-Type: application/json' -d '{"name":"test"}' http://admin:custom@localhost:8080/application | jq -r '.token') 15 | info "Set up Gotify application using token $GOTIFY_TOKEN" 16 | 17 | docker compose exec backup backup 18 | 19 | NUM_MESSAGES=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages | length') 20 | if [ "$NUM_MESSAGES" != 0 ]; then 21 | fail "Expected no notifications to be sent when not configured" 22 | fi 23 | pass "No notifications were sent when not configured." 24 | 25 | docker compose down 26 | 27 | NOTIFICATION_URLS="gotify://gotify/${GOTIFY_TOKEN}?disableTLS=true" docker compose up -d 28 | 29 | docker compose exec backup backup 30 | 31 | NUM_MESSAGES=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages | length') 32 | if [ "$NUM_MESSAGES" != 1 ]; then 33 | fail "Expected one notifications to be sent when configured" 34 | fi 35 | pass "Correct number of notifications were sent when configured." 36 | 37 | MESSAGE_TITLE=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages[0].title') 38 | MESSAGE_BODY=$(curl -sSL http://admin:custom@localhost:8080/message | jq -r '.messages[0].message') 39 | 40 | if [ "$MESSAGE_TITLE" != "Successful test run with extra-value, yay!" ]; then 41 | fail "Unexpected notification title $MESSAGE_TITLE" 42 | fi 43 | pass "Custom notification title was used." 44 | 45 | if [ "$MESSAGE_BODY" != "Backing up /tmp/test.tar.gz succeeded." ]; then 46 | fail "Unexpected notification body $MESSAGE_BODY" 47 | fi 48 | pass "Custom notification body was used." 49 | -------------------------------------------------------------------------------- /test/ownership/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | db: 3 | image: postgres:14-alpine 4 | restart: unless-stopped 5 | labels: 6 | - docker-volume-backup.stop-during-backup=true 7 | volumes: 8 | - postgres_data:/var/lib/postgresql/data 9 | environment: 10 | POSTGRES_PASSWORD: 1FHJMSwt0yhIN1zS7I4DilGUhThBKq0x 11 | POSTGRES_USER: test 12 | POSTGRES_DB: test 13 | 14 | backup: 15 | image: offen/docker-volume-backup:${TEST_VERSION} 16 | restart: always 17 | environment: 18 | BACKUP_FILENAME: backup.tar.gz 19 | volumes: 20 | - postgres_data:/backup/postgres:ro 21 | - /var/run/docker.sock:/var/run/docker.sock:ro 22 | - ${LOCAL_DIR:-./local}:/archive 23 | 24 | volumes: 25 | postgres_data: 26 | -------------------------------------------------------------------------------- /test/ownership/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # This test refers to https://github.com/offen/docker-volume-backup/issues/71 3 | 4 | set -e 5 | 6 | cd $(dirname $0) 7 | . ../util.sh 8 | current_test=$(basename $(pwd)) 9 | 10 | export LOCAL_DIR=$(mktemp -d) 11 | 12 | docker compose up -d --quiet-pull 13 | sleep 5 14 | 15 | docker compose exec backup backup 16 | 17 | TMP_DIR=$(mktemp -d) 18 | tar --same-owner -xvf "$LOCAL_DIR/backup.tar.gz" -C $TMP_DIR 19 | 20 | find $TMP_DIR/backup/postgres > /dev/null 21 | pass "Backup contains files at expected location" 22 | 23 | for file in $(find $TMP_DIR/backup/postgres); do 24 | if [ "$(stat -c '%u:%g' $file)" != "70:70" ]; then 25 | fail "Unexpected file ownership for $file: $(stat -c '%u:%g' $file)" 26 | fi 27 | done 28 | pass "All files and directories in backup preserved their ownership." 29 | -------------------------------------------------------------------------------- /test/pgzip/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd $(dirname $0) 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | docker network create test_network 10 | docker volume create app_data 11 | 12 | LOCAL_DIR=$(mktemp -d) 13 | 14 | docker run -d -q \ 15 | --name offen \ 16 | --network test_network \ 17 | -v app_data:/var/opt/offen/ \ 18 | offen/offen:latest 19 | 20 | sleep 5 21 | 22 | docker run --rm -q \ 23 | --network test_network \ 24 | -v app_data:/backup/app_data \ 25 | -v $LOCAL_DIR:/archive \ 26 | -v /var/run/docker.sock:/var/run/docker.sock:ro \ 27 | --env BACKUP_COMPRESSION=gz \ 28 | --env GZIP_PARALLELISM=0 \ 29 | --env BACKUP_FILENAME='test.{{ .Extension }}' \ 30 | --entrypoint backup \ 31 | offen/docker-volume-backup:${TEST_VERSION:-canary} 32 | 33 | tmp_dir=$(mktemp -d) 34 | tar -xvf "$LOCAL_DIR/test.tar.gz" -C $tmp_dir 35 | if [ ! -f "$tmp_dir/backup/app_data/offen.db" ]; then 36 | fail "Could not find expected file in untared archive." 37 | fi 38 | pass "Found relevant files in untared local backup." 39 | 40 | # This test does not stop containers during backup. This is happening on 41 | # purpose in order to cover this setup as well. 42 | expect_running_containers "1" 43 | -------------------------------------------------------------------------------- /test/proxy/docker-compose.swarm.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2021 - offen.software 2 | # SPDX-License-Identifier: Unlicense 3 | 4 | services: 5 | backup: 6 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 7 | environment: 8 | BACKUP_FILENAME: test.tar.gz 9 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 10 | DOCKER_HOST: tcp://docker_socket_proxy:2375 11 | volumes: 12 | - pg_data:/backup/pg_data:ro 13 | - ${LOCAL_DIR:-local}:/archive 14 | 15 | docker_socket_proxy: 16 | image: tecnativa/docker-socket-proxy:0.1 17 | environment: 18 | INFO: ${ALLOW_INFO:-1} 19 | CONTAINERS: ${ALLOW_CONTAINERS:-1} 20 | SERVICES: ${ALLOW_SERVICES:-1} 21 | POST: ${ALLOW_POST:-1} 22 | TASKS: ${ALLOW_TASKS:-1} 23 | NODES: ${ALLOW_NODES:-1} 24 | volumes: 25 | - /var/run/docker.sock:/var/run/docker.sock:ro 26 | 27 | pg: 28 | image: postgres:14-alpine 29 | environment: 30 | POSTGRES_PASSWORD: example 31 | volumes: 32 | - pg_data:/var/lib/postgresql/data 33 | deploy: 34 | labels: 35 | - docker-volume-backup.stop-during-backup=true 36 | 37 | volumes: 38 | pg_data: 39 | -------------------------------------------------------------------------------- /test/proxy/docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2021 - offen.software 2 | # SPDX-License-Identifier: Unlicense 3 | 4 | services: 5 | backup: 6 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 7 | environment: 8 | BACKUP_FILENAME: test.tar.gz 9 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 10 | DOCKER_HOST: tcp://docker_socket_proxy:2375 11 | volumes: 12 | - pg_data:/backup/pg_data:ro 13 | - ${LOCAL_DIR:-local}:/archive 14 | 15 | docker_socket_proxy: 16 | image: tecnativa/docker-socket-proxy:0.1 17 | environment: 18 | INFO: ${ALLOW_INFO:-1} 19 | CONTAINERS: ${ALLOW_CONTAINERS:-1} 20 | POST: ${ALLOW_POST:-1} 21 | volumes: 22 | - /var/run/docker.sock:/var/run/docker.sock:ro 23 | 24 | pg: 25 | image: postgres:14-alpine 26 | environment: 27 | POSTGRES_PASSWORD: example 28 | volumes: 29 | - pg_data:/var/lib/postgresql/data 30 | labels: 31 | - docker-volume-backup.stop-during-backup=true 32 | 33 | volumes: 34 | pg_data: 35 | -------------------------------------------------------------------------------- /test/proxy/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd $(dirname $0) 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export LOCAL_DIR=$(mktemp -d) 10 | 11 | docker compose up -d --quiet-pull 12 | sleep 5 13 | 14 | # The default configuration in docker-compose.yml should 15 | # successfully create a backup. 16 | docker compose exec backup backup 17 | 18 | sleep 5 19 | 20 | expect_running_containers "3" 21 | 22 | if [ ! -f "$LOCAL_DIR/test.tar.gz" ]; then 23 | fail "Archive was not created" 24 | fi 25 | pass "Found relevant archive file." 26 | 27 | # Disabling POST should make the backup run fail 28 | ALLOW_POST="0" docker compose up -d 29 | sleep 5 30 | 31 | set +e 32 | docker compose exec backup backup 33 | if [ $? = "0" ]; then 34 | fail "Expected invocation to exit non-zero." 35 | fi 36 | set -e 37 | pass "Invocation exited non-zero." 38 | 39 | docker compose down --volumes 40 | 41 | # Next, the test is run against a Swarm setup 42 | 43 | docker swarm init 44 | 45 | export LOCAL_DIR=$(mktemp -d) 46 | 47 | docker stack deploy --compose-file=docker-compose.swarm.yml test_stack 48 | 49 | sleep 20 50 | 51 | # The default configuration in docker-compose.swarm.yml should 52 | # successfully create a backup in Swarm mode. 53 | docker exec $(docker ps -q -f name=backup) backup 54 | 55 | if [ ! -f "$LOCAL_DIR/test.tar.gz" ]; then 56 | fail "Archive was not created" 57 | fi 58 | 59 | pass "Found relevant archive file." 60 | 61 | sleep 5 62 | expect_running_containers "3" 63 | 64 | # Disabling POST should make the backup run fail 65 | ALLOW_POST="0" docker stack deploy --compose-file=docker-compose.swarm.yml test_stack 66 | 67 | sleep 20 68 | 69 | set +e 70 | docker exec $(docker ps -q -f name=backup) backup 71 | if [ $? = "0" ]; then 72 | fail "Expected invocation to exit non-zero." 73 | fi 74 | set -e 75 | 76 | pass "Invocation exited non-zero." 77 | -------------------------------------------------------------------------------- /test/pruning/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | minio: 3 | image: minio/minio:RELEASE.2020-08-04T23-10-51Z 4 | environment: 5 | MINIO_ROOT_USER: test 6 | MINIO_ROOT_PASSWORD: test 7 | MINIO_ACCESS_KEY: test 8 | MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ 9 | entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data' 10 | volumes: 11 | - minio_backup_data:/data 12 | 13 | backup: 14 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 15 | hostname: hostnametoken 16 | depends_on: 17 | - minio 18 | restart: always 19 | environment: 20 | AWS_ACCESS_KEY_ID: test 21 | AWS_SECRET_ACCESS_KEY: GMusLtUmILge2by+z890kQ 22 | AWS_ENDPOINT: minio:9000 23 | AWS_ENDPOINT_PROTO: http 24 | AWS_S3_BUCKET_NAME: backup 25 | BACKUP_FILENAME_EXPAND: 'true' 26 | BACKUP_FILENAME: test-$$HOSTNAME.tar.gz 27 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 28 | BACKUP_RETENTION_DAYS: 7 29 | BACKUP_PRUNING_LEEWAY: 5s 30 | BACKUP_PRUNING_PREFIX: test 31 | BACKUP_LATEST_SYMLINK: test-$$HOSTNAME.latest.tar.gz 32 | BACKUP_SKIP_BACKENDS_FROM_PRUNE: 's3' 33 | volumes: 34 | - app_data:/backup/app_data:ro 35 | - /var/run/docker.sock:/var/run/docker.sock:ro 36 | - ${LOCAL_DIR:-./local}:/archive 37 | 38 | offen: 39 | image: offen/offen:latest 40 | labels: 41 | - docker-volume-backup.stop-during-backup=true 42 | volumes: 43 | - app_data:/var/opt/offen 44 | 45 | volumes: 46 | app_data: 47 | minio_backup_data: 48 | name: minio_backup_data 49 | -------------------------------------------------------------------------------- /test/pruning/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Tests prune-skipping with multiple backends (local, s3) 4 | # Pruning itself is tested individually for each storage backend 5 | 6 | set -e 7 | 8 | cd "$(dirname "$0")" 9 | . ../util.sh 10 | current_test=$(basename $(pwd)) 11 | 12 | export LOCAL_DIR=$(mktemp -d) 13 | 14 | docker compose up -d --quiet-pull 15 | sleep 5 16 | 17 | docker compose exec backup backup 18 | 19 | sleep 5 20 | 21 | expect_running_containers "3" 22 | 23 | touch -r "$LOCAL_DIR/test-hostnametoken.tar.gz" -d "14 days ago" "$LOCAL_DIR/test-hostnametoken-old.tar.gz" 24 | 25 | docker run --rm \ 26 | -v minio_backup_data:/minio_data \ 27 | alpine \ 28 | ash -c 'touch -d@$(( $(date +%s) - 1209600 )) /minio_data/backup/test-hostnametoken-old.tar.gz' 29 | 30 | # Skip s3 backend from prune 31 | 32 | docker compose up -d 33 | sleep 5 34 | 35 | info "Create backup with no prune for s3 backend" 36 | docker compose exec backup backup 37 | 38 | info "Check if old backup has been pruned (local)" 39 | if [ -f "$LOCAL_DIR/test-hostnametoken-old.tar.gz" ]; then 40 | fail "Expired backup was not pruned from local storage." 41 | fi 42 | 43 | info "Check if old backup has NOT been pruned (s3)" 44 | docker run --rm \ 45 | -v minio_backup_data:/minio_data \ 46 | alpine \ 47 | ash -c 'test -f /minio_data/backup/test-hostnametoken-old.tar.gz' 48 | 49 | pass "Old remote backup has been pruned locally, skipped S3 backend is untouched." 50 | 51 | # Skip local and s3 backend from prune (all backends) 52 | 53 | touch -r "$LOCAL_DIR/test-hostnametoken.tar.gz" -d "14 days ago" "$LOCAL_DIR/test-hostnametoken-old.tar.gz" 54 | 55 | docker compose up -d 56 | sleep 5 57 | 58 | info "Create backup with no prune for both backends" 59 | docker compose exec -e BACKUP_SKIP_BACKENDS_FROM_PRUNE="s3,local" backup backup 60 | 61 | info "Check if old backup has NOT been pruned (local)" 62 | if [ ! -f "$LOCAL_DIR/test-hostnametoken-old.tar.gz" ]; then 63 | fail "Backdated file has not been deleted" 64 | fi 65 | 66 | info "Check if old backup has NOT been pruned (s3)" 67 | docker run --rm \ 68 | -v minio_backup_data:/minio_data \ 69 | alpine \ 70 | ash -c 'test -f /minio_data/backup/test-hostnametoken-old.tar.gz' 71 | 72 | pass "Skipped all backends while pruning." 73 | -------------------------------------------------------------------------------- /test/s3/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | minio: 3 | image: minio/minio:RELEASE.2020-08-04T23-10-51Z 4 | environment: 5 | MINIO_ROOT_USER: test 6 | MINIO_ROOT_PASSWORD: test 7 | MINIO_ACCESS_KEY: test 8 | MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ 9 | entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data' 10 | volumes: 11 | - minio_backup_data:/data 12 | 13 | backup: 14 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 15 | hostname: hostnametoken 16 | depends_on: 17 | - minio 18 | restart: always 19 | environment: 20 | AWS_ACCESS_KEY_ID: test 21 | AWS_SECRET_ACCESS_KEY: GMusLtUmILge2by+z890kQ 22 | AWS_ENDPOINT: minio:9000 23 | AWS_ENDPOINT_PROTO: http 24 | AWS_S3_BUCKET_NAME: backup 25 | BACKUP_FILENAME_EXPAND: 'true' 26 | BACKUP_FILENAME: test-$$HOSTNAME.tar.gz 27 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 28 | BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} 29 | BACKUP_PRUNING_LEEWAY: 5s 30 | BACKUP_PRUNING_PREFIX: test 31 | volumes: 32 | - app_data:/backup/app_data:ro 33 | - /var/run/docker.sock:/var/run/docker.sock:ro 34 | 35 | offen: 36 | image: offen/offen:latest 37 | labels: 38 | - docker-volume-backup.stop-during-backup=true 39 | volumes: 40 | - app_data:/var/opt/offen 41 | 42 | volumes: 43 | minio_backup_data: 44 | name: minio_backup_data 45 | app_data: 46 | -------------------------------------------------------------------------------- /test/s3/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | docker compose up -d --quiet-pull 10 | sleep 5 11 | 12 | docker compose exec backup backup 13 | 14 | sleep 5 15 | 16 | expect_running_containers "3" 17 | 18 | docker run --rm \ 19 | -v minio_backup_data:/minio_data \ 20 | alpine \ 21 | ash -c 'tar -xvf /minio_data/backup/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db' 22 | 23 | pass "Found relevant files in untared remote backups." 24 | 25 | # The second part of this test checks if backups get deleted when the retention 26 | # is set to 0 days (which it should not as it would mean all backups get deleted) 27 | BACKUP_RETENTION_DAYS="0" docker compose up -d 28 | sleep 5 29 | 30 | docker compose exec backup backup 31 | 32 | docker run --rm \ 33 | -v minio_backup_data:/minio_data \ 34 | alpine \ 35 | ash -c '[ $(find /minio_data/backup/ -type f | wc -l) = "1" ]' 36 | 37 | pass "Remote backups have not been deleted." 38 | 39 | # The third part of this test checks if old backups get deleted when the retention 40 | # is set to 7 days (which it should) 41 | 42 | BACKUP_RETENTION_DAYS="7" docker compose up -d 43 | sleep 5 44 | 45 | info "Create first backup with no prune" 46 | docker compose exec backup backup 47 | 48 | docker run --rm \ 49 | -v minio_backup_data:/minio_data \ 50 | alpine \ 51 | ash -c 'touch -d@$(( $(date +%s) - 1209600 )) /minio_data/backup/test-hostnametoken-old.tar.gz' 52 | 53 | info "Create second backup and prune" 54 | docker compose exec backup backup 55 | 56 | docker run --rm \ 57 | -v minio_backup_data:/minio_data \ 58 | alpine \ 59 | ash -c 'test ! -f /minio_data/backup/test-hostnametoken-old.tar.gz && test -f /minio_data/backup/test-hostnametoken.tar.gz' 60 | 61 | pass "Old remote backup has been pruned, new one is still present." 62 | -------------------------------------------------------------------------------- /test/secrets/docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2021 - offen.software 2 | # SPDX-License-Identifier: Unlicense 3 | 4 | services: 5 | minio: 6 | image: minio/minio:RELEASE.2020-08-04T23-10-51Z 7 | deploy: 8 | restart_policy: 9 | condition: on-failure 10 | environment: 11 | MINIO_ROOT_USER: test 12 | MINIO_ROOT_PASSWORD: test 13 | MINIO_ACCESS_KEY: test 14 | MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ 15 | entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data' 16 | volumes: 17 | - backup_data:/data 18 | 19 | backup: 20 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 21 | depends_on: 22 | - minio 23 | deploy: 24 | restart_policy: 25 | condition: on-failure 26 | environment: 27 | AWS_ACCESS_KEY_ID_FILE: /run/secrets/minio_root_user 28 | AWS_SECRET_ACCESS_KEY_FILE: /run/secrets/minio_root_password 29 | AWS_ENDPOINT: minio:9000 30 | AWS_ENDPOINT_PROTO: http 31 | AWS_S3_BUCKET_NAME: backup 32 | BACKUP_FILENAME: test.tar.gz 33 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 34 | BACKUP_RETENTION_DAYS: 7 35 | BACKUP_PRUNING_LEEWAY: 5s 36 | volumes: 37 | - pg_data:/backup/pg_data:ro 38 | - /var/run/docker.sock:/var/run/docker.sock:ro 39 | secrets: 40 | - minio_root_user 41 | - minio_root_password 42 | 43 | offen: 44 | image: offen/offen:latest 45 | labels: 46 | - docker-volume-backup.stop-during-backup=true 47 | healthcheck: 48 | disable: true 49 | deploy: 50 | replicas: 2 51 | restart_policy: 52 | condition: on-failure 53 | 54 | pg: 55 | image: postgres:14-alpine 56 | environment: 57 | POSTGRES_PASSWORD: example 58 | labels: 59 | - docker-volume-backup.stop-during-backup=true 60 | volumes: 61 | - pg_data:/var/lib/postgresql/data 62 | deploy: 63 | restart_policy: 64 | condition: on-failure 65 | 66 | volumes: 67 | backup_data: 68 | name: backup_data 69 | pg_data: 70 | name: pg_data 71 | 72 | secrets: 73 | minio_root_user: 74 | external: true 75 | minio_root_password: 76 | external: true 77 | -------------------------------------------------------------------------------- /test/secrets/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd $(dirname $0) 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | docker swarm init 10 | 11 | printf "test" | docker secret create minio_root_user - 12 | printf "GMusLtUmILge2by+z890kQ" | docker secret create minio_root_password - 13 | 14 | docker stack deploy --compose-file=docker-compose.yml test_stack 15 | 16 | while [ -z $(docker ps -q -f name=backup) ]; do 17 | info "Backup container not ready yet. Retrying." 18 | sleep 1 19 | done 20 | 21 | sleep 20 22 | 23 | docker exec $(docker ps -q -f name=backup) backup 24 | 25 | docker run --rm \ 26 | -v backup_data:/data alpine \ 27 | ash -c 'tar -xf /data/backup/test.tar.gz && test -f /backup/pg_data/PG_VERSION' 28 | 29 | pass "Found relevant files in untared backup." 30 | 31 | sleep 5 32 | expect_running_containers "5" 33 | 34 | docker exec -e AWS_ACCESS_KEY_ID=test $(docker ps -q -f name=backup) backup \ 35 | && fail "Backup should have failed due to duplicate env variables." 36 | 37 | pass "Backup failed due to duplicate env variables." 38 | 39 | docker exec -e AWS_ACCESS_KEY_ID_FILE=/tmp/nonexistant $(docker ps -q -f name=backup) backup \ 40 | && fail "Backup should have failed due to non existing file env variable." 41 | 42 | pass "Backup failed due to non existing file env variable." 43 | -------------------------------------------------------------------------------- /test/services/docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2021 - offen.software 2 | # SPDX-License-Identifier: Unlicense 3 | 4 | services: 5 | minio: 6 | image: minio/minio:RELEASE.2020-08-04T23-10-51Z 7 | environment: 8 | MINIO_ROOT_USER: test 9 | MINIO_ROOT_PASSWORD: test 10 | MINIO_ACCESS_KEY: test 11 | MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ 12 | entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data' 13 | volumes: 14 | - backup_data:/data 15 | 16 | backup: 17 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 18 | depends_on: 19 | - minio 20 | environment: 21 | AWS_ACCESS_KEY_ID: test 22 | AWS_SECRET_ACCESS_KEY: GMusLtUmILge2by+z890kQ 23 | AWS_ENDPOINT: minio:9000 24 | AWS_ENDPOINT_PROTO: http 25 | AWS_S3_BUCKET_NAME: backup 26 | BACKUP_FILENAME: test.tar.gz 27 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 28 | BACKUP_RETENTION_DAYS: 7 29 | BACKUP_PRUNING_LEEWAY: 5s 30 | volumes: 31 | - pg_data:/backup/pg_data:ro 32 | - /var/run/docker.sock:/var/run/docker.sock:ro 33 | 34 | offen: 35 | image: offen/offen:latest 36 | deploy: 37 | labels: 38 | - docker-volume-backup.stop-during-backup=true 39 | replicas: 2 40 | 41 | pg: 42 | image: postgres:14-alpine 43 | environment: 44 | POSTGRES_PASSWORD: example 45 | volumes: 46 | - pg_data:/var/lib/postgresql/data 47 | deploy: 48 | labels: 49 | - docker-volume-backup.stop-during-backup=true 50 | 51 | volumes: 52 | backup_data: 53 | name: backup_data 54 | pg_data: 55 | name: pg_data 56 | -------------------------------------------------------------------------------- /test/services/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd $(dirname $0) 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | docker swarm init 10 | 11 | docker stack deploy --compose-file=docker-compose.yml test_stack 12 | 13 | while [ -z $(docker ps -q -f name=backup) ]; do 14 | info "Backup container not ready yet. Retrying." 15 | sleep 1 16 | done 17 | 18 | sleep 20 19 | 20 | docker exec $(docker ps -q -f name=backup) backup 21 | 22 | docker run --rm \ 23 | -v backup_data:/data alpine \ 24 | ash -c 'tar -xf /data/backup/test.tar.gz && test -f /backup/pg_data/PG_VERSION' 25 | 26 | pass "Found relevant files in untared backup." 27 | 28 | sleep 5 29 | expect_running_containers "5" 30 | -------------------------------------------------------------------------------- /test/ssh/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | ssh: 3 | image: linuxserver/openssh-server:version-8.6_p1-r3 4 | environment: 5 | - PUID=1000 6 | - PGID=1000 7 | - USER_NAME=test 8 | volumes: 9 | - ${KEY_DIR:-.}/id_rsa.pub:/config/.ssh/authorized_keys 10 | - ssh_backup_data:/tmp 11 | 12 | backup: 13 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 14 | hostname: hostnametoken 15 | depends_on: 16 | - ssh 17 | restart: always 18 | environment: 19 | BACKUP_FILENAME_EXPAND: 'true' 20 | BACKUP_FILENAME: test-$$HOSTNAME.tar.gz 21 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 22 | BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} 23 | BACKUP_PRUNING_LEEWAY: 5s 24 | BACKUP_PRUNING_PREFIX: test 25 | SSH_HOST_NAME: ssh 26 | SSH_PORT: 2222 27 | SSH_USER: test 28 | SSH_REMOTE_PATH: /tmp 29 | SSH_IDENTITY_PASSPHRASE: test1234 30 | volumes: 31 | - ${KEY_DIR:-.}/id_rsa:/root/.ssh/id_rsa 32 | - app_data:/backup/app_data:ro 33 | - /var/run/docker.sock:/var/run/docker.sock:ro 34 | 35 | offen: 36 | image: offen/offen:latest 37 | labels: 38 | - docker-volume-backup.stop-during-backup=true 39 | volumes: 40 | - app_data:/var/opt/offen 41 | 42 | volumes: 43 | ssh_backup_data: 44 | name: ssh_backup_data 45 | app_data: 46 | -------------------------------------------------------------------------------- /test/ssh/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export KEY_DIR=$(mktemp -d) 10 | 11 | ssh-keygen -t rsa -m pem -b 4096 -N "test1234" -f "$KEY_DIR/id_rsa" -C "docker-volume-backup@local" 12 | 13 | docker compose up -d --quiet-pull 14 | sleep 5 15 | 16 | docker compose exec backup backup 17 | 18 | sleep 5 19 | 20 | expect_running_containers 3 21 | 22 | docker run --rm \ 23 | -v ssh_backup_data:/ssh_data \ 24 | alpine \ 25 | ash -c 'tar -xvf /ssh_data/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db' 26 | 27 | pass "Found relevant files in decrypted and untared remote backups." 28 | 29 | # The second part of this test checks if backups get deleted when the retention 30 | # is set to 0 days (which it should not as it would mean all backups get deleted) 31 | BACKUP_RETENTION_DAYS="0" docker compose up -d 32 | sleep 5 33 | 34 | docker compose exec backup backup 35 | 36 | docker run --rm \ 37 | -v ssh_backup_data:/ssh_data \ 38 | alpine \ 39 | ash -c '[ $(find /ssh_data/ -type f | wc -l) = "1" ]' 40 | 41 | pass "Remote backups have not been deleted." 42 | 43 | # The third part of this test checks if old backups get deleted when the retention 44 | # is set to 7 days (which it should) 45 | 46 | BACKUP_RETENTION_DAYS="7" docker compose up -d 47 | sleep 5 48 | 49 | info "Create first backup with no prune" 50 | docker compose exec backup backup 51 | 52 | # Set the modification date of the old backup to 14 days ago 53 | docker run --rm \ 54 | -v ssh_backup_data:/ssh_data \ 55 | --user 1000 \ 56 | alpine \ 57 | ash -c 'touch -d@$(( $(date +%s) - 1209600 )) /ssh_data/test-hostnametoken-old.tar.gz' 58 | 59 | info "Create second backup and prune" 60 | docker compose exec backup backup 61 | 62 | docker run --rm \ 63 | -v ssh_backup_data:/ssh_data \ 64 | alpine \ 65 | ash -c 'test ! -f /ssh_data/test-hostnametoken-old.tar.gz && test -f /ssh_data/test-hostnametoken.tar.gz' 66 | 67 | pass "Old remote backup has been pruned, new one is still present." 68 | -------------------------------------------------------------------------------- /test/swarm/docker-compose.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2020-2021 - offen.software 2 | # SPDX-License-Identifier: Unlicense 3 | 4 | services: 5 | minio: 6 | image: minio/minio:RELEASE.2020-08-04T23-10-51Z 7 | deploy: 8 | restart_policy: 9 | condition: on-failure 10 | environment: 11 | MINIO_ROOT_USER: test 12 | MINIO_ROOT_PASSWORD: test 13 | MINIO_ACCESS_KEY: test 14 | MINIO_SECRET_KEY: GMusLtUmILge2by+z890kQ 15 | entrypoint: /bin/ash -c 'mkdir -p /data/backup && minio server /data' 16 | volumes: 17 | - backup_data:/data 18 | 19 | backup: 20 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 21 | depends_on: 22 | - minio 23 | deploy: 24 | restart_policy: 25 | condition: on-failure 26 | environment: 27 | AWS_ACCESS_KEY_ID: test 28 | AWS_SECRET_ACCESS_KEY: GMusLtUmILge2by+z890kQ 29 | AWS_ENDPOINT: minio:9000 30 | AWS_ENDPOINT_PROTO: http 31 | AWS_S3_BUCKET_NAME: backup 32 | BACKUP_FILENAME: test.tar.gz 33 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 34 | BACKUP_RETENTION_DAYS: 7 35 | BACKUP_PRUNING_LEEWAY: 5s 36 | volumes: 37 | - pg_data:/backup/pg_data:ro 38 | - /var/run/docker.sock:/var/run/docker.sock:ro 39 | 40 | offen: 41 | image: offen/offen:latest 42 | labels: 43 | - docker-volume-backup.stop-during-backup=true 44 | healthcheck: 45 | disable: true 46 | deploy: 47 | replicas: 2 48 | restart_policy: 49 | condition: on-failure 50 | 51 | pg: 52 | image: postgres:14-alpine 53 | environment: 54 | POSTGRES_PASSWORD: example 55 | labels: 56 | - docker-volume-backup.stop-during-backup=true 57 | volumes: 58 | - pg_data:/var/lib/postgresql/data 59 | deploy: 60 | restart_policy: 61 | condition: on-failure 62 | 63 | volumes: 64 | backup_data: 65 | name: backup_data 66 | pg_data: 67 | name: pg_data 68 | -------------------------------------------------------------------------------- /test/swarm/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd $(dirname $0) 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | docker swarm init 10 | 11 | docker stack deploy --compose-file=docker-compose.yml test_stack 12 | 13 | while [ -z $(docker ps -q -f name=backup) ]; do 14 | info "Backup container not ready yet. Retrying." 15 | sleep 1 16 | done 17 | 18 | sleep 20 19 | 20 | docker exec $(docker ps -q -f name=backup) backup 21 | 22 | docker run --rm \ 23 | -v backup_data:/data alpine \ 24 | ash -c 'tar -xf /data/backup/test.tar.gz && test -f /backup/pg_data/PG_VERSION' 25 | 26 | pass "Found relevant files in untared backup." 27 | 28 | sleep 5 29 | expect_running_containers "5" 30 | -------------------------------------------------------------------------------- /test/tar/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | backup: 3 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 4 | restart: always 5 | environment: 6 | BACKUP_FILENAME: test.{{ .Extension }} 7 | BACKUP_COMPRESSION: none 8 | volumes: 9 | - app_data:/backup/app_data:ro 10 | - /var/run/docker.sock:/var/run/docker.sock:ro 11 | - ${LOCAL_DIR:-./local}:/archive 12 | 13 | offen: 14 | image: offen/offen:latest 15 | labels: 16 | - docker-volume-backup.stop-during-backup=true 17 | volumes: 18 | - app_data:/var/opt/offen 19 | 20 | volumes: 21 | app_data: 22 | -------------------------------------------------------------------------------- /test/tar/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export LOCAL_DIR=$(mktemp -d) 10 | 11 | docker compose up -d --quiet-pull 12 | sleep 5 13 | 14 | docker compose exec backup backup 15 | 16 | sleep 5 17 | 18 | expect_running_containers "2" 19 | 20 | tmp_dir=$(mktemp -d) 21 | tar -xvf "$LOCAL_DIR/test.tar" -C $tmp_dir 22 | if [ ! -f "$tmp_dir/backup/app_data/offen.db" ]; then 23 | fail "Could not find expected file in untared archive." 24 | fi 25 | pass "Expected file was found." 26 | -------------------------------------------------------------------------------- /test/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | MATCH_PATTERN=$1 6 | IMAGE_TAG=${IMAGE_TAG:-canary} 7 | 8 | sandbox="docker_volume_backup_test_sandbox" 9 | tarball="$(mktemp -d)/image.tar.gz" 10 | 11 | trap finish EXIT INT TERM 12 | 13 | finish () { 14 | rm -rf $(dirname $tarball) 15 | if [ ! -z $(docker ps -aq --filter=name=$sandbox) ]; then 16 | docker rm -f $(docker stop $sandbox) 17 | fi 18 | if [ ! -z $(docker volume ls -q --filter=name="^${sandbox}\$") ]; then 19 | docker volume rm $sandbox 20 | fi 21 | } 22 | 23 | docker build -t offen/docker-volume-backup:test-sandbox . 24 | 25 | if [ ! -z "$BUILD_IMAGE" ]; then 26 | docker build -t offen/docker-volume-backup:$IMAGE_TAG $(dirname $(pwd)) 27 | fi 28 | 29 | docker save offen/docker-volume-backup:$IMAGE_TAG -o $tarball 30 | 31 | find_args="-mindepth 1 -maxdepth 1 -type d" 32 | if [ ! -z "$MATCH_PATTERN" ]; then 33 | find_args="$find_args -name $MATCH_PATTERN" 34 | fi 35 | 36 | for dir in $(find $find_args | sort); do 37 | dir=$(echo $dir | cut -c 3-) 38 | echo "################################################" 39 | echo "Now running ${dir}" 40 | echo "################################################" 41 | echo "" 42 | 43 | test="${dir}/run.sh" 44 | docker_run_args="--name "$sandbox" --detach \ 45 | --privileged \ 46 | -v $(dirname $(pwd)):/code \ 47 | -v $tarball:/cache/image.tar.gz \ 48 | -v $sandbox:/var/lib/docker" 49 | 50 | if [ -z "$NO_IMAGE_CACHE" ]; then 51 | docker_run_args="$docker_run_args \ 52 | -v "${sandbox}_image":/var/lib/docker/image \ 53 | -v "${sandbox}_overlay2":/var/lib/docker/overlay2" 54 | fi 55 | 56 | docker run $docker_run_args offen/docker-volume-backup:test-sandbox 57 | 58 | retry_counter=0 59 | until timeout 5 docker exec $sandbox /bin/sh -c 'docker info' > /dev/null 2>&1; do 60 | if [ $retry_counter -gt 20 ]; then 61 | echo "Gave up waiting for Docker daemon to become ready after 20 attempts" 62 | exit 1 63 | fi 64 | 65 | if [ "$(docker inspect $sandbox --format '{{ .State.Running }}')" = "false" ]; then 66 | docker rm $sandbox 67 | docker run $docker_run_args offen/docker-volume-backup:test-sandbox 68 | fi 69 | 70 | sleep 0.5 71 | retry_counter=$((retry_counter+1)) 72 | done 73 | 74 | docker exec $sandbox /bin/sh -c "docker load -i /cache/image.tar.gz" 75 | docker exec -e TEST_VERSION=$IMAGE_TAG $sandbox /bin/sh -c "/code/test/$test" 76 | 77 | docker rm $(docker stop $sandbox) 78 | docker volume rm $sandbox 79 | echo "" 80 | echo "$test passed" 81 | echo "" 82 | done 83 | -------------------------------------------------------------------------------- /test/user/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | alpine: 3 | image: alpine:3.17.3 4 | tty: true 5 | volumes: 6 | - app_data:/tmp 7 | labels: 8 | - docker-volume-backup.archive-pre.user=testuser 9 | - docker-volume-backup.archive-pre=/bin/sh -c 'whoami > /tmp/whoami.txt' 10 | 11 | backup: 12 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 13 | deploy: 14 | restart_policy: 15 | condition: on-failure 16 | environment: 17 | BACKUP_FILENAME: test.tar.gz 18 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 19 | EXEC_FORWARD_OUTPUT: "true" 20 | volumes: 21 | - ${LOCAL_DIR:-./local}:/archive 22 | - app_data:/backup/data:ro 23 | - /var/run/docker.sock:/var/run/docker.sock:ro 24 | 25 | volumes: 26 | app_data: 27 | archive: 28 | -------------------------------------------------------------------------------- /test/user/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd $(dirname $0) 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | export LOCAL_DIR=$(mktemp -d) 10 | export TMP_DIR=$(mktemp -d) 11 | 12 | echo "LOCAL_DIR $LOCAL_DIR" 13 | echo "TMP_DIR $TMP_DIR" 14 | 15 | docker compose up -d --quiet-pull 16 | user_name=testuser 17 | docker exec user-alpine-1 adduser --disabled-password "$user_name" 18 | 19 | docker compose exec backup backup 20 | 21 | tar -xvf "$LOCAL_DIR/test.tar.gz" -C "$TMP_DIR" 22 | if [ ! -f "$TMP_DIR/backup/data/whoami.txt" ]; then 23 | fail "Could not find file written by pre command." 24 | fi 25 | pass "Found expected file." 26 | 27 | if [ "$(cat $TMP_DIR/backup/data/whoami.txt)" != "$user_name" ]; then 28 | fail "Could not find expected user name." 29 | fi 30 | pass "Found expected user." 31 | -------------------------------------------------------------------------------- /test/util.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | info () { 6 | echo "[test:${current_test:-none}:info] "$1"" 7 | } 8 | 9 | pass () { 10 | echo "[test:${current_test:-none}:pass] "$1"" 11 | } 12 | 13 | fail () { 14 | echo "[test:${current_test:-none}:fail] "$1"" 15 | exit 1 16 | } 17 | 18 | skip () { 19 | echo "[test:${current_test:-none}:skip] "$1"" 20 | exit 0 21 | } 22 | 23 | expect_running_containers () { 24 | if [ "$(docker ps -q | wc -l)" != "$1" ]; then 25 | fail "Expected $1 containers to be running, instead seen: "$(docker ps -q | wc -l)"" 26 | fi 27 | pass "$1 containers running." 28 | } 29 | 30 | docker() { 31 | case $1 in 32 | compose) 33 | shift 34 | case $1 in 35 | up) 36 | shift 37 | command docker compose up --timeout 3 "$@";; 38 | down) 39 | shift 40 | command docker compose down --timeout 3 "$@";; 41 | *) 42 | command docker compose "$@";; 43 | esac 44 | ;; 45 | *) 46 | command docker "$@";; 47 | esac 48 | } 49 | -------------------------------------------------------------------------------- /test/webdav/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | webdav: 3 | image: bytemark/webdav:2.4 4 | environment: 5 | AUTH_TYPE: Digest 6 | USERNAME: test 7 | PASSWORD: test 8 | volumes: 9 | - webdav_backup_data:/var/lib/dav 10 | 11 | backup: 12 | image: offen/docker-volume-backup:${TEST_VERSION:-canary} 13 | hostname: hostnametoken 14 | depends_on: 15 | - webdav 16 | restart: always 17 | environment: 18 | BACKUP_FILENAME_EXPAND: 'true' 19 | BACKUP_FILENAME: test-$$HOSTNAME.tar.gz 20 | BACKUP_CRON_EXPRESSION: 0 0 5 31 2 ? 21 | BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-7} 22 | BACKUP_PRUNING_LEEWAY: 5s 23 | BACKUP_PRUNING_PREFIX: test 24 | WEBDAV_URL: http://webdav/ 25 | WEBDAV_URL_INSECURE: 'true' 26 | WEBDAV_PATH: /my/new/path/ 27 | WEBDAV_USERNAME: test 28 | WEBDAV_PASSWORD: test 29 | volumes: 30 | - app_data:/backup/app_data:ro 31 | - /var/run/docker.sock:/var/run/docker.sock:ro 32 | 33 | offen: 34 | image: offen/offen:latest 35 | labels: 36 | - docker-volume-backup.stop-during-backup=true 37 | volumes: 38 | - app_data:/var/opt/offen 39 | 40 | volumes: 41 | webdav_backup_data: 42 | name: webdav_backup_data 43 | app_data: 44 | -------------------------------------------------------------------------------- /test/webdav/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")" 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | docker compose up -d --quiet-pull 10 | sleep 5 11 | 12 | docker compose exec backup backup 13 | 14 | sleep 5 15 | 16 | expect_running_containers "3" 17 | 18 | docker run --rm \ 19 | -v webdav_backup_data:/webdav_data \ 20 | alpine \ 21 | ash -c 'tar -xvf /webdav_data/data/my/new/path/test-hostnametoken.tar.gz -C /tmp && test -f /tmp/backup/app_data/offen.db' 22 | 23 | pass "Found relevant files in untared remote backup." 24 | 25 | # The second part of this test checks if backups get deleted when the retention 26 | # is set to 0 days (which it should not as it would mean all backups get deleted) 27 | BACKUP_RETENTION_DAYS="0" docker compose up -d 28 | sleep 5 29 | 30 | docker compose exec backup backup 31 | 32 | docker run --rm \ 33 | -v webdav_backup_data:/webdav_data \ 34 | alpine \ 35 | ash -c '[ $(find /webdav_data/data/my/new/path/ -type f | wc -l) = "1" ]' 36 | 37 | pass "Remote backups have not been deleted." 38 | 39 | # The third part of this test checks if old backups get deleted when the retention 40 | # is set to 7 days (which it should) 41 | 42 | BACKUP_RETENTION_DAYS="7" docker compose up -d 43 | sleep 5 44 | 45 | info "Create first backup with no prune" 46 | docker compose exec backup backup 47 | 48 | # Set the modification date of the old backup to 14 days ago 49 | docker run --rm \ 50 | -v webdav_backup_data:/webdav_data \ 51 | --user 82 \ 52 | alpine \ 53 | ash -c 'touch -d@$(( $(date +%s) - 1209600 )) /webdav_data/data/my/new/path/test-hostnametoken-old.tar.gz' 54 | 55 | info "Create second backup and prune" 56 | docker compose exec backup backup 57 | 58 | docker run --rm \ 59 | -v webdav_backup_data:/webdav_data \ 60 | alpine \ 61 | ash -c 'test ! -f /webdav_data/data/my/new/path/test-hostnametoken-old.tar.gz && test -f /webdav_data/data/my/new/path/test-hostnametoken.tar.gz' 62 | 63 | pass "Old remote backup has been pruned, new one is still present." 64 | -------------------------------------------------------------------------------- /test/zstd/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | cd $(dirname $0) 6 | . ../util.sh 7 | current_test=$(basename $(pwd)) 8 | 9 | docker network create test_network 10 | docker volume create app_data 11 | 12 | LOCAL_DIR=$(mktemp -d) 13 | 14 | docker run -d -q \ 15 | --name offen \ 16 | --network test_network \ 17 | -v app_data:/var/opt/offen/ \ 18 | offen/offen:latest 19 | 20 | sleep 10 21 | 22 | docker run --rm -q \ 23 | --network test_network \ 24 | -v app_data:/backup/app_data \ 25 | -v $LOCAL_DIR:/archive \ 26 | -v /var/run/docker.sock:/var/run/docker.sock:ro \ 27 | --env BACKUP_COMPRESSION=zst \ 28 | --env BACKUP_FILENAME='test.{{ .Extension }}' \ 29 | --entrypoint backup \ 30 | offen/docker-volume-backup:${TEST_VERSION:-canary} 31 | 32 | tmp_dir=$(mktemp -d) 33 | tar -xvf "$LOCAL_DIR/test.tar.zst" --zstd -C $tmp_dir 34 | if [ ! -f "$tmp_dir/backup/app_data/offen.db" ]; then 35 | fail "Could not find expected file in untared archive." 36 | fi 37 | pass "Found relevant files in untared local backup." 38 | 39 | # This test does not stop containers during backup. This is happening on 40 | # purpose in order to cover this setup as well. 41 | expect_running_containers "1" 42 | --------------------------------------------------------------------------------