├── prepost_strategies ├── mongodb │ ├── Dockerfile │ ├── postexecute │ │ └── restore │ │ │ └── mongorestore.sh │ ├── preexecute │ │ ├── backup │ │ │ └── mongodump.sh │ │ └── utils │ │ │ └── check-env.sh │ └── README.md └── mysql │ ├── Dockerfile │ ├── postexecute │ └── restore │ │ └── mysqlimport.sh │ ├── preexecute │ ├── backup │ │ └── mysqldump.sh │ └── utils │ │ └── check-env.sh │ └── README.md ├── scripts ├── cleanCacheLocks ├── cleanup ├── list ├── remove-all-but-n-full ├── remove-older-than ├── remove-all-inc-of-but-n-full ├── verify ├── backup ├── periodicBackup ├── backupFull ├── backupIncremental └── restore ├── Jenkinsfile ├── imagescripts ├── create_gdocs_file.sh ├── stopContainers.sh ├── startContainers.sh ├── prepoststrategy ├── docker-entrypoint.sh ├── create_jobber.sh └── base.sh ├── dc-pwd.yml ├── LICENSE ├── .circleci └── config.yml ├── backends ├── Mega │ └── README.md ├── BackblazeB2 │ └── README.md ├── Dropbox │ └── README.md ├── GoogleDrive │ └── README.md └── AmazonS3 │ └── README.md ├── Dockerfile └── README.md /prepost_strategies/mongodb/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM blacklabelops/volumerize 2 | 3 | RUN apk add --no-cache \ 4 | mongodb-tools 5 | 6 | COPY postexecute /postexecute 7 | COPY preexecute /preexecute -------------------------------------------------------------------------------- /prepost_strategies/mysql/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM blacklabelops/volumerize 2 | 3 | RUN apk add --no-cache \ 4 | mysql-client pv 5 | 6 | COPY postexecute /postexecute 7 | COPY preexecute /preexecute 8 | -------------------------------------------------------------------------------- /scripts/cleanCacheLocks: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | [[ ${DEBUG} == true ]] && set -x 6 | 7 | source /opt/volumerize/base.sh 8 | 9 | find /volumerize-cache/ -maxdepth 2 -type f -name lockfile.lock | while read filename ; do fuser -s $filename || rm -fv $filename ; done 10 | -------------------------------------------------------------------------------- /prepost_strategies/mysql/postexecute/restore/mysqlimport.sh: -------------------------------------------------------------------------------- 1 | source /preexecute/utils/check-env.sh 2 | 3 | check_env "mysqlimport" "MYSQL_PASSWORD" "MYSQL_USERNAME" "MYSQL_HOST" "MYSQL_DATABASE" 4 | 5 | echo "mysql import starts" 6 | pv ${VOLUMERIZE_SOURCE}/dump-${MYSQL_DATABASE}.sql | mysql -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD} $MYSQL_DATABASE 7 | echo "Import done" -------------------------------------------------------------------------------- /prepost_strategies/mongodb/postexecute/restore/mongorestore.sh: -------------------------------------------------------------------------------- 1 | source /preexecute/utils/check-env.sh 2 | 3 | check_env "mongorestore" "MONGO_USERNAME" "MONGO_PASSWORD" "MONGO_HOST" "MONGO_PORT" 4 | 5 | echo "mongorestore starts" 6 | mongorestore --host ${MONGO_HOST} --port ${MONGO_PORT} --username ${MONGO_USERNAME} --password "${MONGO_PASSWORD}" ${VOLUMERIZE_SOURCE} 7 | echo "Import done" -------------------------------------------------------------------------------- /prepost_strategies/mongodb/preexecute/backup/mongodump.sh: -------------------------------------------------------------------------------- 1 | source /preexecute/utils/check-env.sh 2 | 3 | check_env "mongodump" "MONGO_USERNAME" "MONGO_PASSWORD" "MONGO_HOST" "MONGO_PORT" 4 | 5 | echo "Creating $VOLUMERIZE_SOURCE folder if not exists" 6 | mkdir -p $VOLUMERIZE_SOURCE 7 | 8 | echo "mongodump starts" 9 | mongodump --host ${MONGO_HOST} --port ${MONGO_PORT} --username ${MONGO_USERNAME} --password "${MONGO_PASSWORD}" --out ${VOLUMERIZE_SOURCE} -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | /** Jenkins 2.0 Buildfile 2 | * 3 | * Slave 'docker' can be started by typing: 4 | * docker run -d -v /var/run/docker.sock:/var/run/docker.sock --link jenkins:jenkins -e "SWARM_CLIENT_LABELS=docker" blacklabelops/swarm-dockerhost 5 | **/ 6 | node { 7 | checkout scm 8 | stage 'Build Images' 9 | parallel("image-alpine": { load './buildscripts/alpineBuildImages.groovy' }) 10 | stage 'Test Images' 11 | parallel("image-alpine": { load './buildscripts/alpineTestImages.groovy' }) 12 | } 13 | -------------------------------------------------------------------------------- /imagescripts/create_gdocs_file.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | if [ -n "${GOOGLE_DRIVE_ID}" ] && [ -n "${GOOGLE_DRIVE_SECRET}" ]; then 6 | cat > /credentials/cred.file <<_EOF_ 7 | client_config_backend: settings 8 | client_config: 9 | client_id: ${GOOGLE_DRIVE_ID} 10 | client_secret: ${GOOGLE_DRIVE_SECRET} 11 | save_credentials: True 12 | save_credentials_backend: file 13 | save_credentials_file: ${GOOGLE_DRIVE_CREDENTIAL_FILE} 14 | get_refresh_token: True 15 | _EOF_ 16 | fi 17 | -------------------------------------------------------------------------------- /imagescripts/stopContainers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | [[ ${DEBUG} == true ]] && set -x 6 | 7 | function stopContainers() { 8 | local arrayContainers= 9 | IFS=' ' read -r -a arrayContainers <<< "$1" 10 | local min=0 11 | local max=$(( ${#arrayContainers[@]} )) 12 | 13 | for (( i=$min; i<$max; i++ )) 14 | do 15 | docker stop "${arrayContainers[$i]}" || true 16 | done 17 | } 18 | 19 | if [ -n "${VOLUMERIZE_CONTAINERS}" ]; then 20 | stopContainers "${VOLUMERIZE_CONTAINERS}" 21 | fi 22 | -------------------------------------------------------------------------------- /imagescripts/startContainers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | [[ ${DEBUG} == true ]] && set -x 6 | 7 | function startContainers() { 8 | local arrayContainers= 9 | IFS=' ' read -r -a arrayContainers <<< "$1" 10 | local min=0 11 | local max=$(( ${#arrayContainers[@]} -1)) 12 | 13 | for (( i=$max; i>=$min; i-- )) 14 | do 15 | docker start ${arrayContainers[$i]} || true 16 | done 17 | } 18 | 19 | if [ -n "${VOLUMERIZE_CONTAINERS}" ]; then 20 | startContainers "${VOLUMERIZE_CONTAINERS}" 21 | fi 22 | -------------------------------------------------------------------------------- /dc-pwd.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | volumerize: 5 | depends_on: 6 | - jenkins 7 | image: blacklabelops/volumerize 8 | container_name: volumerize 9 | volumes: 10 | - jenkins_volume:/source 11 | - backup_volume:/backup 12 | - cache_volume:/volumerize-cache 13 | environment: 14 | - 'VOLUMERIZE_SOURCE=/source' 15 | - 'VOLUMERIZE_TARGET=file:///backup' 16 | 17 | jenkins: 18 | image: blacklabelops/jenkins 19 | container_name: jenkins 20 | volumes: 21 | - jenkins_volume:/jenkins 22 | ports: 23 | - '80:8080' 24 | 25 | volumes: 26 | jenkins_volume: 27 | external: false 28 | backup_volume: 29 | external: false 30 | cache_volume: 31 | external: false 32 | -------------------------------------------------------------------------------- /imagescripts/prepoststrategy: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | strategy_path="" 6 | 7 | case $1 in 8 | preAction ) strategy_path=/preexecute ;; 9 | postAction ) strategy_path=/postexecute ;; 10 | *) echo "Error: prepoststrategy first parameter 'execution phase' must be preAction or postAction"; exit 1 ;; 11 | esac 12 | 13 | case $2 in 14 | backup | verify | restore ) ;; 15 | *) echo "Error: porepoststrategy second parameter 'duplicity action' must be backup, verify or restore"; exit 1 ;; 16 | esac 17 | 18 | strategy_path=$strategy_path/$2 19 | 20 | if [ -d "$strategy_path" ]; then 21 | for f in $strategy_path/*; do 22 | case "$f" in 23 | *.sh) echo "running $f"; . "$f" ;; 24 | *) echo "ignoring $f" ;; 25 | esac 26 | echo 27 | done 28 | fi 29 | -------------------------------------------------------------------------------- /prepost_strategies/mysql/preexecute/backup/mysqldump.sh: -------------------------------------------------------------------------------- 1 | source /preexecute/utils/check-env.sh 2 | 3 | check_env "Mysqldump" "MYSQL_PASSWORD" "MYSQL_USERNAME" "MYSQL_HOST" "MYSQL_DATABASE" 4 | 5 | echo "Creating $VOLUMERIZE_SOURCE folder if not exists" 6 | mkdir -p $VOLUMERIZE_SOURCE 7 | 8 | # Based on this answer https://stackoverflow.com/a/32361604 9 | SIZE_BYTES=$(mysql --skip-column-names -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD} $MYSQL_DATABASE -e "SELECT ROUND(SUM(data_length * 0.8), 0) FROM information_schema.TABLES WHERE table_schema='${MYSQL_DATABASE}';") 10 | 11 | echo "mysqldump starts (Progress is aproximated)" 12 | mysqldump --databases "${MYSQL_DATABASE}" --single-transaction --add-drop-database --user="${MYSQL_USERNAME}" --password="${MYSQL_PASSWORD}" --host="${MYSQL_HOST}" | pv --progress --size "$SIZE_BYTES" > ${VOLUMERIZE_SOURCE}/dump-${MYSQL_DATABASE}.sql -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Steffen Bleul 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /scripts/cleanup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | [[ ${DEBUG} == true ]] && set -x 6 | 7 | source /opt/volumerize/base.sh 8 | 9 | DUPLICITY_RETURN_CODE=0 10 | 11 | function commandLoop() { 12 | local jobcount=$JOB_COUNT 13 | local counter; 14 | 15 | for (( counter=1; counter<=$jobcount; counter++ )) 16 | do 17 | commandJob "$counter" $@ 18 | done 19 | } 20 | 21 | function commandJob() { 22 | local jobNumber=$1 23 | local returnCode=0; 24 | prepareJob "$jobNumber" 25 | shift; 26 | ${DUPLICITY_JOB_COMMAND} cleanup $@ ${DUPLICITY_JOB_OPTIONS} ${VOLUMERIZE_JOB_INCUDES} ${VOLUMERIZE_JOB_TARGET} || returnCode=$? && true ; 27 | if [ "$returnCode" -gt "$DUPLICITY_RETURN_CODE" ]; then 28 | DUPLICITY_RETURN_CODE=$returnCode 29 | fi 30 | } 31 | 32 | function commandExecution() { 33 | if [ -n "${VOLUMERIZE_SOURCE}" ]; then 34 | ${DUPLICITY_COMMAND} cleanup $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET} || DUPLICITY_RETURN_CODE=$? && true ; 35 | elif [ ! -z "${1##*[!0-9]*}" ]; then 36 | commandJob $@ 37 | elif [ -n "${JOB_COUNT}" ]; then 38 | commandLoop $@ 39 | fi 40 | } 41 | 42 | commandExecution $@ 43 | exit $DUPLICITY_RETURN_CODE 44 | -------------------------------------------------------------------------------- /prepost_strategies/mongodb/preexecute/utils/check-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ## 4 | # @private 5 | # Executed if a variable is not set or is empty string 6 | # @param1 - Name of the failed environment variable 7 | ## 8 | function _check_env_failed { 9 | echo "Environment variable $1 is not set." 10 | echo "Environment variables failed, exit 1" 11 | exit 1 12 | } 13 | 14 | ## 15 | # @private 16 | # Executed if a variable is setted 17 | # @param1 - Name of the environment variable 18 | ## 19 | function _check_env_ok { 20 | echo "Env var $1 ok." 21 | } 22 | 23 | ## 24 | # Use it to check if environment variables are set 25 | # @param1 - Name of the context 26 | # @param2 to ∞ - Environment variables to check 27 | ## 28 | function check_env { 29 | echo "Checking environment variables for $1." 30 | 31 | for e_var in "$@"; do 32 | if [ $e_var = $1 ]; then continue; fi # Jump first arg 33 | 34 | # Check if env var is setted, if not raise error 35 | if [ "${!e_var}" = "" ]; then 36 | _check_env_failed $e_var; 37 | else 38 | _check_env_ok $e_var; 39 | fi 40 | 41 | done 42 | echo "Environment variables ok." 43 | } -------------------------------------------------------------------------------- /prepost_strategies/mysql/preexecute/utils/check-env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ## 4 | # @private 5 | # Executed if a variable is not set or is empty string 6 | # @param1 - Name of the failed environment variable 7 | ## 8 | function _check_env_failed { 9 | echo "Environment variable $1 is not set." 10 | echo "Environment variables failed, exit 1" 11 | exit 1 12 | } 13 | 14 | ## 15 | # @private 16 | # Executed if a variable is setted 17 | # @param1 - Name of the environment variable 18 | ## 19 | function _check_env_ok { 20 | echo "Env var $1 ok." 21 | } 22 | 23 | ## 24 | # Use it to check if environment variables are set 25 | # @param1 - Name of the context 26 | # @param2 to ∞ - Environment variables to check 27 | ## 28 | function check_env { 29 | echo "Checking environment variables for $1." 30 | 31 | for e_var in "$@"; do 32 | if [ $e_var = $1 ]; then continue; fi # Jump first arg 33 | 34 | # Check if env var is setted, if not raise error 35 | if [ "${!e_var}" = "" ]; then 36 | _check_env_failed $e_var; 37 | else 38 | _check_env_ok $e_var; 39 | fi 40 | 41 | done 42 | echo "Environment variables ok." 43 | } -------------------------------------------------------------------------------- /scripts/list: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | [[ ${DEBUG} == true ]] && set -x 6 | 7 | source /opt/volumerize/base.sh 8 | 9 | DUPLICITY_RETURN_CODE=0 10 | 11 | function commandLoop() { 12 | local jobcount=$JOB_COUNT 13 | local counter; 14 | 15 | for (( counter=1; counter<=$jobcount; counter++ )) 16 | do 17 | commandJob "$counter" $@ 18 | done 19 | } 20 | 21 | function commandJob() { 22 | local jobNumber=$1 23 | local returnCode=0; 24 | prepareJob "$jobNumber" 25 | shift; 26 | ${DUPLICITY_JOB_COMMAND} collection-status $@ ${DUPLICITY_JOB_OPTIONS} ${VOLUMERIZE_JOB_INCUDES} ${VOLUMERIZE_JOB_TARGET} || returnCode=$? && true ; 27 | if [ "$returnCode" -gt "$DUPLICITY_RETURN_CODE" ]; then 28 | DUPLICITY_RETURN_CODE=$returnCode 29 | fi 30 | } 31 | 32 | function commandExecution() { 33 | if [ -n "${VOLUMERIZE_SOURCE}" ]; then 34 | ${DUPLICITY_COMMAND} collection-status $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET} || DUPLICITY_RETURN_CODE=$? && true ; 35 | elif [ ! -z "${1##*[!0-9]*}" ]; then 36 | commandJob $@ 37 | elif [ -n "${JOB_COUNT}" ]; then 38 | commandLoop $@ 39 | fi 40 | } 41 | 42 | commandExecution $@ 43 | exit $DUPLICITY_RETURN_CODE 44 | -------------------------------------------------------------------------------- /scripts/remove-all-but-n-full: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | [[ ${DEBUG} == true ]] && set -x 6 | 7 | source /opt/volumerize/base.sh 8 | 9 | DUPLICITY_RETURN_CODE=0 10 | 11 | function commandLoop() { 12 | local jobcount=$JOB_COUNT 13 | local counter; 14 | 15 | for (( counter=1; counter<=$jobcount; counter++ )) 16 | do 17 | commandJob "$counter" $@ 18 | done 19 | } 20 | 21 | function commandJob() { 22 | local jobNumber=$1 23 | local returnCode=0; 24 | prepareJob "$jobNumber" 25 | shift; 26 | ${DUPLICITY_JOB_COMMAND} remove-all-but-n-full $@ ${DUPLICITY_JOB_OPTIONS} ${VOLUMERIZE_JOB_TARGET} || returnCode=$? && true ; 27 | if [ "$returnCode" -gt "$DUPLICITY_RETURN_CODE" ]; then 28 | DUPLICITY_RETURN_CODE=$returnCode 29 | fi 30 | } 31 | 32 | function commandExecution() { 33 | if [ -n "${VOLUMERIZE_SOURCE}" ]; then 34 | ${DUPLICITY_COMMAND} remove-all-but-n-full $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_TARGET} || DUPLICITY_RETURN_CODE=$? && true ; 35 | elif [ ! -z "${1##*[!0-9]*}" ] && [ ! -z "${2##*[!0-9]*}" ]; then 36 | commandJob $@ 37 | elif [ -n "${JOB_COUNT}" ]; then 38 | commandLoop $@ 39 | fi 40 | } 41 | 42 | commandExecution $@ 43 | exit $DUPLICITY_RETURN_CODE 44 | -------------------------------------------------------------------------------- /scripts/remove-older-than: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | [[ ${DEBUG} == true ]] && set -x 6 | 7 | source /opt/volumerize/base.sh 8 | 9 | DUPLICITY_RETURN_CODE=0 10 | 11 | function commandLoop() { 12 | local jobcount=$JOB_COUNT 13 | local counter; 14 | 15 | for (( counter=1; counter<=$jobcount; counter++ )) 16 | do 17 | commandJob "$counter" $@ 18 | done 19 | } 20 | 21 | function commandJob() { 22 | local jobNumber=$1 23 | local returnCode=0; 24 | prepareJob "$jobNumber" 25 | shift; 26 | ${DUPLICITY_JOB_COMMAND} remove-older-than $@ ${DUPLICITY_JOB_OPTIONS} ${VOLUMERIZE_JOB_INCUDES} ${VOLUMERIZE_JOB_TARGET} || returnCode=$? && true ; 27 | if [ "$returnCode" -gt "$DUPLICITY_RETURN_CODE" ]; then 28 | DUPLICITY_RETURN_CODE=$returnCode 29 | fi 30 | } 31 | 32 | function commandExecution() { 33 | if [ -n "${VOLUMERIZE_SOURCE}" ]; then 34 | ${DUPLICITY_COMMAND} remove-older-than $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET} || DUPLICITY_RETURN_CODE=$? && true ; 35 | elif [ ! -z "${1##*[!0-9]*}" ]; then 36 | commandJob $@ 37 | elif [ -n "${JOB_COUNT}" ]; then 38 | commandLoop $@ 39 | fi 40 | } 41 | 42 | commandExecution $@ 43 | exit $DUPLICITY_RETURN_CODE 44 | -------------------------------------------------------------------------------- /scripts/remove-all-inc-of-but-n-full: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | [[ ${DEBUG} == true ]] && set -x 6 | 7 | source /opt/volumerize/base.sh 8 | 9 | DUPLICITY_RETURN_CODE=0 10 | 11 | function commandLoop() { 12 | local jobcount=$JOB_COUNT 13 | local counter; 14 | 15 | for (( counter=1; counter<=$jobcount; counter++ )) 16 | do 17 | commandJob "$counter" $@ 18 | done 19 | } 20 | 21 | function commandJob() { 22 | local jobNumber=$1 23 | local returnCode=0; 24 | prepareJob "$jobNumber" 25 | shift; 26 | ${DUPLICITY_JOB_COMMAND} remove-all-inc-of-but-n-full $@ ${DUPLICITY_JOB_OPTIONS} ${VOLUMERIZE_JOB_TARGET} || returnCode=$? && true ; 27 | if [ "$returnCode" -gt "$DUPLICITY_RETURN_CODE" ]; then 28 | DUPLICITY_RETURN_CODE=$returnCode 29 | fi 30 | } 31 | 32 | function commandExecution() { 33 | if [ -n "${VOLUMERIZE_SOURCE}" ]; then 34 | ${DUPLICITY_COMMAND} remove-all-inc-of-but-n-full $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_TARGET} || DUPLICITY_RETURN_CODE=$? && true ; 35 | elif [ ! -z "${1##*[!0-9]*}" ] && [ ! -z "${2##*[!0-9]*}" ]; then 36 | commandJob $@ 37 | elif [ -n "${JOB_COUNT}" ]; then 38 | commandLoop $@ 39 | fi 40 | } 41 | 42 | commandExecution $@ 43 | exit $DUPLICITY_RETURN_CODE 44 | -------------------------------------------------------------------------------- /scripts/verify: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | [[ ${DEBUG} == true ]] && set -x 6 | 7 | source /opt/volumerize/base.sh 8 | 9 | DUPLICITY_RETURN_CODE=0 10 | 11 | function commandLoop() { 12 | local jobcount=$JOB_COUNT 13 | local counter; 14 | 15 | for (( counter=1; counter<=$jobcount; counter++ )) 16 | do 17 | commandJob "$counter" $@ 18 | done 19 | } 20 | 21 | function commandJob() { 22 | local jobNumber=$1 23 | local returnCode=0; 24 | prepareJob "$jobNumber" 25 | shift; 26 | ${DUPLICITY_JOB_COMMAND} verify --compare-data $@ ${DUPLICITY_JOB_OPTIONS} ${VOLUMERIZE_JOB_INCUDES} ${VOLUMERIZE_JOB_TARGET} ${VOLUMERIZE_JOB_SOURCE} || returnCode=$? && true ; 27 | if [ "$returnCode" -gt "$DUPLICITY_RETURN_CODE" ]; then 28 | DUPLICITY_RETURN_CODE=$returnCode 29 | fi 30 | } 31 | 32 | function commandExecution() { 33 | if [ -n "${VOLUMERIZE_SOURCE}" ]; then 34 | ${DUPLICITY_COMMAND} verify --compare-data $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET} ${VOLUMERIZE_SOURCE} || DUPLICITY_RETURN_CODE=$? && true ; 35 | elif [ ! -z "${1##*[!0-9]*}" ]; then 36 | commandJob $@ 37 | elif [ -n "${JOB_COUNT}" ]; then 38 | commandLoop $@ 39 | fi 40 | } 41 | 42 | ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction verify 43 | commandExecution $@ 44 | ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction verify 45 | exit $DUPLICITY_RETURN_CODE 46 | -------------------------------------------------------------------------------- /scripts/backup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | [[ ${DEBUG} == true ]] && set -x 6 | 7 | source /opt/volumerize/base.sh 8 | 9 | DUPLICITY_RETURN_CODE=0 10 | 11 | function commandLoop() { 12 | local jobcount=$JOB_COUNT 13 | local counter; 14 | 15 | for (( counter=1; counter<=$jobcount; counter++ )) 16 | do 17 | commandJob "$counter" $@ 18 | done 19 | } 20 | 21 | function commandJob() { 22 | local jobNumber=$1 23 | local returnCode=0; 24 | prepareJob "$jobNumber" 25 | shift; 26 | ${DUPLICITY_JOB_COMMAND} $@ ${DUPLICITY_JOB_OPTIONS} ${VOLUMERIZE_JOB_INCUDES} ${VOLUMERIZE_JOB_SOURCE} ${VOLUMERIZE_JOB_TARGET} || returnCode=$? && true ; 27 | if [ "$returnCode" -gt "$DUPLICITY_RETURN_CODE" ]; then 28 | DUPLICITY_RETURN_CODE=$returnCode 29 | fi 30 | } 31 | 32 | function commandExecution() { 33 | if [ -n "${VOLUMERIZE_SOURCE}" ]; then 34 | ${DUPLICITY_COMMAND} $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} || DUPLICITY_RETURN_CODE=$? && true ; 35 | elif [ ! -z "${1##*[!0-9]*}" ]; then 36 | commandJob $@ 37 | elif [ -n "${JOB_COUNT}" ]; then 38 | commandLoop $@ 39 | fi 40 | } 41 | 42 | ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction backup 43 | ${VOLUMERIZE_SCRIPT_DIR}/stopContainers.sh 44 | commandExecution $@ 45 | ${VOLUMERIZE_SCRIPT_DIR}/startContainers.sh 46 | export DUPLICITY_RETURN_CODE 47 | ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction backup 48 | exit $DUPLICITY_RETURN_CODE 49 | -------------------------------------------------------------------------------- /scripts/periodicBackup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | [[ ${DEBUG} == true ]] && set -x 6 | 7 | source /opt/volumerize/base.sh 8 | 9 | DUPLICITY_RETURN_CODE=0 10 | 11 | function commandLoop() { 12 | local jobcount=$JOB_COUNT 13 | local counter; 14 | 15 | for (( counter=1; counter<=$jobcount; counter++ )) 16 | do 17 | commandJob "$counter" $@ 18 | done 19 | } 20 | 21 | function commandJob() { 22 | local jobNumber=$1 23 | local returnCode=0; 24 | prepareJob "$jobNumber" 25 | shift; 26 | ${DUPLICITY_JOB_COMMAND} $@ ${DUPLICITY_JOB_OPTIONS} ${VOLUMERIZE_JOB_INCUDES} ${VOLUMERIZE_JOB_SOURCE} ${VOLUMERIZE_JOB_TARGET} || returnCode=$? && true ; 27 | if [ "$returnCode" -gt "$DUPLICITY_RETURN_CODE" ]; then 28 | DUPLICITY_RETURN_CODE=$returnCode 29 | fi 30 | } 31 | 32 | function commandExecution() { 33 | if [ -n "${VOLUMERIZE_SOURCE}" ]; then 34 | ${DUPLICITY_COMMAND} $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} || DUPLICITY_RETURN_CODE=$? && true ; 35 | elif [ ! -z "${1##*[!0-9]*}" ]; then 36 | commandJob $@ 37 | elif [ -n "${JOB_COUNT}" ]; then 38 | commandLoop $@ 39 | fi 40 | } 41 | 42 | ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction backup 43 | source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers.sh 44 | commandExecution $@ 45 | source ${VOLUMERIZE_SCRIPT_DIR}/startContainers.sh 46 | export DUPLICITY_RETURN_CODE 47 | ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction backup 48 | exit $DUPLICITY_RETURN_CODE 49 | -------------------------------------------------------------------------------- /scripts/backupFull: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | [[ ${DEBUG} == true ]] && set -x 6 | 7 | source /opt/volumerize/base.sh 8 | 9 | DUPLICITY_RETURN_CODE=0 10 | 11 | function commandLoop() { 12 | local jobcount=$JOB_COUNT 13 | local counter; 14 | 15 | for (( counter=1; counter<=$jobcount; counter++ )) 16 | do 17 | commandJob "$counter" $@ 18 | done 19 | } 20 | 21 | function commandJob() { 22 | local jobNumber=$1 23 | local returnCode=0; 24 | prepareJob "$jobNumber" 25 | shift; 26 | ${DUPLICITY_JOB_COMMAND} full $@ ${DUPLICITY_JOB_OPTIONS} ${VOLUMERIZE_JOB_INCUDES} ${VOLUMERIZE_JOB_SOURCE} ${VOLUMERIZE_JOB_TARGET} || returnCode=$? && true ; 27 | if [ "$returnCode" -gt "$DUPLICITY_RETURN_CODE" ]; then 28 | DUPLICITY_RETURN_CODE=$returnCode 29 | fi 30 | } 31 | 32 | function commandExecution() { 33 | if [ -n "${VOLUMERIZE_SOURCE}" ]; then 34 | ${DUPLICITY_COMMAND} full $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} || DUPLICITY_RETURN_CODE=$? && true ; 35 | elif [ ! -z "${1##*[!0-9]*}" ]; then 36 | commandJob $@ 37 | elif [ -n "${JOB_COUNT}" ]; then 38 | commandLoop $@ 39 | fi 40 | } 41 | 42 | ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction backup 43 | source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers.sh 44 | commandExecution $@ 45 | source ${VOLUMERIZE_SCRIPT_DIR}/startContainers.sh 46 | export DUPLICITY_RETURN_CODE 47 | ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction backup 48 | exit $DUPLICITY_RETURN_CODE 49 | -------------------------------------------------------------------------------- /scripts/backupIncremental: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | [[ ${DEBUG} == true ]] && set -x 6 | 7 | source /opt/volumerize/base.sh 8 | 9 | DUPLICITY_RETURN_CODE=0 10 | 11 | function commandLoop() { 12 | local jobcount=$JOB_COUNT 13 | local counter; 14 | 15 | for (( counter=1; counter<=$jobcount; counter++ )) 16 | do 17 | commandJob "$counter" $@ 18 | done 19 | } 20 | 21 | function commandJob() { 22 | local jobNumber=$1 23 | local returnCode=0; 24 | prepareJob "$jobNumber" 25 | shift; 26 | ${DUPLICITY_JOB_COMMAND} incremental $@ ${DUPLICITY_JOB_OPTIONS} ${VOLUMERIZE_JOB_INCUDES} ${VOLUMERIZE_JOB_SOURCE} ${VOLUMERIZE_JOB_TARGET} || returnCode=$? && true ; 27 | if [ "$returnCode" -gt "$DUPLICITY_RETURN_CODE" ]; then 28 | DUPLICITY_RETURN_CODE=$returnCode 29 | fi 30 | } 31 | 32 | function commandExecution() { 33 | if [ -n "${VOLUMERIZE_SOURCE}" ]; then 34 | ${DUPLICITY_COMMAND} incremental $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} || DUPLICITY_RETURN_CODE=$? && true ; 35 | elif [ ! -z "${1##*[!0-9]*}" ]; then 36 | commandJob $@ 37 | elif [ -n "${JOB_COUNT}" ]; then 38 | commandLoop $@ 39 | fi 40 | } 41 | 42 | ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction backup 43 | source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers.sh 44 | commandExecution $@ 45 | source ${VOLUMERIZE_SCRIPT_DIR}/startContainers.sh 46 | export DUPLICITY_RETURN_CODE 47 | ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction backup 48 | exit $DUPLICITY_RETURN_CODE 49 | -------------------------------------------------------------------------------- /scripts/restore: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | [[ ${DEBUG} == true ]] && set -x 6 | 7 | source /opt/volumerize/base.sh 8 | 9 | DUPLICITY_RETURN_CODE=0 10 | 11 | function commandLoop() { 12 | local jobcount=$JOB_COUNT 13 | local counter; 14 | 15 | for (( counter=1; counter<=$jobcount; counter++ )) 16 | do 17 | commandJob "$counter" $@ 18 | done 19 | } 20 | 21 | function commandJob() { 22 | local jobNumber=$1 23 | local returnCode=0; 24 | prepareJob "$jobNumber" 25 | shift; 26 | ${DUPLICITY_JOB_COMMAND} restore --force $@ ${DUPLICITY_JOB_OPTIONS} ${VOLUMERIZE_JOB_INCUDES} ${VOLUMERIZE_JOB_TARGET} ${VOLUMERIZE_JOB_SOURCE} || returnCode=$? && true ; 27 | if [ "$returnCode" -gt "$DUPLICITY_RETURN_CODE" ]; then 28 | DUPLICITY_RETURN_CODE=$returnCode 29 | fi 30 | } 31 | 32 | function commandExecution() { 33 | if [ -n "${VOLUMERIZE_SOURCE}" ]; then 34 | ${DUPLICITY_COMMAND} restore --force $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET} ${VOLUMERIZE_SOURCE} || DUPLICITY_RETURN_CODE=$? && true ; 35 | elif [ ! -z "${1##*[!0-9]*}" ]; then 36 | commandJob $@ 37 | elif [ -n "${JOB_COUNT}" ]; then 38 | commandLoop $@ 39 | fi 40 | } 41 | 42 | ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction restore 43 | source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers.sh 44 | commandExecution $@ 45 | source ${VOLUMERIZE_SCRIPT_DIR}/startContainers.sh 46 | export DUPLICITY_RETURN_CODE 47 | ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction restore 48 | exit $DUPLICITY_RETURN_CODE 49 | -------------------------------------------------------------------------------- /imagescripts/docker-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | [[ ${DEBUG} == true ]] && set -x 6 | 7 | function pipeEnvironmentVariables() { 8 | local environmentfile="/etc/profile.d/jobber.sh" 9 | cat > ${environmentfile} <<_EOF_ 10 | #!/bin/sh 11 | _EOF_ 12 | sh -c export >> ${environmentfile} 13 | } 14 | 15 | GPG_KEY_ID="" 16 | 17 | # Install GPG Key 18 | if [ ! -f "/root/.gnupg/pubring.kbx" ]; then 19 | if [ -n "${VOLUMERIZE_GPG_PRIVATE_KEY}" ]; then 20 | gpg --allow-secret-key-import --import ${VOLUMERIZE_GPG_PRIVATE_KEY} 21 | GPG_KEY_ID=$(gpg2 --list-secret-keys --keyid-format LONG | grep sec | awk 'NR==1{print $2; exit}') 22 | GPG_KEY_ID=$(cut -d "/" -f 2 <<< "$GPG_KEY_ID") 23 | fi 24 | 25 | if [ -n "${VOLUMERIZE_GPG_PUBLIC_KEY}" ]; then 26 | gpg --import ${VOLUMERIZE_GPG_PUBLIC_KEY} 27 | GPG_KEY_ID=$(gpg2 --list-keys --keyid-format LONG | grep pub | awk 'NR==2{print $2; exit}') 28 | GPG_KEY_ID=$(cut -d "/" -f 2 <<< "$GPG_KEY_ID") 29 | fi 30 | fi 31 | 32 | # Setting environment variables 33 | readonly CUR_DIR=$(cd $(dirname ${BASH_SOURCE:-$0}); pwd) 34 | 35 | if [ -n "${VOLUMERIZE_DELAYED_START}" ]; then 36 | sleep ${VOLUMERIZE_DELAYED_START} 37 | fi 38 | 39 | if [ -n "${GOOGLE_DRIVE_ID}" ]; then 40 | source $CUR_DIR/create_gdocs_file.sh 41 | fi 42 | 43 | if [ -n "${VOLUMERIZE_JOBBER_TIME}" ]; then 44 | source $CUR_DIR/create_jobber.sh 45 | fi 46 | 47 | if [ "$1" = 'volumerize' ]; then 48 | pipeEnvironmentVariables 49 | exec /usr/libexec/jobbermaster 50 | else 51 | exec "$@" 52 | fi 53 | -------------------------------------------------------------------------------- /prepost_strategies/mysql/README.md: -------------------------------------------------------------------------------- 1 | # Using a prepost strategy to create mySQL backups 2 | 3 | Volumerize can execute scripts before and after the backup process. 4 | 5 | With this prepost strategy you can create a .sql backup of your MySQL containers and save it with Volumerize. 6 | 7 | ## Environment Variables 8 | 9 | Aside of the required environment variables by Volumerize, this prepost strategy will require a couple of extra variables. 10 | 11 | | Name | Description | 12 | | -------------- | ---------------------------------------------------------- | 13 | | MYSQL_USERNAME | Username of the user who will perform the restore or dump. | 14 | | MYSQL_PASSWORD | Password of the user who will perform the restore or dump. | 15 | | MYSQL_HOST | IP or domain of the host machine. | 16 | | MYSQL_DATABASE | Database to backup / restore. | 17 | 18 | ## Example with Docker Compose 19 | 20 | ```YAML 21 | version: "3" 22 | 23 | services: 24 | mariadb: 25 | image: mariadb 26 | ports: 27 | - 3306:3306 28 | environment: 29 | - MYSQL_ROOT_PASSWORD=1234 30 | - MYSQL_DATABASE=somedatabase 31 | volumes: 32 | - mariadb:/var/lib/mysql 33 | 34 | volumerize: 35 | build: . 36 | environment: 37 | - VOLUMERIZE_SOURCE=/source 38 | - VOLUMERIZE_TARGET=file:///backup 39 | - MYSQL_USERNAME=root 40 | - MYSQL_PASSWORD=1234 41 | - MYSQL_HOST=mariadb 42 | - MYSQL_DATABASE=somedatabase 43 | volumes: 44 | - volumerize-cache:/volumerize-cache 45 | - backup:/backup 46 | depends_on: 47 | - mariadb 48 | 49 | volumes: 50 | volumerize-cache: 51 | mariadb: 52 | backup: 53 | ``` 54 | 55 | Then execute `docker-compose exec volumerize backup` to create a backup of your database and `docker-compose exec volumerize restore` to restore it from your backup. -------------------------------------------------------------------------------- /imagescripts/create_jobber.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | readonly JOBBER_SCRIPT_DIR=$VOLUMERIZE_HOME 6 | 7 | source $CUR_DIR/base.sh 8 | 9 | JOBBER_CRON_SCHEDULE='0 0 4 * * *' 10 | 11 | if [ -n "${VOLUMERIZE_JOBBER_TIME}" ]; then 12 | JOBBER_CRON_SCHEDULE=${VOLUMERIZE_JOBBER_TIME} 13 | fi 14 | 15 | JOB_NAME1=VolumerizeBackupJob 16 | JOB_COMMAND1=${JOBBER_SCRIPT_DIR}/periodicBackup 17 | JOB_TIME1=$JOBBER_CRON_SCHEDULE 18 | JOB_ON_ERROR1=Continue 19 | JOB_NOTIFY_ERR1=false 20 | JOB_NOTIFY_FAIL1=false 21 | 22 | readonly configfile="/root/.jobber" 23 | 24 | function pipeEnvironmentVariables() { 25 | local environmentfile="/etc/profile.d/jobber.sh" 26 | cat > ${environmentfile} <<_EOF_ 27 | #!/bin/sh 28 | _EOF_ 29 | sh -c export >> ${environmentfile} 30 | sed -i.bak '/^export [a-zA-Z0-9_]*:/d' ${environmentfile} 31 | } 32 | 33 | if [ ! -f "${configfile}" ]; then 34 | touch ${configfile} 35 | 36 | cat >> ${configfile} <<_EOF_ 37 | --- 38 | 39 | _EOF_ 40 | for (( i = 1; ; i++ )) 41 | do 42 | VAR_JOB_ON_ERROR="JOB_ON_ERROR$i" 43 | VAR_JOB_NAME="JOB_NAME$i" 44 | VAR_JOB_COMMAND="JOB_COMMAND$i" 45 | VAR_JOB_TIME="JOB_TIME$i" 46 | VAR_JOB_NOTIFY_ERR="JOB_NOTIFY_ERR$i" 47 | VAR_JOB_NOTIFY_FAIL="JOB_NOTIFY_FAIL$i" 48 | 49 | if [ ! -n "${!VAR_JOB_NAME}" ]; then 50 | break 51 | fi 52 | 53 | it_job_on_error=${!VAR_JOB_ON_ERROR:-"Continue"} 54 | it_job_name=${!VAR_JOB_NAME} 55 | it_job_time=${!VAR_JOB_TIME} 56 | it_job_command=${!VAR_JOB_COMMAND} 57 | it_job_notify_error=${!VAR_JOB_NOTIFY_ERR:-"false"} 58 | it_job_notify_failure=${!VAR_JOB_NOTIFY_FAIL:-"false"} 59 | 60 | cat >> ${configfile} <<_EOF_ 61 | - name: ${it_job_name} 62 | cmd: ${it_job_command} 63 | time: '${it_job_time}' 64 | onError: ${it_job_on_error} 65 | notifyOnError: ${it_job_notify_error} 66 | notifyOnFailure: ${it_job_notify_failure} 67 | 68 | _EOF_ 69 | done 70 | fi 71 | 72 | cat ${configfile} 73 | -------------------------------------------------------------------------------- /prepost_strategies/mongodb/README.md: -------------------------------------------------------------------------------- 1 | # Using a prepost strategy to create MongoDB backups 2 | 3 | Volumerize can execute scripts before and after the backup process. 4 | 5 | With this prepost strategy you can create dump of your MongoDB containers and save it with Volumerize. 6 | 7 | ## Environment Variables 8 | 9 | Aside of the required environment variables by Volumerize, this prepost strategy will require a couple of extra variables. 10 | MONGO_USERNAME MONGO_PASSWORD MONGO_HOST MONGO_PORT 11 | | Name | Description | 12 | | -------------- | ---------------------------------------------------------- | 13 | | MONGO_USERNAME | Username of the user who will perform the restore or dump. | 14 | | MONGO_PASSWORD | Password of the user who will perform the restore or dump. | 15 | | MONGO_HOST | MongoDB IP or domain. | 16 | | MONGO_PORT | MongoDB port. | 17 | 18 | ## Example with Docker Compose 19 | 20 | ```YAML 21 | version: "3" 22 | 23 | services: 24 | mongodb: 25 | image: mongo 26 | ports: 27 | - 27017:27017 28 | environment: 29 | - MONGO_INITDB_ROOT_USERNAME=root 30 | - MONGO_INITDB_ROOT_PASSWORD=1234 31 | volumes: 32 | - mongodb:/data/db 33 | 34 | volumerize: 35 | build: ./prepost_strategies/mongodb/ 36 | environment: 37 | - VOLUMERIZE_SOURCE=/source 38 | - VOLUMERIZE_TARGET=file:///backup 39 | - MONGO_USERNAME=root 40 | - MONGO_PASSWORD=1234 41 | - MONGO_PORT=27017 42 | - MONGO_HOST=mongodb 43 | volumes: 44 | - volumerize-cache:/volumerize-cache 45 | - backup:/backup 46 | depends_on: 47 | - mongodb 48 | 49 | volumes: 50 | volumerize-cache: 51 | mongodb: 52 | backup: 53 | ``` 54 | 55 | Then execute `docker-compose exec volumerize backup` to create a backup of your database and `docker-compose exec volumerize restore` to restore it from your backup. -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build_branch: 4 | working_directory: /app 5 | docker: 6 | - image: docker:17.05.0-ce-git 7 | steps: 8 | - checkout 9 | - setup_remote_docker 10 | - run: 11 | name: Install dependencies 12 | command: | 13 | apk add --no-cache \ 14 | py-pip \ 15 | curl \ 16 | curl-dev 17 | - run: 18 | name: Build Branch Images 19 | command: | 20 | set +o pipefail 21 | sh buildscripts/buildSupportedAlpineImages.sh 22 | - run: 23 | name: Test Branch Images 24 | command: | 25 | sh buildscripts/testSupportedAlpineImages.sh 26 | 27 | build_releases: 28 | working_directory: /app 29 | docker: 30 | - image: docker:17.05.0-ce-git 31 | steps: 32 | - checkout 33 | - setup_remote_docker 34 | - run: 35 | name: Install dependencies 36 | command: | 37 | apk add --no-cache \ 38 | py-pip \ 39 | curl \ 40 | curl-dev 41 | - run: 42 | name: Build Docker Release Images 43 | command: | 44 | set +o pipefail 45 | sh buildscripts/buildSupportedAlpineImages.sh 46 | - run: 47 | name: Test Docker Release Images 48 | command: | 49 | sh buildscripts/testSupportedAlpineImages.sh 50 | - run: 51 | name: Push Dockerhub Release Images 52 | command: | 53 | docker login -u ${DOCKER_USER} -p ${DOCKER_PASS} 54 | sh buildscripts/releaseSupportedAlpineImages.sh 55 | 56 | workflows: 57 | version: 2 58 | build_pipeline: 59 | jobs: 60 | - build_releases: 61 | filters: 62 | branches: 63 | only: master 64 | - build_branch: 65 | filters: 66 | branches: 67 | ignore: 68 | - master 69 | nightly: 70 | triggers: 71 | - schedule: 72 | cron: "0 1 * * 1" 73 | filters: 74 | branches: 75 | only: 76 | - master 77 | jobs: 78 | - build_releases 79 | -------------------------------------------------------------------------------- /backends/Mega/README.md: -------------------------------------------------------------------------------- 1 | # Using Volumerize With Mega.nz 2 | 3 | Volumerize can backup Docker volumes on Mega.nz. 4 | 5 | You have to perform the following steps: 6 | 7 | Pass your Mega.nz credentials to Volumerize and use the appropriate Mega.nz URL scheme. 8 | 9 | First we start our example container with some data to backup: 10 | 11 | ~~~~ 12 | $ docker run \ 13 | -d -p 80:8080 \ 14 | --name jenkins \ 15 | -v jenkins_volume:/jenkins \ 16 | blacklabelops/jenkins 17 | ~~~~ 18 | 19 | > Starts Jenkins and stores its data inside the Docker volume `jenkins_volume`. 20 | 21 | Start the container with the appriopriate scheme in `VOLUMERIZE_TARGET` and your credentials. 22 | 23 | The scheme can be `mega://`, `megav2://` or `megav3://` depending on your account's creation date. According to [duplicity's documentation](http://duplicity.nongnu.org/vers8/duplicity.1.html#sect7) account created prior to November 2018 must use `mega://`. 24 | 25 | Pass your credentials in the `VOLUMERIZE_TARGET` with the syntax `mega://user:password@mega.nz/some_dir`. Adapt it to your scheme. 26 | You can also pass your credentials through a configuration file with the syntax: 27 | ~~~~ini 28 | [Login] 29 | Username = user 30 | Password = password 31 | ~~~~ 32 | The configuration file must be in `/root/` and it's name must reflect the scheme. `.megarc` for v1, `.megav2rc` for v2 and `.megav3rc` for v3. 33 | 34 | Assuming our local configuration file is named _megarc_ and our account's scheme is v3. 35 | ~~~~ 36 | $ docker run -it --rm \ 37 | -v megarc:/root/.megav3rc \ 38 | -v jenkins_volume:/source:ro \ 39 | -e "VOLUMERIZE_SOURCE=/source" \ 40 | -e "VOLUMERIZE_TARGET=megav3://mega.nz/backup" \ 41 | blacklabelops/volumerize backup 42 | ~~~~ 43 | 44 | # Restore from Mega.nz 45 | 46 | Restore is easy, just pass the same environment variables and start the restore script: 47 | 48 | > Note: Remove the read-only option `:ro` on the source volume. 49 | 50 | ~~~~ 51 | $ docker run --rm \ 52 | -v jenkins_test_restore:/source \ 53 | -v megarc:/root/.megav3rc \ 54 | -e "VOLUMERIZE_SOURCE=/source" \ 55 | -e "VOLUMERIZE_TARGET=megav3://mega.nz/backup" \ 56 | blacklabelops/volumerize restore 57 | ~~~~ 58 | 59 | > Will perform a test restore inside a separate volume `jenkins_test_restore` 60 | 61 | Check the contents of the volume: 62 | 63 | ~~~~ 64 | $ docker run --rm \ 65 | -v jenkins_test_restore:/source \ 66 | blacklabelops/alpine ls -R /source 67 | ~~~~ 68 | 69 | > Lists files inside the source volume 70 | 71 | Verify against the Mega.nz content: 72 | 73 | ~~~~ 74 | $ docker run --rm \ 75 | -v jenkins_test_restore:/source \ 76 | -v megarc:/root/.megav3rc \ 77 | -e "VOLUMERIZE_SOURCE=/source" \ 78 | -e "VOLUMERIZE_TARGET=megav3://mega.nz/backup" \ 79 | blacklabelops/volumerize verify 80 | ~~~~ 81 | 82 | > Will perform a single verification of the volume contents against the Mega.nz archive. 83 | 84 | # Start and Stop Docker Containers 85 | 86 | Volumerize can stop containers before backup and start them after backup. 87 | 88 | First start a test container with the name `jenkins` 89 | 90 | ~~~~ 91 | $ docker run \ 92 | -d -p 80:8080 \ 93 | --name jenkins \ 94 | -v jenkins_volume:/jenkins \ 95 | blacklabelops/jenkins 96 | ~~~~ 97 | 98 | > Starts Jenkins and stores its data inside the Docker volume `jenkins_volume`. 99 | 100 | Now add the containers name inside the environment variable `VOLUMERIZE_CONTAINERS` and start Volumerize in demon mode: 101 | 102 | ~~~~ 103 | $ docker run -d \ 104 | --name volumerize \ 105 | -v jenkins_volume:/source:ro \ 106 | -v volumerize_cache:/volumerize-cache \ 107 | -v megarc:/root/.megav3rc \ 108 | -e "VOLUMERIZE_SOURCE=/source" \ 109 | -e "VOLUMERIZE_TARGET=megav3://mega.nz/backup" \ 110 | -e "VOLUMERIZE_CONTAINERS=jenkins" \ 111 | -v /var/run/docker.sock:/var/run/docker.sock \ 112 | blacklabelops/volumerize 113 | ~~~~ 114 | 115 | > Needs access to the docker host over the directive `-v /var/run/docker.sock:/var/run/docker.sock` 116 | 117 | You can test the backup routine: 118 | 119 | ~~~~ 120 | $ docker exec volumerize backup 121 | ~~~~ 122 | 123 | > Triggers the backup inside the volume, the name `jenkins` should appear on the console. 124 | 125 | > Note: Make sure your container is not running with docker auto restart! 126 | -------------------------------------------------------------------------------- /backends/BackblazeB2/README.md: -------------------------------------------------------------------------------- 1 | # Using Volumerize With Backblaze B2 2 | 3 | Volumerize can backup Docker volumes on Backblaze B2. 4 | 5 | You have to perform the following steps: 6 | 7 | Login to Backblaze B2 and create an application key. 8 | 9 | The Backblaze B2 Buckets (select `Show Account ID and Application Key`): [B2 Cloud Storage Buckets](https://secure.backblaze.com/b2_buckets.htm) 10 | 11 | Remember the following details: 12 | 13 | * The application key ID. 14 | * The application key. 15 | * The bucket name. 16 | 17 | First we start our example container with some data to backup: 18 | 19 | ~~~~ 20 | $ docker run \ 21 | -d -p 80:8080 \ 22 | --name jenkins \ 23 | -v jenkins_volume:/jenkins \ 24 | blacklabelops/jenkins 25 | ~~~~ 26 | 27 | > Starts Jenkins and stores its data inside the Docker volume `jenkins_volume`. 28 | 29 | Now build your B2 url according to the template: 30 | 31 | `b2://account_id[:application_key]@bucket_name/[folder/]`. 32 | 33 | Start the container in demon mode with your url: 34 | 35 | ~~~~ 36 | $ docker run -d \ 37 | --name volumerize \ 38 | -v volumerize_cache:/volumerize-cache \ 39 | -v jenkins_volume:/source:ro \ 40 | -e "VOLUMERIZE_SOURCE=/source" \ 41 | -e "VOLUMERIZE_TARGET=b2://003945h951emn6p0000000001:H006yG2V7+pogtolEFwCexcZUWJdkNc@volumerizetest/" \ 42 | blacklabelops/volumerize 43 | ~~~~ 44 | 45 | > Will backup to application id 003945h951emn6p0000000001 using application key H006yG2V7+pogtolEFwCexcZUWJdkNc inside bucket volumerizetest. 46 | 47 | Start the first backup: 48 | 49 | ~~~~ 50 | $ docker exec volumerize backup 51 | ~~~~ 52 | 53 | # Restore from Backblaze B2 54 | 55 | Restore is easy, just pass the same environment variables and start the restore script: 56 | 57 | > Note: Remove the read-only option `:ro` on the source volume. 58 | 59 | ~~~~ 60 | $ docker run --rm \ 61 | -v jenkins_test_restore:/source \ 62 | -v volumerize_cache:/volumerize-cache \ 63 | -e "VOLUMERIZE_SOURCE=/source" \ 64 | -e "VOLUMERIZE_TARGET=b2://003945h951emn6p0000000001:H006yG2V7+pogtolEFwCexcZUWJdkNc@volumerizetest/" \ 65 | blacklabelops/volumerize restore 66 | ~~~~ 67 | 68 | > Will perform a test restore inside a separate volume `jenkins_test_restore` 69 | 70 | Check the contents of the volume: 71 | 72 | ~~~~ 73 | $ docker run --rm \ 74 | -v jenkins_test_restore:/source \ 75 | blacklabelops/alpine ls -R /source 76 | ~~~~ 77 | 78 | > Lists files inside the source volume 79 | 80 | Verify against the Backblaze B2 content: 81 | 82 | ~~~~ 83 | $ docker run --rm \ 84 | -v jenkins_test_restore:/source \ 85 | -v volumerize_cache:/volumerize-cache \ 86 | -e "VOLUMERIZE_SOURCE=/source" \ 87 | -e "VOLUMERIZE_TARGET=b2://003945h951emn6p0000000001:H006yG2V7+pogtolEFwCexcZUWJdkNc@volumerizetest/" \ 88 | blacklabelops/volumerize verify 89 | ~~~~ 90 | 91 | > Will perform a single verification of the volume contents against the Backblaze B2 archive. 92 | 93 | # Start and Stop Docker Containers 94 | 95 | Volumerize can stop containers before backup and start them after backup. 96 | 97 | First start a test container with the name `jenkins` 98 | 99 | ~~~~ 100 | $ docker run \ 101 | -d -p 80:8080 \ 102 | --name jenkins \ 103 | -v jenkins_volume:/jenkins \ 104 | blacklabelops/jenkins 105 | ~~~~ 106 | 107 | > Starts Jenkins and stores its data inside the Docker volume `jenkins_volume`. 108 | 109 | Now add the containers name inside the environment variable `VOLUMERIZE_CONTAINERS` and start Volumerize in demon mode: 110 | 111 | ~~~~ 112 | $ docker run -d \ 113 | --name volumerize \ 114 | -v jenkins_volume:/source:ro \ 115 | -v volumerize_cache:/volumerize-cache \ 116 | -e "VOLUMERIZE_SOURCE=/source" \ 117 | -e "VOLUMERIZE_TARGET=b2://003945h951emn6p0000000001:H006yG2V7+pogtolEFwCexcZUWJdkNc@volumerizetest/" \ 118 | -e "VOLUMERIZE_CONTAINERS=jenkins" \ 119 | -v /var/run/docker.sock:/var/run/docker.sock \ 120 | blacklabelops/volumerize 121 | ~~~~ 122 | 123 | > Needs access to the docker host over the directive `-v /var/run/docker.sock:/var/run/docker.sock` 124 | 125 | You can test the backup routine: 126 | 127 | ~~~~ 128 | $ docker exec volumerize backup 129 | ~~~~ 130 | 131 | > Triggers the backup inside the volume, the name `jenkins` should appear on the console. 132 | 133 | > Note: Make sure your container is not running with docker auto restart! 134 | -------------------------------------------------------------------------------- /backends/Dropbox/README.md: -------------------------------------------------------------------------------- 1 | # Using Volumerize With Dropbox 2 | 3 | Volumerize can backup Docker volumes on Dropbox. 4 | 5 | You have to perform the following steps: 6 | 7 | Login to you Dropbox account and create an app key. 8 | 9 | Read on how to create the app key on Dropbox: [Note on Dropbox Access](http://duplicity.nongnu.org/duplicity.1.html#toc12) 10 | 11 | The Dropbox App Creation Page: [Dropbox App Creation](https://www.dropbox.com/developers/apps/create) 12 | 13 | On the app page you need to generate the `Generated access token` for the environment variable `DPBX_ACCESS_TOKEN`. 14 | 15 | First we start our example container with some data to backup: 16 | 17 | ~~~~ 18 | $ docker run \ 19 | -d -p 80:8080 \ 20 | --name jenkins \ 21 | -v jenkins_volume:/jenkins \ 22 | blacklabelops/jenkins 23 | ~~~~ 24 | 25 | > Starts Jenkins and stores its data inside the Docker volume `jenkins_volume`. 26 | 27 | Start the container in demon mode and pass the access token through the environment variable `DPBX_ACCESS_TOKEN`. 28 | 29 | Setup Volumerize to use Dropbox for backups of the volume `jenkins_volume`. Make sure you have already created the backup folder inside Dropbox, e.g. here `/Apps/Volumerize`. 30 | 31 | Start the container in demon mode: 32 | 33 | ~~~~ 34 | $ docker run -d \ 35 | --name volumerize \ 36 | -v volumerize_cache:/volumerize-cache \ 37 | -v jenkins_volume:/source:ro \ 38 | -e "VOLUMERIZE_SOURCE=/source" \ 39 | -e "VOLUMERIZE_TARGET=dpbx:///Apps/Volumerize" \ 40 | -e "DPBX_ACCESS_TOKEN=JUtoLXXwNNMAAAAAAA" \ 41 | blacklabelops/volumerize 42 | ~~~~ 43 | 44 | > `volumerize_cache` is the local data cache. 45 | 46 | You can start an initial backup: 47 | 48 | ~~~~ 49 | $ docker exec volumerize backup 50 | ~~~~ 51 | 52 | # Restore from Dropbox 53 | 54 | Restore is easy, just pass the same environment variables and start the restore script: 55 | 56 | > Note: Remove the read-only option `:ro` on the source volume. 57 | 58 | ~~~~ 59 | $ docker run --rm \ 60 | -v jenkins_test_restore:/source \ 61 | -e "VOLUMERIZE_SOURCE=/source" \ 62 | -e "VOLUMERIZE_TARGET=dpbx:///Apps/Volumerize" \ 63 | -e "DPBX_ACCESS_TOKEN=JUtoLXXwNNMAAAAAAA" \ 64 | blacklabelops/volumerize restore 65 | ~~~~ 66 | 67 | > Will perform a test restore inside a separate volume `jenkins_test_restore` 68 | 69 | Check the contents of the volume: 70 | 71 | ~~~~ 72 | $ docker run --rm \ 73 | -v jenkins_test_restore:/source \ 74 | blacklabelops/alpine ls -R /source 75 | ~~~~ 76 | 77 | > Lists files inside the source volume 78 | 79 | Verify against the dropbox content: 80 | 81 | ~~~~ 82 | $ docker run --rm \ 83 | -v jenkins_test_restore:/source \ 84 | -e "VOLUMERIZE_SOURCE=/source" \ 85 | -e "VOLUMERIZE_TARGET=dpbx:///Apps/Volumerize" \ 86 | -e "DPBX_ACCESS_TOKEN=JUtoLXXwNNMAAAAAAA" \ 87 | blacklabelops/volumerize verify 88 | ~~~~ 89 | 90 | > Will perform a single verification of the volume contents against the dropbox archive. 91 | 92 | # Start and Stop Docker Containers 93 | 94 | Volumerize can stop containers before backup and start them after backup. 95 | 96 | First start a test container with the name `jenkins` 97 | 98 | ~~~~ 99 | $ docker run \ 100 | -d -p 80:8080 \ 101 | --name jenkins \ 102 | -v jenkins_volume:/jenkins \ 103 | blacklabelops/jenkins 104 | ~~~~ 105 | 106 | > Starts Jenkins and stores its data inside the Docker volume `jenkins_volume`. 107 | 108 | Now add the containers name inside the environment variable `VOLUMERIZE_CONTAINERS` and start Volumerize in demon mode: 109 | 110 | ~~~~ 111 | $ docker run -d \ 112 | --name volumerize \ 113 | -v volumerize_cache:/volumerize-cache \ 114 | -v jenkins_volume:/source:ro \ 115 | -e "VOLUMERIZE_SOURCE=/source" \ 116 | -e "VOLUMERIZE_TARGET=dpbx:///Apps/Volumerize" \ 117 | -e "DPBX_ACCESS_TOKEN=JUtoLXXwNNMAAAAAAA" \ 118 | -e "VOLUMERIZE_CONTAINERS=jenkins" \ 119 | -v /var/run/docker.sock:/var/run/docker.sock \ 120 | blacklabelops/volumerize 121 | ~~~~ 122 | 123 | > Needs access to the docker host over the directive `-v /var/run/docker.sock:/var/run/docker.sock` 124 | 125 | You can test the backup routine: 126 | 127 | ~~~~ 128 | $ docker exec volumerize backup 129 | ~~~~ 130 | 131 | > Triggers the backup inside the volume, the name `jenkins` should appear on the console. 132 | 133 | > Note: Make sure your container is not running with docker auto restart! 134 | -------------------------------------------------------------------------------- /imagescripts/base.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -o errexit 4 | 5 | DUPLICITY_COMMAND="duplicity" 6 | 7 | DUPLICITY_OPTIONS="" 8 | 9 | DUPLICITY_INCLUDES="" 10 | 11 | DUPLICITY_TARGET=${VOLUMERIZE_TARGET} 12 | 13 | DUPLICITY_MODE="" 14 | 15 | function resolveOptions() { 16 | DUPLICITY_OPTIONS="--allow-source-mismatch --archive-dir=${VOLUMERIZE_CACHE}" 17 | if [ -n "${VOLUMERIZE_DUPLICITY_OPTIONS}" ]; then 18 | DUPLICITY_OPTIONS=$DUPLICITY_OPTIONS" "${VOLUMERIZE_DUPLICITY_OPTIONS} 19 | fi 20 | if [ ! -n "${PASSPHRASE}" ] && [ ! -n "${VOLUMERIZE_GPG_PUBLIC_KEY}" ] && [ ! -n "${VOLUMERIZE_GPG_PRIVATE_KEY}" ]; then 21 | DUPLICITY_OPTIONS=$DUPLICITY_OPTIONS" --no-encryption" 22 | fi 23 | if [ -n "${GPG_KEY_ID}" ]; then 24 | DUPLICITY_OPTIONS=$DUPLICITY_OPTIONS" --gpg-options --trust-model=always --encrypt-key ${GPG_KEY_ID}" 25 | fi 26 | if [ -n "${VOLUMERIZE_FULL_IF_OLDER_THAN}" ]; then 27 | DUPLICITY_OPTIONS=$DUPLICITY_OPTIONS" --full-if-older-than ${VOLUMERIZE_FULL_IF_OLDER_THAN}" 28 | fi 29 | if [ "${VOLUMERIZE_ASYNCHRONOUS_UPLOAD}" = 'true' ]; then 30 | DUPLICITY_OPTIONS=$DUPLICITY_OPTIONS" --asynchronous-upload" 31 | fi 32 | } 33 | 34 | function resolveIncludes() { 35 | local x 36 | for (( x=1; ; x++ )) 37 | do 38 | VOLUMERIZE_INCLUDE="VOLUMERIZE_INCLUDE${x}" 39 | if [ ! -n "${!VOLUMERIZE_INCLUDE}" ]; then 40 | break 41 | fi 42 | VOLUMERIZE_INCUDES=$VOLUMERIZE_INCLUDES" --include "${!VOLUMERIZE_INCLUDE} 43 | done 44 | } 45 | 46 | JOB_COUNT= 47 | 48 | function discoverJobs() { 49 | local x 50 | for (( x=1; ; x++ )) 51 | do 52 | JOB_VARIABLE="VOLUMERIZE_SOURCE${x}" 53 | if [ ! -n "${!JOB_VARIABLE}" ]; then 54 | break 55 | else 56 | JOB_COUNT=$x 57 | fi 58 | done 59 | } 60 | 61 | DUPLICITY_JOB_COMMAND= 62 | DUPLICITY_JOB_OPTIONS= 63 | VOLUMERIZE_JOB_SOURCE= 64 | VOLUMERIZE_JOB_TARGET= 65 | VOLUMERIZE_JOB_INCUDES= 66 | 67 | function prepareJobCommand() { 68 | local jobNumber=$1 69 | DUPLICITY_JOB_COMMAND=$DUPLICITY_COMMAND 70 | DUPLICITY_JOB_OPTIONS="--allow-source-mismatch" 71 | local CACHE_VARIABLE="VOLUMERIZE_CACHE${jobNumber}" 72 | if [ -n "${!CACHE_VARIABLE}" ]; then 73 | DUPLICITY_JOB_OPTIONS=$DUPLICITY_JOB_OPTIONS" --archive-dir=${!CACHE_VARIABLE}" 74 | else 75 | DUPLICITY_JOB_OPTIONS=$DUPLICITY_JOB_OPTIONS" --archive-dir=${VOLUMERIZE_CACHE}" 76 | fi 77 | if [ -n "${VOLUMERIZE_DUPLICITY_OPTIONS}" ]; then 78 | DUPLICITY_JOB_OPTIONS=$DUPLICITY_JOB_OPTIONS" "${VOLUMERIZE_DUPLICITY_OPTIONS} 79 | fi 80 | if [ ! -n "${PASSPHRASE}" ] && [ ! -n "${VOLUMERIZE_GPG_PUBLIC_KEY}" ] && [ ! -n "${VOLUMERIZE_GPG_PRIVATE_KEY}" ]; then 81 | DUPLICITY_JOB_OPTIONS=$DUPLICITY_JOB_OPTIONS" --no-encryption" 82 | fi 83 | if [ -n "${GPG_KEY_ID}" ]; then 84 | DUPLICITY_JOB_OPTIONS=$DUPLICITY_JOB_OPTIONS" --gpg-options --trust-model=always --encrypt-key ${GPG_KEY_ID}" 85 | fi 86 | if [ -n "${VOLUMERIZE_FULL_IF_OLDER_THAN}" ]; then 87 | DUPLICITY_JOB_OPTIONS=$DUPLICITY_JOB_OPTIONS" --full-if-older-than ${VOLUMERIZE_FULL_IF_OLDER_THAN}" 88 | fi 89 | if [ "${VOLUMERIZE_ASYNCHRONOUS_UPLOAD}" = 'true' ]; then 90 | DUPLICITY_JOB_OPTIONS=$DUPLICITY_JOB_OPTIONS" --asynchronous-upload" 91 | fi 92 | } 93 | 94 | function prepareJobConfiguration() { 95 | local jobNumber=$1 96 | local VARIABLE_SOURCE="VOLUMERIZE_SOURCE${jobNumber}" 97 | local VARIABLE_TARGET="VOLUMERIZE_TARGET${jobNumber}" 98 | 99 | if [ -n "${!VARIABLE_SOURCE}" ]; then 100 | VOLUMERIZE_JOB_SOURCE=${!VARIABLE_SOURCE} 101 | else 102 | VOLUMERIZE_JOB_SOURCE= 103 | fi 104 | if [ -n "${!VARIABLE_TARGET}" ]; then 105 | VOLUMERIZE_JOB_TARGET=${!VARIABLE_TARGET} 106 | else 107 | VOLUMERIZE_JOB_TARGET= 108 | fi 109 | } 110 | 111 | function resolveJobIncludes() { 112 | local jobNumber=$1 113 | local x 114 | local VARIABLE_INCLUDE 115 | VOLUMERIZE_JOB_INCUDES= 116 | for (( x=1; ; x++ )) 117 | do 118 | VARIABLE_INCLUDE="VOLUMERIZE_INCLUDE${jobNumber}_${x}" 119 | if [ ! -n "${!VARIABLE_INCLUDE}" ]; then 120 | break 121 | fi 122 | VOLUMERIZE_JOB_INCUDES=$VOLUMERIZE_JOB_INCUDES" --include "${!VARIABLE_INCLUDE} 123 | done 124 | } 125 | 126 | function prepareJob() { 127 | local jobNumber=$1 128 | JOB_VARIABLE="VOLUMERIZE_SOURCE${jobNumber}" 129 | if [ -n "${!JOB_VARIABLE}" ]; then 130 | prepareJobCommand $jobNumber 131 | prepareJobConfiguration $jobNumber 132 | resolveJobIncludes $jobNumber 133 | fi 134 | } 135 | 136 | resolveIncludes 137 | resolveOptions 138 | discoverJobs 139 | -------------------------------------------------------------------------------- /backends/GoogleDrive/README.md: -------------------------------------------------------------------------------- 1 | # Using Volumerize With Google Drive 2 | 3 | Volumerize can backup Docker volumes on Google Drive. 4 | 5 | You have to perform the following steps: 6 | 7 | Login to Google developers console and create a service account. 8 | 9 | Read on how to create the app key on Google directive: [Note on PyDrive](http://duplicity.nongnu.org/duplicity.1.html#sect22) 10 | 11 | The Google developers console: [Google Developers Console](https://console.developers.google.com./) 12 | 13 | On the app page you need to generate the `OAuth client ID` and retrieve the `Client ID` and `Client Secret`. 14 | 15 | First we start our example container with some data to backup: 16 | 17 | ~~~~ 18 | $ docker run \ 19 | -d -p 80:8080 \ 20 | --name jenkins \ 21 | -v jenkins_volume:/jenkins \ 22 | blacklabelops/jenkins 23 | ~~~~ 24 | 25 | > Starts Jenkins and stores its data inside the Docker volume `jenkins_volume`. 26 | 27 | Start the container in `Authorization Mode` follow the authorization instructions and store your credentials inside a Docker volume! 28 | 29 | ~~~~ 30 | $ docker run -it --rm \ 31 | -v volumerize_cache:/volumerize-cache \ 32 | -v volumerize_credentials:/credentials \ 33 | -v jenkins_volume:/source:ro \ 34 | -e "VOLUMERIZE_SOURCE=/source" \ 35 | -e "VOLUMERIZE_TARGET=gdocs://youremail@gmail.com/backup" \ 36 | -e "GOOGLE_DRIVE_ID=12312786-e99grj1k5lwjepofjwpoejfpe5nqvkd3e.apps.googleusercontent.com" \ 37 | -e "GOOGLE_DRIVE_SECRET=FWeofWefkefnkef" \ 38 | blacklabelops/volumerize backup 39 | ~~~~ 40 | 41 | > Note: The routine will fail, you still have to enable the Google Drive API for your project. See the URL inside the log output. 42 | 43 | Setup Volumerize to use Google Drive for backups of the volume `jenkins_volume`. 44 | 45 | Start the container in demon mode: 46 | 47 | ~~~~ 48 | $ docker run -d \ 49 | --name volumerize \ 50 | -v volumerize_cache:/volumerize-cache \ 51 | -v volumerize_credentials:/credentials \ 52 | -v jenkins_volume:/source:ro \ 53 | -e "VOLUMERIZE_SOURCE=/source" \ 54 | -e "VOLUMERIZE_TARGET=gdocs://youremail@gmail.com/backup" \ 55 | blacklabelops/volumerize 56 | ~~~~ 57 | 58 | > `volumerize_cache` is the local data cache. 59 | 60 | You can start an initial full backup: 61 | 62 | ~~~~ 63 | $ docker exec volumerize backupFull 64 | ~~~~ 65 | 66 | # Restore from Google Drive 67 | 68 | Restore is easy, just pass the same environment variables and start the restore script: 69 | 70 | > Note: Remove the read-only option `:ro` on the source volume. 71 | 72 | ~~~~ 73 | $ docker run --rm \ 74 | -v jenkins_test_restore:/source \ 75 | -v volumerize_credentials:/credentials \ 76 | -e "VOLUMERIZE_SOURCE=/source" \ 77 | -e "VOLUMERIZE_TARGET=gdocs://youremail@gmail.com/backup" \ 78 | blacklabelops/volumerize restore 79 | ~~~~ 80 | 81 | > Will perform a test restore inside a separate volume `jenkins_test_restore` 82 | 83 | Check the contents of the volume: 84 | 85 | ~~~~ 86 | $ docker run --rm \ 87 | -v jenkins_test_restore:/source \ 88 | blacklabelops/alpine ls -R /source 89 | ~~~~ 90 | 91 | > Lists files inside the source volume 92 | 93 | Verify against the Google Drive content: 94 | 95 | ~~~~ 96 | $ docker run --rm \ 97 | -v jenkins_test_restore:/source \ 98 | -v volumerize_credentials:/credentials \ 99 | -e "VOLUMERIZE_SOURCE=/source" \ 100 | -e "VOLUMERIZE_TARGET=gdocs://youremail@gmail.com/backup" \ 101 | blacklabelops/volumerize verify 102 | ~~~~ 103 | 104 | > Will perform a single verification of the volume contents against the Google Drive archive. 105 | 106 | # Start and Stop Docker Containers 107 | 108 | Volumerize can stop containers before backup and start them after backup. 109 | 110 | First start a test container with the name `jenkins` 111 | 112 | ~~~~ 113 | $ docker run \ 114 | -d -p 80:8080 \ 115 | --name jenkins \ 116 | -v jenkins_volume:/jenkins \ 117 | blacklabelops/jenkins 118 | ~~~~ 119 | 120 | > Starts Jenkins and stores its data inside the Docker volume `jenkins_volume`. 121 | 122 | Now add the containers name inside the environment variable `VOLUMERIZE_CONTAINERS` and start Volumerize in demon mode: 123 | 124 | ~~~~ 125 | $ docker run -d \ 126 | --name volumerize \ 127 | -v jenkins_volume:/source:ro \ 128 | -v volumerize_cache:/volumerize-cache \ 129 | -v volumerize_credentials:/credentials \ 130 | -e "VOLUMERIZE_SOURCE=/source" \ 131 | -e "VOLUMERIZE_TARGET=gdocs://youremail@gmail.com/backup" \ 132 | -e "VOLUMERIZE_CONTAINERS=jenkins" \ 133 | -v /var/run/docker.sock:/var/run/docker.sock \ 134 | blacklabelops/volumerize 135 | ~~~~ 136 | 137 | > Needs access to the docker host over the directive `-v /var/run/docker.sock:/var/run/docker.sock` 138 | 139 | You can test the backup routine: 140 | 141 | ~~~~ 142 | $ docker exec volumerize backup 143 | ~~~~ 144 | 145 | > Triggers the backup inside the volume, the name `jenkins` should appear on the console. 146 | 147 | > Note: Make sure your container is not running with docker auto restart! 148 | -------------------------------------------------------------------------------- /backends/AmazonS3/README.md: -------------------------------------------------------------------------------- 1 | # Using Volumerize With Amazon S3 2 | 3 | Volumerize can backup Docker volumes on Amazon S3. 4 | 5 | You have to perform the following steps: 6 | 7 | Login to Amazon Web Service developers console and create a service account. 8 | 9 | The Amazon developers console: [Amazon Developers Console](https://aws.amazon.com/console/) 10 | 11 | In order to use all Volumerize features you need the following policy: 12 | 13 | ~~~~ 14 | { 15 | "Version":"2012-10-17", 16 | "Statement": [ 17 | { 18 | "Effect": "Allow", 19 | "Action": "s3:ListAllMyBuckets", 20 | "Resource": "arn:aws:s3:::*" 21 | }, 22 | { 23 | "Effect": "Allow", 24 | "Action": "s3:*", 25 | "Resource": [ 26 | "arn:aws:s3:::BUCKET_NAME", 27 | "arn:aws:s3:::BUCKET_NAME/*" 28 | ] 29 | } 30 | ] 31 | } 32 | ~~~~ 33 | 34 | > Replace BUCKET_NAME with your bucket name! 35 | 36 | Remember the following details: 37 | 38 | * The AWS Access key of your user. 39 | * The AWS Secret key of your user. 40 | * The region of your bucket. 41 | * The bucket name. 42 | 43 | First we start our example container with some data to backup: 44 | 45 | ~~~~ 46 | $ docker run \ 47 | -d -p 80:8080 \ 48 | --name jenkins \ 49 | -v jenkins_volume:/jenkins \ 50 | blacklabelops/jenkins 51 | ~~~~ 52 | 53 | > Starts Jenkins and stores its data inside the Docker volume `jenkins_volume`. 54 | 55 | Now build your s3 url according to the template: 56 | 57 | `s3:://s3..amazonaws.com/[/]`. Regions: [Amazon S3 Regions](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html) 58 | 59 | Start the container in demon mode with your AWS credentials: 60 | 61 | ~~~~ 62 | $ docker run -d \ 63 | --name volumerize \ 64 | -v volumerize_cache:/volumerize-cache \ 65 | -v jenkins_volume:/source:ro \ 66 | -e "VOLUMERIZE_SOURCE=/source" \ 67 | -e "VOLUMERIZE_TARGET=s3://s3.eu-central-1.amazonaws.com/duplicitytest" \ 68 | -e "AWS_ACCESS_KEY_ID=QQWDQIWIDO1QO" \ 69 | -e "AWS_SECRET_ACCESS_KEY=ewlfkwkejflkjwlkej3fjw381" \ 70 | blacklabelops/volumerize 71 | ~~~~ 72 | 73 | > `volumerize_cache` is the local data cache. 74 | 75 | You have to start an initial full backup: 76 | 77 | ~~~~ 78 | $ docker exec volumerize backupFull 79 | ~~~~ 80 | 81 | # Restore from Amazon S3 82 | 83 | Restore is easy, just pass the same environment variables and start the restore script: 84 | 85 | > Note: Remove the read-only option `:ro` on the source volume. 86 | 87 | ~~~~ 88 | $ docker run --rm \ 89 | -v jenkins_test_restore:/source \ 90 | -e "VOLUMERIZE_SOURCE=/source" \ 91 | -e "VOLUMERIZE_TARGET=s3://s3.eu-central-1.amazonaws.com/duplicitytest" \ 92 | -e "AWS_ACCESS_KEY_ID=QQWDQIWIDO1QO" \ 93 | -e "AWS_SECRET_ACCESS_KEY=ewlfkwkejflkjwlkej3fjw381" \ 94 | blacklabelops/volumerize restore 95 | ~~~~ 96 | 97 | > Will perform a test restore inside a separate volume `jenkins_test_restore` 98 | 99 | Check the contents of the volume: 100 | 101 | ~~~~ 102 | $ docker run --rm \ 103 | -v jenkins_test_restore:/source \ 104 | blacklabelops/alpine ls -R /source 105 | ~~~~ 106 | 107 | > Lists files inside the source volume 108 | 109 | Verify against the Amazon S3 Drive content: 110 | 111 | ~~~~ 112 | $ docker run --rm \ 113 | -v jenkins_test_restore:/source \ 114 | -e "VOLUMERIZE_SOURCE=/source" \ 115 | -e "VOLUMERIZE_TARGET=s3://s3.eu-central-1.amazonaws.com/duplicitytest" \ 116 | -e "AWS_ACCESS_KEY_ID=QQWDQIWIDO1QO" \ 117 | -e "AWS_SECRET_ACCESS_KEY=ewlfkwkejflkjwlkej3fjw381" \ 118 | blacklabelops/volumerize verify 119 | ~~~~ 120 | 121 | > Will perform a single verification of the volume contents against the Amazon S3 archive. 122 | 123 | # Start and Stop Docker Containers 124 | 125 | Volumerize can stop containers before backup and start them after backup. 126 | 127 | First start a test container with the name `jenkins` 128 | 129 | ~~~~ 130 | $ docker run \ 131 | -d -p 80:8080 \ 132 | --name jenkins \ 133 | -v jenkins_volume:/jenkins \ 134 | blacklabelops/jenkins 135 | ~~~~ 136 | 137 | > Starts Jenkins and stores its data inside the Docker volume `jenkins_volume`. 138 | 139 | Now add the containers name inside the environment variable `VOLUMERIZE_CONTAINERS` and start Volumerize in demon mode: 140 | 141 | ~~~~ 142 | $ docker run -d \ 143 | --name volumerize \ 144 | -v jenkins_volume:/source:ro \ 145 | -v volumerize_cache:/volumerize-cache \ 146 | -e "VOLUMERIZE_SOURCE=/source" \ 147 | -e "VOLUMERIZE_TARGET=s3://s3.eu-central-1.amazonaws.com/duplicitytest" \ 148 | -e "AWS_ACCESS_KEY_ID=QQWDQIWIDO1QO" \ 149 | -e "AWS_SECRET_ACCESS_KEY=ewlfkwkejflkjwlkej3fjw381" \ 150 | -e "VOLUMERIZE_CONTAINERS=jenkins" \ 151 | -v /var/run/docker.sock:/var/run/docker.sock \ 152 | blacklabelops/volumerize 153 | ~~~~ 154 | 155 | > Needs access to the docker host over the directive `-v /var/run/docker.sock:/var/run/docker.sock` 156 | 157 | You can test the backup routine: 158 | 159 | ~~~~ 160 | $ docker exec volumerize backup 161 | ~~~~ 162 | 163 | > Triggers the backup inside the volume, the name `jenkins` should appear on the console. 164 | 165 | > Note: Make sure your container is not running with docker auto restart! 166 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.13.5 as megacmd-compiler 2 | 3 | RUN apk add --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing --update \ 4 | libtool \ 5 | autoconf \ 6 | automake \ 7 | c-ares-dev \ 8 | crypto++-dev \ 9 | curl \ 10 | curl-dev \ 11 | file \ 12 | g++ \ 13 | gcc \ 14 | git \ 15 | sqlite-dev \ 16 | pcre-dev \ 17 | libc-dev \ 18 | libffi-dev \ 19 | libressl-dev \ 20 | libsodium \ 21 | libsodium-dev \ 22 | libuv-dev \ 23 | make \ 24 | openssl \ 25 | openssl-dev \ 26 | readline-dev \ 27 | zlib-dev \ 28 | freeimage-dev && \ 29 | git clone https://github.com/meganz/MEGAcmd.git /opt/MEGAcmd && \ 30 | cd /opt/MEGAcmd && \ 31 | git submodule update --init --recursive && \ 32 | sh autogen.sh && \ 33 | ./configure && \ 34 | make -j $(nproc) && \ 35 | make install 36 | 37 | 38 | FROM alpine:3.13.5 39 | MAINTAINER Steffen Bleul 40 | 41 | ARG JOBBER_VERSION=1.4.4 42 | ARG DOCKER_VERSION=20.10.6 43 | ARG DUPLICITY_VERSION=0.8.21 44 | ARG DUPLICITY_SERIES=0.8 45 | 46 | COPY --from=megacmd-compiler /usr/local/bin/mega-* /usr/local/bin/ 47 | COPY --from=megacmd-compiler /usr/local/lib/libmega* /usr/local/lib/ 48 | 49 | RUN apk upgrade --update && \ 50 | apk add \ 51 | bash \ 52 | tzdata \ 53 | vim \ 54 | tini \ 55 | su-exec \ 56 | gzip \ 57 | tar \ 58 | wget \ 59 | curl \ 60 | build-base \ 61 | glib-dev \ 62 | gmp-dev \ 63 | asciidoc \ 64 | curl-dev \ 65 | tzdata \ 66 | openssh \ 67 | libressl-dev \ 68 | libressl \ 69 | duply \ 70 | ca-certificates \ 71 | libffi-dev \ 72 | librsync-dev \ 73 | gcc \ 74 | alpine-sdk \ 75 | linux-headers \ 76 | musl-dev \ 77 | rsync \ 78 | lftp \ 79 | py-cryptography \ 80 | librsync \ 81 | librsync-dev \ 82 | python3-dev \ 83 | duplicity \ 84 | py3-pip && \ 85 | pip install --upgrade pip && \ 86 | pip install \ 87 | setuptools \ 88 | fasteners \ 89 | google-api-python-client>=2.2.0 \ 90 | PyDrive \ 91 | chardet \ 92 | azure-storage-blob \ 93 | azure-storage-queue \ 94 | boto \ 95 | lockfile \ 96 | paramiko \ 97 | python-keystoneclient \ 98 | python-swiftclient \ 99 | requests \ 100 | requests_oauthlib \ 101 | urllib3 \ 102 | b2 \ 103 | b2sdk \ 104 | dropbox && \ 105 | mkdir -p /etc/volumerize /volumerize-cache /opt/volumerize && \ 106 | curl -fSL "https://code.launchpad.net/duplicity/${DUPLICITY_SERIES}-series/${DUPLICITY_VERSION}/+download/duplicity-${DUPLICITY_VERSION}.tar.gz" -o /tmp/duplicity.tar.gz && \ 107 | export DUPLICITY_SHA=2d048377c839ae56fc2828997c9aa7ba8c339e815e1e2ae738652037508ec276a2c72583687da34408fadd4839011e242b51bc73cca954227fc51db5683c258c && \ 108 | echo 'Calculated checksum: '$(sha512sum /tmp/duplicity.tar.gz) && \ 109 | # echo "$DUPLICITY_SHA /tmp/duplicity.tar.gz" | sha512sum -c - && \ 110 | tar -xzvf /tmp/duplicity.tar.gz -C /tmp && \ 111 | cd /tmp/duplicity-${DUPLICITY_VERSION} && python3 setup.py install && \ 112 | # Install Jobber 113 | export CONTAINER_UID=1000 && \ 114 | export CONTAINER_GID=1000 && \ 115 | export CONTAINER_USER=jobber_client && \ 116 | export CONTAINER_GROUP=jobber_client && \ 117 | # Install tools 118 | apk add \ 119 | go \ 120 | git \ 121 | curl \ 122 | wget \ 123 | make && \ 124 | # Install Jobber 125 | addgroup -g $CONTAINER_GID jobber_client && \ 126 | adduser -u $CONTAINER_UID -G jobber_client -s /bin/bash -S jobber_client && \ 127 | wget --directory-prefix=/tmp https://github.com/dshearer/jobber/releases/download/v${JOBBER_VERSION}/jobber-${JOBBER_VERSION}-r0.apk && \ 128 | apk add --allow-untrusted --no-scripts /tmp/jobber-${JOBBER_VERSION}-r0.apk && \ 129 | # Install Docker CLI 130 | curl -fSL "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz" -o /tmp/docker.tgz && \ 131 | export DOCKER_SHA=3aab01ab17734866df8b98938243f3f4c835592c && \ 132 | echo 'Calculated checksum: '$(sha1sum /tmp/docker.tgz) && \ 133 | echo "$DOCKER_SHA /tmp/docker.tgz" | sha1sum -c - && \ 134 | tar -xzvf /tmp/docker.tgz -C /tmp && \ 135 | cp /tmp/docker/docker /usr/local/bin/ && \ 136 | # Install MEGAtools 137 | curl -fSL "https://megatools.megous.com/builds/megatools-1.10.3.tar.gz" -o /tmp/megatools.tgz && \ 138 | tar -xzvf /tmp/megatools.tgz -C /tmp && \ 139 | cd /tmp/megatools-1.10.3 && \ 140 | ./configure && \ 141 | make && \ 142 | make install && \ 143 | # Install MegaCMD dependencies 144 | apk add --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing --update --no-cache \ 145 | c-ares \ 146 | crypto++ \ 147 | libcurl \ 148 | libtool \ 149 | libuv \ 150 | libpcrecpp \ 151 | libsodium \ 152 | sqlite-libs \ 153 | sqlite \ 154 | pcre \ 155 | readline \ 156 | freeimage \ 157 | zlib && \ 158 | # Test MegaCMD binaries 159 | find /usr/local/bin -type f -executable -name 'mega-*' | \ 160 | while read binary; do command -v $binary > /dev/null; done && \ 161 | # Cleanup 162 | apk del \ 163 | go \ 164 | git \ 165 | curl \ 166 | wget \ 167 | python3-dev \ 168 | libffi-dev \ 169 | libressl-dev \ 170 | libressl \ 171 | alpine-sdk \ 172 | linux-headers \ 173 | gcc \ 174 | musl-dev \ 175 | librsync-dev \ 176 | make && \ 177 | apk add \ 178 | openssl && \ 179 | rm -rf /var/cache/apk/* && rm -rf /tmp/* 180 | 181 | ENV VOLUMERIZE_HOME=/etc/volumerize \ 182 | VOLUMERIZE_CACHE=/volumerize-cache \ 183 | VOLUMERIZE_SCRIPT_DIR=/opt/volumerize \ 184 | PATH=$PATH:/etc/volumerize \ 185 | GOOGLE_DRIVE_SETTINGS=/credentials/cred.file \ 186 | GOOGLE_DRIVE_CREDENTIAL_FILE=/credentials/googledrive.cred \ 187 | GPG_TTY=/dev/console 188 | 189 | USER root 190 | WORKDIR /etc/volumerize 191 | VOLUME ["/volumerize-cache"] 192 | COPY imagescripts/ /opt/volumerize/ 193 | COPY scripts/ /etc/volumerize/ 194 | ENTRYPOINT ["/sbin/tini","--","/opt/volumerize/docker-entrypoint.sh"] 195 | CMD ["volumerize"] 196 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Blacklabelops Volumerize 2 | 3 | [![Circle CI](https://circleci.com/gh/blacklabelops/volumerize.svg?style=shield)](https://circleci.com/gh/blacklabelops/volumerize) 4 | [![Open Issues](https://img.shields.io/github/issues/blacklabelops/volumerize.svg)](https://github.com/blacklabelops/volumerize/issues) [![Stars on GitHub](https://img.shields.io/github/stars/blacklabelops/volumerize.svg)](https://github.com/blacklabelops/volumerize/stargazers) 5 | [![Docker Stars](https://img.shields.io/docker/stars/blacklabelops/volumerize.svg)](https://hub.docker.com/r/blacklabelops/volumerize/) [![Docker Pulls](https://img.shields.io/docker/pulls/blacklabelops/volumerize.svg)](https://hub.docker.com/r/blacklabelops/volumerize/) 6 | 7 | [![Try in PWD](https://raw.githubusercontent.com/play-with-docker/stacks/master/assets/images/button.png)](https://labs.play-with-docker.com/?stack=https://raw.githubusercontent.com/blacklabelops/volumerize/master/dc-pwd.yml) 8 | 9 | Blacklabelops backup and restore solution for Docker volume backups. It is based on the command line tool Duplicity. Dockerized and Parameterized for easier use and configuration. 10 | 11 | This is not a tool that can clone and backup data from running databases. You should always stop all containers running on your data before doing backups. Always make sure you're not a victim of unexpected data corruption. 12 | 13 | Also note that the easier the tools the easier it is to lose data! Always make sure the tool works correct by checking the backup data itself, e.g. S3 bucket. Check the configuration double time and enable some check options this image offers. E.g. attaching volumes read only. 14 | 15 | Features: 16 | 17 | * Multiple Backends 18 | * Cron Schedule 19 | * Start and Stop Containers 20 | 21 | Supported backends: 22 | 23 | * Filesystem 24 | * Amazon S3 25 | * DropBox 26 | * Google Drive 27 | * ssh/scp 28 | * rsync 29 | 30 | and many more: [Duplicity Supported Backends](http://duplicity.nongnu.org/index.html) 31 | 32 | # Volume Backups Tutorials 33 | 34 | Docker Volume Backups on: 35 | 36 | Backblaze B2: [Readme](https://github.com/blacklabelops/volumerize/tree/master/backends/BackblazeB2) 37 | 38 | Amazon S3: [Readme](https://github.com/blacklabelops/volumerize/tree/master/backends/AmazonS3) 39 | 40 | Dropbox: [Readme](https://github.com/blacklabelops/volumerize/tree/master/backends/Dropbox) 41 | 42 | Google Drive: [Readme](https://github.com/blacklabelops/volumerize/tree/master/backends/GoogleDrive) 43 | 44 | Mega: [Readme](https://github.com/blacklabelops/volumerize/tree/master/backends/Mega) 45 | 46 | # Make It Short 47 | 48 | You can make backups of your Docker application volume just by typing: 49 | 50 | ~~~~ 51 | $ docker run --rm \ 52 | --name volumerize \ 53 | -v yourvolume:/source:ro \ 54 | -v backup_volume:/backup \ 55 | -v cache_volume:/volumerize-cache \ 56 | -e "VOLUMERIZE_SOURCE=/source" \ 57 | -e "VOLUMERIZE_TARGET=file:///backup" \ 58 | blacklabelops/volumerize backup 59 | ~~~~ 60 | 61 | > Hooks up your volume with the name `yourvolume` and backups to the volume `backup_volume` 62 | 63 | # How It Works 64 | 65 | The container has a default startup mode. Any specific behavior is done by defining envrionment variables at container startup (`docker run`). The default container behavior is to start in daemon mode and do incremental daily backups. 66 | 67 | Your application data must be saved inside a Docker volume. You can list your volumes with the Docker command `docker volume ls`. You have to attach the volume to the backup container using the `-v` option. Choose an arbitrary name for the folder and add the `:ro`option to make the sources read only. 68 | 69 | Example using Jenkins: 70 | 71 | ~~~~ 72 | $ docker run \ 73 | -d -p 80:8080 \ 74 | --name jenkins \ 75 | -v jenkins_volume:/jenkins \ 76 | blacklabelops/jenkins 77 | ~~~~ 78 | 79 | > Starts Jenkins and stores its data inside the Docker volume `jenkins_volume`. 80 | 81 | Now attach the Jenkins data to folders inside the container and tell blacklabelops/volumerize to backup folder `/source` to folder `/backup`. 82 | 83 | ~~~~ 84 | $ docker run -d \ 85 | --name volumerize \ 86 | -v jenkins_volume:/source:ro \ 87 | -v backup_volume:/backup \ 88 | -v cache_volume:/volumerize-cache \ 89 | -e "VOLUMERIZE_SOURCE=/source" \ 90 | -e "VOLUMERIZE_TARGET=file:///backup" \ 91 | blacklabelops/volumerize 92 | ~~~~ 93 | 94 | > Will start the Volumerizer. The volume jenkins_volume is now folder `/source` and backups_volume is now folder `/backup` inside the container. 95 | 96 | You can execute commands inside the container, e.g. doing an immediate backup or even restore: 97 | 98 | ~~~~ 99 | $ docker exec volumerize backup 100 | ~~~~ 101 | 102 | > Will trigger a backup. 103 | 104 | # Backup Multiple volumes 105 | 106 | The container can backup one source folder, see environment variable `VOLUMERIZE_TARGET`. If you want to backup multiple volumes you will have to hook up multiple volumes under the same source folder. 107 | 108 | Example: 109 | 110 | * Volume: application_data 111 | * Volume: application_database_data 112 | * Volume: application_configuration 113 | 114 | Now start the container hook them up under the same folder `source`. 115 | 116 | ~~~~ 117 | $ docker run -d \ 118 | --name volumerize \ 119 | -v application_data:/source/application_data:ro \ 120 | -v application_database_data:/source/application_database_data:ro \ 121 | -v application_configuration:/source/application_configuration:ro \ 122 | -v backup_volume:/backup \ 123 | -v cache_volume:/volumerize-cache \ 124 | -e "VOLUMERIZE_SOURCE=/source" \ 125 | -e "VOLUMERIZE_TARGET=file:///backup" \ 126 | blacklabelops/volumerize 127 | ~~~~ 128 | 129 | > Will run Volumerize on the common parent folder `/source`. 130 | 131 | # Backup Restore 132 | 133 | A restore is simple. First stop your Volumerize container and start a another container with the same 134 | environment variables and the same volume but without read-only mode! This is important in order to get the same directory structure as when you did your backup! 135 | 136 | Tip: Now add the read-only option to your backup container! 137 | 138 | Example: 139 | 140 | You did your backups with the following settings: 141 | 142 | ~~~~ 143 | $ docker run -d \ 144 | --name volumerize \ 145 | -v jenkins_volume:/source:ro \ 146 | -v backup_volume:/backup \ 147 | -v cache_volume:/volumerize-cache \ 148 | -e "VOLUMERIZE_SOURCE=/source" \ 149 | -e "VOLUMERIZE_TARGET=file:///backup" \ 150 | blacklabelops/volumerize 151 | ~~~~ 152 | 153 | Then stop the backup container and restore with the following command. The only difference is that we exclude the read-only option `:ro` from the source volume and added it to the backup volume: 154 | 155 | ~~~~ 156 | $ docker stop volumerize 157 | $ docker run --rm \ 158 | -v jenkins_volume:/source \ 159 | -v backup_volume:/backup:ro \ 160 | -v cache_volume:/volumerize-cache \ 161 | -e "VOLUMERIZE_SOURCE=/source" \ 162 | -e "VOLUMERIZE_TARGET=file:///backup" \ 163 | blacklabelops/volumerize restore 164 | $ docker start volumerize 165 | ~~~~ 166 | 167 | > Triggers a once time restore. The container for executing the restore command will be deleted afterwards 168 | 169 | You can restore from a particular backup by adding a time parameter to the command `restore`. For example, using `restore -t 3D` at the end in the above command will restore a backup from 3 days ago. See [the Duplicity manual](http://duplicity.nongnu.org/vers7/duplicity.1.html#sect8) to view the accepted time formats. 170 | 171 | To see the available backups, use the command `list` before doing a `restore`. 172 | 173 | ## Dry run 174 | 175 | You can pass the `--dry-run` parameter to the restore command in order to test the restore functionality: 176 | 177 | ~~~~ 178 | $ docker run --rm \ 179 | -v jenkins_volume:/source \ 180 | -v backup_volume:/backup:ro \ 181 | -v cache_volume:/volumerize-cache \ 182 | -e "VOLUMERIZE_SOURCE=/source" \ 183 | -e "VOLUMERIZE_TARGET=file:///backup" \ 184 | blacklabelops/volumerize restore --dry-run 185 | ~~~~ 186 | 187 | But in order to see the differences between backup and source you need the verify command: 188 | 189 | ~~~~ 190 | $ docker run --rm \ 191 | -v jenkins_volume:/source \ 192 | -v backup_volume:/backup:ro \ 193 | -v cache_volume:/volumerize-cache \ 194 | -e "VOLUMERIZE_SOURCE=/source" \ 195 | -e "VOLUMERIZE_TARGET=file:///backup" \ 196 | blacklabelops/volumerize verify 197 | ~~~~ 198 | 199 | # Periodic Backups 200 | 201 | The default cron setting for this container is: `0 0 4 * * *`. That's four o'clock in the morning UTC. You can set your own schedule with the environment variable `VOLUMERIZE_JOBBER_TIME`. 202 | 203 | You can set the time zone with the environment variable `TZ`. 204 | 205 | The syntax is different from cron because I use Jobber as a cron tool: [Jobber Time Strings](http://dshearer.github.io/jobber/doc/v1.1/#/time-strings) 206 | 207 | Example: 208 | 209 | ~~~~ 210 | $ docker run -d \ 211 | --name volumerize \ 212 | -v jenkins_volume:/source:ro \ 213 | -v backup_volume:/backup \ 214 | -v cache_volume:/volumerize-cache \ 215 | -e "TZ=Europe/Berlin" \ 216 | -e "VOLUMERIZE_SOURCE=/source" \ 217 | -e "VOLUMERIZE_TARGET=file:///backup" \ 218 | -e "VOLUMERIZE_JOBBER_TIME=0 0 3 * * *" \ 219 | blacklabelops/volumerize 220 | ~~~~ 221 | 222 | > Backups at three o'clock in the morning according to german local time. 223 | 224 | # Docker Container Restarts 225 | 226 | This image can stop and start Docker containers before and after backup. Docker containers are specified using the environment variable `VOLUMERIZE_CONTAINERS`. Just enter their names in a empty space separated list. 227 | 228 | Example: 229 | 230 | * Docker container application with name `application` 231 | * Docker container application database with name `application_database` 232 | 233 | Note: Needs the parameter `-v /var/run/docker.sock:/var/run/docker.sock` in order to be able to start and stop containers on the host. 234 | 235 | Example: 236 | 237 | ~~~~ 238 | $ docker run -d \ 239 | --name volumerize \ 240 | -v /var/run/docker.sock:/var/run/docker.sock \ 241 | -v jenkins_volume:/source:ro \ 242 | -v backup_volume:/backup \ 243 | -v cache_volume:/volumerize-cache \ 244 | -e "VOLUMERIZE_SOURCE=/source" \ 245 | -e "VOLUMERIZE_TARGET=file:///backup" \ 246 | -e "VOLUMERIZE_CONTAINERS=application application_database" \ 247 | blacklabelops/volumerize 248 | ~~~~ 249 | 250 | > The startup routine will be applied to the following scripts: backup, backupFull, restore and periodBackup. 251 | 252 | Test the routine! 253 | 254 | ~~~~ 255 | $ docker exec volumerize backup 256 | ~~~~ 257 | 258 | ### Additional Docker CLI API configurations 259 | > If the docker host version is earlier than 1.12 then include the following docker api setting, Volumerize uses docker CLI ver 1.12 which uses Docker API version 1.24. One needs to set the compatible API version of the docker host 260 | ie. Docker host version 1.11 uses API 1.23 261 | 262 | ~~~~ 263 | docker version 264 | Client: 265 | Version: 1.11.2 266 | API version: 1.23 267 | Go version: go1.8 268 | Git commit: 5be46ee-synology 269 | Built: Fri May 12 16:36:47 2017 270 | OS/Arch: linux/amd64 271 | 272 | Server: 273 | Version: 1.11.2 274 | API version: 1.23 275 | Go version: go1.8 276 | Git commit: 5be46ee-synology 277 | Built: Fri May 12 16:36:47 2017 278 | OS/Arch: linux/amd64 279 | ~~~~ 280 | Then use the following -e argument 281 | ~~~~ 282 | $ docker run -d \ 283 | --name volumerize \ 284 | -v /var/run/docker.sock:/var/run/docker.sock \ 285 | ... 286 | ... 287 | -e "DOCKER_API_VERSION=1.23" \ 288 | ... 289 | ... 290 | blacklabelops/volumerize 291 | ~~~~ 292 | ### Additional Docker considerations 293 | Warning: Make sure your container is running under the correct restart policy. Tools like Docker, Docker-Compose, Docker-Swarm, Kubernetes and Cattle may restart the container even when Volumerize stops it. Backups done under running instances may end in corrupted backups and even corrupted data. Always make sure that the command `docker stop` really stops an instance and there will be no restart of the underlying deployment technology. You can test this by running `docker stop` and check with `docker ps` that the container is really stopped. 294 | 295 | # Duplicity Parameters 296 | 297 | Under the hood blacklabelops/volumerize uses duplicity. See here for duplicity command line options: [Duplicity CLI Options](http://duplicity.nongnu.org/vers7/duplicity.1.html#sect5) 298 | 299 | You can pass duplicity options inside Volumerize. Duplicity options will be passed by the environment-variable `VOLUMERIZE_DUPLICITY_OPTIONS`. The options will be added to all blacklabelops/volumerize commands and scripts. E.g. the option `--dry-run` will put the whole container in demo mode as all duplicity commands will only be simulated. 300 | 301 | Example: 302 | 303 | ~~~~ 304 | $ docker run -d \ 305 | --name volumerize \ 306 | -v jenkins_volume:/source:ro \ 307 | -v backup_volume:/backup \ 308 | -v cache_volume:/volumerize-cache \ 309 | -e "VOLUMERIZE_SOURCE=/source" \ 310 | -e "VOLUMERIZE_TARGET=file:///backup" \ 311 | -e "VOLUMERIZE_DUPLICITY_OPTIONS=--dry-run" \ 312 | blacklabelops/volumerize 313 | ~~~~ 314 | 315 | > Will only operate in dry-run simulation mode. 316 | 317 | # Symmetric Backup Encryption 318 | 319 | You can encrypt your backups by setting a secure passphrase inside the environment variable `PASSPHRASE`. 320 | 321 | Creating a secure passphrase: 322 | 323 | ~~~~ 324 | $ docker run --rm blacklabelops/volumerize openssl rand -base64 128 325 | ~~~~ 326 | 327 | > Prints an appropriate password on the console. 328 | 329 | Example: 330 | 331 | ~~~~ 332 | $ docker run -d \ 333 | --name volumerize \ 334 | -v jenkins_volume:/source:ro \ 335 | -v backup_volume:/backup \ 336 | -v cache_volume:/volumerize-cache \ 337 | -e "VOLUMERIZE_SOURCE=/source" \ 338 | -e "VOLUMERIZE_TARGET=file:///backup" \ 339 | -e "PASSPHRASE=Jzwv1V83LHwtsbulVS7mMyijStBAs7Qr/V2MjuYtKg4KQVadRM" \ 340 | blacklabelops/volumerize 341 | ~~~~ 342 | 343 | > Same functionality as described above but all backups will be encrypted. 344 | 345 | # Asymmetric Key-Based Backup Encryption 346 | 347 | You can encrypt your backups with secure secret keys. 348 | 349 | You need: 350 | 351 | * A key, specified by the environment-variable `VOLUMERIZE_GPG_PRIVATE_KEY` 352 | * A key passphrase, specified by the environment-variable `PASSPHRASE` 353 | 354 | Creating a key? Install gpg on your comp and type: 355 | 356 | ~~~~ 357 | $ gpg2 --full-gen-key 358 | Please select what kind of key you want: 359 | (1) RSA and RSA (default) 360 | (2) DSA and Elgamal 361 | (3) DSA (sign only) 362 | (4) RSA (sign only) 363 | Your selection? 1 364 | RSA keys may be between 1024 and 4096 bits long. 365 | What keysize do you want? (2048) 366 | Requested keysize is 2048 bits 367 | Please specify how long the key should be valid. 368 | 0 = key does not expire 369 | = key expires in n days 370 | w = key expires in n weeks 371 | m = key expires in n months 372 | y = key expires in n years 373 | Key is valid for? (0) 374 | Key does not expire at all 375 | Is this correct? (y/N) y 376 | 377 | GnuPG needs to construct a user ID to identify your key. 378 | 379 | Real name: YourName 380 | Email address: yourname@youremail.com 381 | Comment: 382 | You selected this USER-ID: 383 | "YourName " 384 | 385 | Change (N)ame, (C)omment, (E)mail or (O)kay/(Q)uit? O 386 | $ gpg2 --export-secret-keys --armor yourname@youremail.com > MyKey.asc 387 | ~~~~ 388 | 389 | > Note: Currently, this image only supports keys without passwords. The import routine is at fault, it would always prompt for passwords. 390 | 391 | You need to get the key id: 392 | ```shell 393 | $ gpg -k yourname@youremail.com | head -n 2 | tail -n 1 | awk '{print $1}' 394 | ``` 395 | 396 | Example: 397 | 398 | ~~~~ 399 | $ docker run -d \ 400 | --name volumerize \ 401 | -v jenkins_volume:/source:ro \ 402 | -v backup_volume:/backup \ 403 | -v cache_volume:/volumerize-cache \ 404 | -v $(pwd)/MyKey.asc:/key/MyKey.asc \ 405 | -e "VOLUMERIZE_SOURCE=/source" \ 406 | -e "VOLUMERIZE_TARGET=file:///backup" \ 407 | -e "VOLUMERIZE_GPG_PRIVATE_KEY=/key/MyKey.asc" \ 408 | -e GPG_KEY_ID= 409 | -e "PASSPHRASE=" \ 410 | blacklabelops/volumerize 411 | ~~~~ 412 | 413 | > This will import a key without a password set. 414 | 415 | Test the routine! 416 | 417 | ~~~~ 418 | $ docker exec volumerize backup 419 | ~~~~ 420 | 421 | # Enforcing Full Backups Periodically 422 | 423 | The default behavior is that the initial backup is a full backup. Afterwards, Volumerize will perform incremental backups. You can enforce another full backup periodically by specifying the environment variable `VOLUMERIZE_FULL_IF_OLDER_THAN`. 424 | 425 | The format is a number followed by one of the characters s, m, h, D, W, M, or Y. (indicating seconds, minutes, hours, days, weeks, months, or years) 426 | 427 | Examples: 428 | 429 | * After three Days: 3D 430 | * After one month: 1M 431 | * After 55 minutes: 55m 432 | 433 | Volumerize Example: 434 | 435 | ~~~~ 436 | $ docker run -d \ 437 | --name volumerize \ 438 | -v jenkins_volume:/source:ro \ 439 | -v backup_volume:/backup \ 440 | -v cache_volume:/volumerize-cache \ 441 | -e "TZ=Europe/Berlin" \ 442 | -e "VOLUMERIZE_SOURCE=/source" \ 443 | -e "VOLUMERIZE_TARGET=file:///backup" \ 444 | -e "VOLUMERIZE_FULL_IF_OLDER_THAN=7D" \ 445 | blacklabelops/volumerize 446 | ~~~~ 447 | 448 | > Will enforce a full backup after seven days. 449 | 450 | For the difference between a full and incremental backup, see [Duplicity's documentation](http://duplicity.nongnu.org/vers7/duplicity.1.html). 451 | 452 | # Post scripts and pre scripts (prepost strategies) 453 | 454 | 455 | Pre-scripts must be located at `/preexecute/$duplicity_action/$your_scripts_here`. 456 | 457 | Post-scripts must be located at `/postexecute/$duplicity_action/$your_scripts_here`. 458 | 459 | `$duplicity_action` folder must be named `backup`, `restore` or `verify`. 460 | 461 | > Note: `backup` action is the same for the scripts `backup`, `backupFull`, `backupIncremental` and `periodicBackup`. 462 | 463 | All `.sh` files located in the `$duplicity_action` folder will be executed in alphabetical order. 464 | 465 | When using prepost strategies, this will be the execution flow: `pre-scripts -> stop containers -> duplicity action -> start containers -> post-scripts`. 466 | 467 | Some premade strategies are available at [prepost strategies](prepost_strategies). 468 | 469 | # Container Scripts 470 | 471 | This image creates at container startup some convenience scripts. 472 | Under the hood blacklabelops/volumerize uses duplicity. To pass script parameters, see here for duplicity command line options: [Duplicity CLI Options](http://duplicity.nongnu.org/vers7/duplicity.1.html#sect5) 473 | 474 | | Script | Description | 475 | |--------|-------------| 476 | | backup | Creates an backup with the containers configuration | 477 | | backupFull | Creates a full backup with the containers configuration | 478 | | backupIncremental | Creates an incremental backup with the containers configuration | 479 | | list | List all available backups | 480 | | verify | Compare the latest backup to your local files | 481 | | restore | Be Careful! Triggers an immediate force restore with the latest backup | 482 | | periodicBackup | Same script that will be triggered by the periodic schedule | 483 | | startContainers | Starts the specified Docker containers | 484 | | stopContainers | Stops the specified Docker containers | 485 | | remove-older-than | Delete older backups ([Time formats](http://duplicity.nongnu.org/vers7/duplicity.1.html#sect8))| 486 | | cleanCacheLocks | Cleanup of old Cache locks. | 487 | | prepoststrategy `$execution_phase` `$duplicity_action` | Execute all `.sh` files for the specified exeuction phase and duplicity action in alphabetical order. | 488 | 489 | `$execution_phase` must be `preAction` or `postAction`. 490 | 491 | `$duplicity_action` must be `backup`, `verify` or `restpore`. 492 | 493 | Example triggering script inside running container: 494 | 495 | ~~~~ 496 | $ docker exec volumerize backup 497 | ~~~~ 498 | 499 | > Executes script `backup` inside container with name `volumerize` 500 | 501 | Example passing script parameter: 502 | 503 | ~~~~ 504 | $ docker exec volumerize backup --dry-run 505 | ~~~~ 506 | 507 | > `--dry-run` will simulate not execute the backup procedure. 508 | 509 | 510 | # Build The Project 511 | 512 | Check out the project at Github. 513 | 514 | # Multiple Backups 515 | 516 | You can specify multiple backup jobs with one container with enumerated environment variables. Each environment variable must be followed by a number starting with 1. Example `VOLUMERIZE_SOURCE1`, `VOLUMERIZE_SOURCE2` or `VOLUMERIZE_SOURCE3`. 517 | 518 | The following environment variables can be enumerated: 519 | 520 | * VOLUMERIZE_SOURCE 521 | * VOLUMERIZE_TARGET 522 | * VOLUMERIZE_CACHE 523 | * VOLUMERIZE_INCLUDE 524 | 525 | When using multiple backup jobs you will have to specify a cache directory for each backup. The minimum required environment variables for each job is: 526 | 527 | * VOLUMERIZE_SOURCE 528 | * VOLUMERIZE_TARGET 529 | * VOLUMERIZE_CACHE 530 | 531 | Also the included helper scripts will change their behavior when you use enumerated environment variables. By default each script will run on all backup jobs. 532 | 533 | Example: Executing the script `backup` will backup all jobs. 534 | 535 | The first parameter of each script can be a job number, e.g. `1`, `2` or `3`. 536 | 537 | Example: Executing the script `backup 1` will only trigger backup on job 1. 538 | 539 | Full example for multiple job specifications: 540 | 541 | ~~~~ 542 | $ docker run -d \ 543 | --name volumerize \ 544 | -v /var/run/docker.sock:/var/run/docker.sock \ 545 | -v jenkins_volume:/source:ro \ 546 | -v jenkins_volume2:/source2:ro \ 547 | -v backup_volume:/backup \ 548 | -v backup_volume2:/backup2 \ 549 | -v cache_volume:/volumerize-cache \ 550 | -v cache_volume2:/volumerize-cache2 \ 551 | -e "VOLUMERIZE_CONTAINERS=jenkins jenkins2" \ 552 | -e "VOLUMERIZE_SOURCE1=/source" \ 553 | -e "VOLUMERIZE_TARGET1=file:///backup" \ 554 | -e "VOLUMERIZE_CACHE1=/volumerize-cache" \ 555 | -e "VOLUMERIZE_SOURCE2=/source2" \ 556 | -e "VOLUMERIZE_TARGET2=file:///backup2" \ 557 | -e "VOLUMERIZE_CACHE2=/volumerize-cache2" \ 558 | blacklabelops/volumerize 559 | ~~~~ 560 | 561 | ## Build the Image 562 | 563 | ~~~~ 564 | $ docker build -t blacklabelops/volumerize . 565 | ~~~~ 566 | 567 | ## Run the Image 568 | 569 | ~~~~ 570 | $ docker run -it --rm blacklabelops/volumerize bash 571 | ~~~~ 572 | --------------------------------------------------------------------------------