├── .gitignore ├── cloudvps-boss ├── backup.conf ├── exclude.conf ├── cloudvps-boss.cron ├── post-fail-backup.d │ ├── 10-upload-fail-status.sh │ └── 20-failure-notify.sh ├── post-backup.d │ └── 10-upload-completed-status.sh ├── cloudvps-boss-list-current-files.sh ├── cloudvps-boss-stats.sh ├── cloudvps-boss-cleanup.sh ├── pre-backup.d │ ├── 10-upload-starting-status.sh │ ├── 15-postgresql_backup.sh │ ├── 11_lockfile_check.sh │ └── 15-mysql_backup.sh ├── cloudvps-boss-verify.sh ├── uninstall.sh ├── cloudvps-boss-update.sh ├── cloudvps-boss-manual-full.sh ├── cloudvps-boss.sh ├── common.sh ├── cloudvps-boss-encryption-setup.sh └── cloudvps-boss-restore.sh ├── CHANGELOG.md ├── credentials.sh ├── install.sh ├── LICENSE.md ├── install_duplicity.sh └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .fuse* 2 | -------------------------------------------------------------------------------- /cloudvps-boss/backup.conf: -------------------------------------------------------------------------------- 1 | # See README for more information on options, retention and other settings. 2 | 3 | # Server hostname. Will be replaced during install. Must be unique among backuped servers. 4 | HOSTNAME="replace_me" 5 | 6 | # Create a full backup if the last is older than X days. 7 | FULL_IF_OLDER_THAN="14D" 8 | 9 | # Keep at max X full backups. 10 | FULL_TO_KEEP="12" 11 | 12 | # Only change this if your tmp folder is to small. See README 13 | TEMPDIR="/tmp" 14 | -------------------------------------------------------------------------------- /cloudvps-boss/exclude.conf: -------------------------------------------------------------------------------- 1 | /backup/cpbackup/ 2 | /dev/ 3 | /home/cpbackuptmp/ 4 | /home/tmp/ 5 | /home/virtfs/ 6 | /lib/firmware/ 7 | /lib/modules/ 8 | /lib/x86_64-linux-gnu/ 9 | /media/ 10 | /mnt/ 11 | /proc/ 12 | /run/ 13 | /selinux/ 14 | /sys/ 15 | /tmp/ 16 | /usr/include/ 17 | /usr/local/cpanel/3rdparty/mailman/cgi-bin/ 18 | /usr/local/psa/admin/logs/ 19 | /usr/tmpDSK/ 20 | /var/cache/ 21 | /var/installatron/ 22 | /var/lib/varnish/ 23 | /var/lib/mysql/ 24 | /var/log/ 25 | /var/named/run-root/ 26 | /var/run/ 27 | /var/tmp/ -------------------------------------------------------------------------------- /cloudvps-boss/cloudvps-boss.cron: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | # Replace root with your email address to receive backup reports. 22 | MAILTO="root" 23 | 24 | # CloudVPS Boss Backup 25 | 1 1 * * * root /usr/local/bin/cloudvps-boss 26 | 27 | -------------------------------------------------------------------------------- /cloudvps-boss/post-fail-backup.d/10-upload-fail-status.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | VERSION="1.9.17" 22 | TITLE="CloudVPS Boss Fail Status Upload ${VERSION}" 23 | 24 | if [[ ! -f "/etc/cloudvps-boss/common.sh" ]]; then 25 | lerror "Cannot find /etc/cloudvps-boss/common.sh" 26 | exit 1 27 | fi 28 | source /etc/cloudvps-boss/common.sh 29 | 30 | touch "/etc/cloudvps-boss/status/${HOSTNAME}/failed" 31 | if [[ $? -ne 0 ]]; then 32 | lerror "Cannot update status" 33 | exit 1 34 | fi 35 | 36 | OLD_IFS="${IFS}" 37 | IFS=$'\n' 38 | SWIFTTOUCH=$(swift upload ${CONTAINER_NAME} "/etc/cloudvps-boss/status/${HOSTNAME}/failed" --object-name "status/${HOSTNAME}/failed" 2>&1 | grep -v -e Warning -e pkg_resources -e oslo) 39 | if [[ $? -ne 0 ]]; then 40 | lerror "Could not upload failed status" 41 | for line in ${SWIFTTOUCH}; do 42 | lerror ${line} 43 | done 44 | fi 45 | IFS="${OLD_IFS}" 46 | 47 | lecho "${TITLE} ended on ${HOSTNAME} at $(date)." 48 | -------------------------------------------------------------------------------- /cloudvps-boss/post-backup.d/10-upload-completed-status.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | VERSION="1.9.17" 22 | TITLE="CloudVPS Boss Completed Status Upload ${VERSION}" 23 | 24 | if [[ ! -f "/etc/cloudvps-boss/common.sh" ]]; then 25 | lerror "Cannot find /etc/cloudvps-boss/common.sh" 26 | exit 1 27 | fi 28 | source /etc/cloudvps-boss/common.sh 29 | 30 | touch "/etc/cloudvps-boss/status/${HOSTNAME}/completed" 31 | if [[ $? -ne 0 ]]; then 32 | lerror "Cannot update status" 33 | exit 1 34 | fi 35 | 36 | OLD_IFS="${IFS}" 37 | IFS=$'\n' 38 | SWIFTTOUCH=$(swift upload ${CONTAINER_NAME} "/etc/cloudvps-boss/status/${HOSTNAME}/completed" --object-name "status/${HOSTNAME}/completed" 2>&1 | grep -v -e Warning -e pkg_resources -e oslo) 39 | if [[ $? -ne 0 ]]; then 40 | lerror "Could not upload completed status" 41 | for line in ${SWIFTTOUCH}; do 42 | lerror ${line} 43 | done 44 | fi 45 | IFS="${OLD_IFS}" 46 | 47 | lecho "${TITLE} ended on ${HOSTNAME} at $(date)." 48 | -------------------------------------------------------------------------------- /cloudvps-boss/cloudvps-boss-list-current-files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | VERSION="1.9.17" 22 | TITLE="CloudVPS Boss File Overview ${VERSION}" 23 | 24 | if [[ ! -f "/etc/cloudvps-boss/common.sh" ]]; then 25 | lerror "Cannot find /etc/cloudvps-boss/common.sh" 26 | exit 1 27 | fi 28 | source /etc/cloudvps-boss/common.sh 29 | 30 | if [[ -n "$1" ]]; then 31 | TIME="$1" 32 | TIMEOPT="--time $1" 33 | TIME_MESS="Requested Time: $1" 34 | fi 35 | 36 | echo "=========================================" 37 | lecho "Start of CloudVPS Boss File Overview" 38 | lecho "Hostname: ${HOSTNAME}" 39 | lecho "$TIME_MESS" 40 | echo "-----------------------------------------" 41 | lecho "duplicity list-current-files --file-prefix=\"${HOSTNAME}.\" --name=\"${HOSTNAME}.\" ${ENCRYPTION_OPTIONS} ${CUSTOM_DUPLICITY_OPTIONS} --allow-source-mismatch --num-retries 100 ${TIMEOPT} ${BACKUP_BACKEND}" 42 | duplicity list-current-files \ 43 | --file-prefix="${HOSTNAME}." \ 44 | --name="${HOSTNAME}." \ 45 | ${ENCRYPTION_OPTIONS} \ 46 | ${CUSTOM_DUPLICITY_OPTIONS} \ 47 | --allow-source-mismatch \ 48 | --num-retries 100 \ 49 | ${TIMEOPT} \ 50 | ${BACKUP_BACKEND} 2>&1 | grep -v -e Warning -e pkg_resources -e oslo 51 | lecho "End of CloudVPS Boss File Overview" 52 | echo "=========================================" 53 | 54 | -------------------------------------------------------------------------------- /cloudvps-boss/cloudvps-boss-stats.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | VERSION="1.9.17" 22 | TITLE="CloudVPS Boss Stats ${VERSION}" 23 | 24 | if [[ ! -f "/etc/cloudvps-boss/common.sh" ]]; then 25 | lerror "Cannot find /etc/cloudvps-boss/common.sh" 26 | exit 1 27 | fi 28 | source /etc/cloudvps-boss/common.sh 29 | 30 | 31 | USED="$(swift stat --lh ${CONTAINER_NAME} 2>&1 | awk '/Bytes/ { print $2}' | grep -v -e Warning -e pkg_resources -e oslo)" 32 | 33 | echo "=========================================" 34 | lecho "Start of CloudVPS Boss Status ${VERSION}" 35 | lecho "Hostname: ${HOSTNAME}" 36 | lecho "External IP: $(curl -s http://ip.cloudvps.nl)" 37 | lecho "Username: ${SWIFT_USERNAME}" 38 | lecho "Storage used: ${USED}" 39 | lecho "Full backups to keep: ${FULL_TO_KEEP}" 40 | lecho "Create full backup if last full backup is older than: ${FULL_IF_OLDER_THAN}" 41 | echo "-----------------------------------------" 42 | lecho "Duplicity collection status:" 43 | OLD_IFS="${IFS}" 44 | IFS=$'\n' 45 | DUPLICITY_STATS="$( 46 | duplicity collection-status \ 47 | --file-prefix="${HOSTNAME}." \ 48 | --name="${HOSTNAME}." \ 49 | ${CUSTOM_DUPLICITY_OPTIONS} \ 50 | ${BACKUP_BACKEND} 2>&1 | grep -v -e Warning -e pkg_resources -e oslo -e tar -e attr -e kwargs)" 51 | for line in ${DUPLICITY_STATS}; do 52 | lecho "${line}" 53 | done 54 | IFS="${OLD_IFS}" 55 | lecho "End of CloudVPS Boss Status" 56 | echo "=========================================" 57 | 58 | -------------------------------------------------------------------------------- /cloudvps-boss/cloudvps-boss-cleanup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | VERSION="1.9.17" 22 | TITLE="CloudVPS Boss Backup Cleanup ${VERSION}" 23 | 24 | ## does not remove backup data. Manpage entry for cleanup: 25 | ## Delete the extraneous duplicity files on the given backend. Non-duplicity files, or files in complete data sets will not be deleted. This should only be necessary after a duplicity session fails or is aborted prematurely. Note that --force will be needed to delete the files instead of just listing them. 26 | 27 | if [[ ! -f "/etc/cloudvps-boss/common.sh" ]]; then 28 | lerror "Cannot find /etc/cloudvps-boss/common.sh" 29 | exit 1 30 | fi 31 | source /etc/cloudvps-boss/common.sh 32 | 33 | lecho "${TITLE} started on ${HOSTNAME} at $(date)." 34 | 35 | lecho "duplicity cleanup --extra-clean --force ${BACKUP_BACKEND}" 36 | 37 | OLD_IFS="${IFS}" 38 | IFS=$'\n' 39 | DUPLICITY_OUTPUT=$(duplicity \ 40 | cleanup \ 41 | --extra-clean \ 42 | --force \ 43 | ${ENCRYPTION_OPTIONS} \ 44 | ${BACKUP_BACKEND} 2>&1 | grep -v -e Warning -e pkg_resources -e oslo) 45 | 46 | if [[ $? -ne 0 ]]; then 47 | for line in ${DUPLICITY_OUTPUT}; do 48 | lerror ${line} 49 | done 50 | lerror "CloudVPS Boss Cleanup FAILED!. Please check server ${HOSTNAME}." 51 | fi 52 | 53 | for line in ${DUPLICITY_OUTPUT}; do 54 | lecho "${line}" 55 | done 56 | IFS="${OLD_IFS}" 57 | 58 | echo 59 | lecho "CloudVPS Boss Cleanup ${VERSION} ended on $(date)." 60 | -------------------------------------------------------------------------------- /cloudvps-boss/pre-backup.d/10-upload-starting-status.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | VERSION="1.9.17" 22 | TITLE="CloudVPS Boss Start Status Upload ${VERSION}" 23 | 24 | if [[ ! -f "/etc/cloudvps-boss/common.sh" ]]; then 25 | lerror "Cannot find /etc/cloudvps-boss/common.sh" 26 | exit 1 27 | fi 28 | source /etc/cloudvps-boss/common.sh 29 | 30 | touch "/etc/cloudvps-boss/status/${HOSTNAME}/started" 31 | if [[ $? -ne 0 ]]; then 32 | lerror "Cannot update status" 33 | exit 1 34 | fi 35 | 36 | OLD_IFS="${IFS}" 37 | IFS=$'\n' 38 | SWIFTTOUCH=$(swift upload ${CONTAINER_NAME} "/etc/cloudvps-boss/status/${HOSTNAME}/started" --object-name "status/${HOSTNAME}/started" 2>&1 | grep -v -e Warning -e pkg_resources -e oslo) 39 | if [[ $? -ne 0 ]]; then 40 | lerror "Could not upload status" 41 | for line in ${SWIFTTOUCH}; do 42 | lerror ${line} 43 | done 44 | fi 45 | IFS="${OLD_IFS}" 46 | 47 | 48 | lecho "Logging version of CloudVPS Boss to Object Store: ${VERSION}" 49 | 50 | touch "/etc/cloudvps-boss/status/${HOSTNAME}/version-${VERSION}" 51 | if [[ $? -ne 0 ]]; then 52 | lerror "Cannot update version" 53 | fi 54 | 55 | OLD_IFS="${IFS}" 56 | IFS=$'\n' 57 | SWIFTTOUCH=$(swift upload ${CONTAINER_NAME} "/etc/cloudvps-boss/status/${HOSTNAME}/version-${VERSION}" --object-name "status/${HOSTNAME}/version-${VERSION}" 2>&1 | grep -v -e UserWarning -e pkg_resources -e oslo) 58 | if [[ $? -ne 0 ]]; then 59 | lerror "Could not upload version" 60 | for line in ${SWIFTTOUCH}; do 61 | lerror ${line} 62 | done 63 | fi 64 | IFS="${OLD_IFS}" 65 | -------------------------------------------------------------------------------- /cloudvps-boss/cloudvps-boss-verify.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | VERSION="1.9.17" 22 | TITLE="CloudVPS Boss Backup Verify ${VERSION}" 23 | 24 | if [[ ! -f "/etc/cloudvps-boss/common.sh" ]]; then 25 | lerror "Cannot find /etc/cloudvps-boss/common.sh" 26 | exit 1 27 | fi 28 | source /etc/cloudvps-boss/common.sh 29 | 30 | lecho "${TITLE} started on ${HOSTNAME} at $(date)." 31 | 32 | lecho "duplicity verify --volsize ${VOLUME_SIZE} --tempdir=\"${TEMPDIR}\" --file-prefix=\"${HOSTNAME}.\" --name=\"${HOSTNAME}.\" --exclude-device-files --allow-source-mismatch --num-retries 100 --exclude-filelist=/etc/cloudvps-boss/exclude.conf ${ENCRYPTION_OPTIONS} ${BACKUP_BACKEND} /" 33 | 34 | OLD_IFS="${IFS}" 35 | IFS=$'\n' 36 | DUPLICITY_OUTPUT=$(duplicity \ 37 | verify \ 38 | --volsize=${VOLUME_SIZE} \ 39 | --tempdir="${TEMPDIR}" \ 40 | --file-prefix="${HOSTNAME}." \ 41 | --name="${HOSTNAME}." \ 42 | --exclude-device-files \ 43 | --allow-source-mismatch \ 44 | --num-retries 100 \ 45 | --exclude-filelist=/etc/cloudvps-boss/exclude.conf \ 46 | ${ENCRYPTION_OPTIONS} \ 47 | ${BACKUP_BACKEND} \ 48 | / 2>&1 | grep -v -e Warning -e pkg_resources -e oslo) 49 | 50 | if [[ $? -ne 0 ]]; then 51 | for line in ${DUPLICITY_OUTPUT}; do 52 | lerror ${line} 53 | done 54 | lerror "CloudVPS Boss Verify FAILED!. Please check server ${HOSTNAME}." 55 | fi 56 | 57 | for line in ${DUPLICITY_OUTPUT}; do 58 | lecho "${line}" 59 | done 60 | IFS="${OLD_IFS}" 61 | 62 | echo 63 | lecho "CloudVPS Boss Verify ${VERSION} ended on $(date)." 64 | -------------------------------------------------------------------------------- /cloudvps-boss/uninstall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | VERSION="1.9.17" 22 | 23 | TITLE="CloudVPS Boss Uninstall ${VERSION}" 24 | 25 | if [[ ! -f "/etc/cloudvps-boss/common.sh" ]]; then 26 | lerror "Cannot find /etc/cloudvps-boss/common.sh" 27 | exit 1 28 | fi 29 | source /etc/cloudvps-boss/common.sh 30 | 31 | read -p "Would you like to completely remove CloudVPS Boss? Your backups will NOT be removed. [y/N]? " choice 32 | 33 | if [[ "${choice}" = "y" ]]; then 34 | lecho "Removing CloudVPS Boss" 35 | for FILE in "/etc/cron.d/cloudvps-boss"; do 36 | remove_file "${FILE}" 37 | done 38 | for SYMLINK in "/usr/local/bin/cloudvps-boss" "/usr/local/bin/cloudvps-boss-restore" "/usr/local/bin/cloudvps-boss-stats" "/usr/local/bin/cloudvps-boss-list-current-files" "/usr/local/bin/cloudvps-boss-update"; do 39 | remove_symlink "${SYMLINK}" 40 | done 41 | for PIP_INSTALLED in "python-swiftclient" "python-keystoneclient" "argparse" "babel" "debtcollector" "futures" "iso8601" "netaddr" "oslo.config" "oslo.i18n" "oslo.serialization" "oslo.utils" "pbr" "prettytable" "requests" "six" "stevedore"; do 42 | for PIP_VERSION in "pip" "pip2" "pip27" "pip2.7"; do 43 | if [[ "$(command_exists_non_verbose ${PIP_VERSION})" ]]; then 44 | lecho "Uninstalling ${PIP_INSTALLED} with ${PIP_VERSION}." 45 | echo "y\n" | ${PIP_VERSION} -q uninstall "${PIP_INSTALLED}" 2>&1 > /dev/null 46 | fi 47 | done 48 | done 49 | for FOLDER in "/usr/local/cloudvps-boss" "/etc/cloudvps-boss/"; do 50 | remove_folder "${FOLDER}" 51 | done 52 | cd 53 | exit 54 | fi 55 | 56 | lecho "Choice was not 'y'. Not removing anything. Exiting." 57 | -------------------------------------------------------------------------------- /cloudvps-boss/cloudvps-boss-update.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2017 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | VERSION="1.9.17" 22 | DUPLICITY_VERSION="0.7.17" 23 | TITLE="CloudVPS Boss Upgrade ${VERSION}" 24 | 25 | DL_SRV="https://download.cloudvps.com/cloudvps-boss/" 26 | 27 | if [[ -f "/etc/cloudvps-boss/duplicity_${DUPLICITY_VERSION}_installed" ]]; then 28 | echo "Duplicity ${DUPLICITY_VERSION} already compiled and installed." 29 | exit 0 30 | fi 31 | 32 | if [[ ! -f "/etc/cloudvps-boss/common.sh" ]]; then 33 | echo "Cannot find /etc/cloudvps-boss/common.sh" 34 | exit 1 35 | fi 36 | source /etc/cloudvps-boss/common.sh 37 | 38 | lecho "${TITLE} started on ${HOSTNAME} at $(date)." 39 | 40 | pushd () { 41 | command pushd "$@" > /dev/null 42 | } 43 | 44 | popd () { 45 | command popd "$@" > /dev/null 46 | } 47 | 48 | if [[ ! -d "/root/.cloudvps-boss" ]]; then 49 | mkdir -p "/root/.cloudvps-boss" 50 | fi 51 | 52 | pushd /root/.cloudvps-boss 53 | 54 | if [[ -f "/root/.cloudvps-boss/cloudvps-boss.tar.gz" ]]; then 55 | lecho "Removing old update file from /root/.cloudvps-boss/cloudvps-boss.tar.gz" 56 | rm -rf /root/.cloudvps-boss/cloudvps-boss.tar.gz 57 | fi 58 | 59 | if [[ -d "/root/.cloudvps-boss/cloudvps-boss" ]]; then 60 | lecho "Removing old update folder from /root/.cloudvps-boss/cloudvps-boss" 61 | rm -rf /root/.cloudvps-boss/cloudvps-boss 62 | fi 63 | 64 | lecho "Downloading CloudVPS Boss from ${DL_SRV}cloudvps-boss_latest.tar.gz" 65 | get_file "/root/.cloudvps-boss/cloudvps-boss.tar.gz" "${DL_SRV}cloudvps-boss_latest.tar.gz" 66 | if [[ $? -ne 0 ]]; then 67 | lecho "Download of cloudvps-boss failed. Check firewall and network connectivity." 68 | exit 1 69 | fi 70 | 71 | tar -xf cloudvps-boss.tar.gz 72 | if [[ $? -ne 0 ]]; then 73 | lecho "Extraction of cloudvps-boss in /root/.cloudvps-boss failed." 74 | exit 1 75 | fi 76 | popd 77 | 78 | pushd /root/.cloudvps-boss/cloudvps-boss 79 | bash install.sh 80 | -------------------------------------------------------------------------------- /cloudvps-boss/pre-backup.d/15-postgresql_backup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | VERSION="1.9.17" 22 | TITLE="CloudVPS Boss PostgreSQL Backup ${VERSION}" 23 | 24 | if [[ ! -f "/etc/cloudvps-boss/common.sh" ]]; then 25 | lerror "Cannot find /etc/cloudvps-boss/common.sh" 26 | exit 1 27 | fi 28 | source /etc/cloudvps-boss/common.sh 29 | 30 | command -v psql >/dev/null 2>&1 31 | if [[ $? -ne 0 ]]; then 32 | PSQLD=1 33 | fi 34 | 35 | command -v pg_dump >/dev/null 2>&1 36 | if [[ $? -ne 0 ]]; then 37 | PSQLD=1 38 | fi 39 | 40 | command -v pg_dumpall >/dev/null 2>&1 41 | if [[ $? -ne 0 ]]; then 42 | PSQLD=1 43 | fi 44 | 45 | if [[ "${PSQLD}" == 1 ]]; then 46 | log "psql, pg_dump or pg_dumpall not found, not dumping postgresql." 47 | exit 48 | fi 49 | 50 | if [[ ! -d "/var/backups/sql" ]]; then 51 | mkdir -p "/var/backups/sql" 52 | fi 53 | 54 | chmod 777 "/var/backups/sql" 55 | 56 | for COMMAND in "psql" "pg_dump" "pg_dumpall"; do 57 | command_exists "${COMMAND}" 58 | done 59 | 60 | id postgres >/dev/null 2>&1 61 | if [[ $? -ne 0 ]]; then 62 | lerror "System user postgres not found. Not backing up postgresql databases." 63 | exit 1 64 | fi 65 | 66 | cd /tmp 67 | 68 | DATABASES=$(su - postgres -c "psql -l -t | cut -d'|' -f1 | sed -e 's/ //g' -e '/^$/d' | grep -v 'template'") 69 | if [[ $? -ne 0 ]]; then 70 | lerror "Could not retreive postgres databases." 71 | exit 1 72 | fi 73 | 74 | for DB in ${DATABASES}; do 75 | lecho "Dumping database ${DB} to /var/backups/sql/${DB}.psql.gz" 76 | su postgres -c "pg_dump ${DB} | gzip > /var/backups/sql/${DB}.psql.gz" 77 | if [[ $? -ne 0 ]]; then 78 | lerror "Failed dumping postgres database ${DB}" 79 | else 80 | lecho "Dumped ${DB}" 81 | fi 82 | done 83 | 84 | lecho "Dumping pg globals (roles and such, pg_dumpall -g)" 85 | su postgres -c "ionice -c2 nice -n19 pg_dumpall -g | gzip > /var/backups/sql/pg_global_data.sql.gz" 86 | if [[ $? -ne 0 ]]; then 87 | lerror "Failed dumping postgres globals." 88 | else 89 | lecho "Dumped globals" 90 | fi 91 | -------------------------------------------------------------------------------- /cloudvps-boss/post-fail-backup.d/20-failure-notify.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | VERSION="1.9.17" 22 | TITLE="CloudVPS Boss Failure Notify ${VERSION}" 23 | 24 | if [[ ! -f "/etc/cloudvps-boss/common.sh" ]]; then 25 | lerror "Cannot find /etc/cloudvps-boss/common.sh" 26 | exit 1 27 | fi 28 | source /etc/cloudvps-boss/common.sh 29 | 30 | if [[ -f "/etc/cloudvps-boss/status/24h" ]]; then 31 | lecho "24 hour backup file found. Not sending email, removing file." 32 | rm "/etc/cloudvps-boss/status/24h" 33 | exit 0 34 | fi 35 | 36 | for COMMAND in "mail"; do 37 | command_exists "${COMMAND}" 38 | done 39 | 40 | getlogging() { 41 | if [[ -f /var/log/duplicity.log ]]; then 42 | lecho "200 most recent lines in /var/log/duplicity.log:" 43 | tail -n 200 /var/log/duplicity.log 44 | 45 | else 46 | if [[ -f "/var/log/messages" ]]; then 47 | lecho "10 most recent lines with cloudvps-boss ERROR in /var/log/messages:" 48 | grep "cloudvps-boss: ERROR" /var/log/messages | tail -n 10 49 | fi 50 | if [[ -f "/var/log/syslog" ]]; then 51 | lecho "10 most recent lines with cloudvps-boss ERROR in /var/log/syslog:" 52 | grep "cloudvps-boss: ERROR" /var/log/syslog | tail -n 10 53 | fi 54 | fi 55 | 56 | } 57 | 58 | errormail() { 59 | 60 | mail -s "[CLOUDVPS BOSS] ${HOSTNAME}/$(curl -s http://ip.raymii.org): Critical error occurred during the backup!" "${recipient}" <&1 | grep -v -e Warning -e pkg_resources -e oslo -e tar -e attr -e kwargs| sed -n -e '/--------------/,/--------------/ p') 70 | 71 | if [[ $? -ne 0 ]]; then 72 | for line in ${DUPLICITY_OUTPUT}; do 73 | lerror ${line} 74 | done 75 | lerror "CloudVPS Boss Backup to Object Store FAILED!. Please check server ${HOSTNAME}." 76 | lerror "Running post-fail-backup scripts from /etc/cloudvps-boss/post-fail-backup.d/" 77 | for SCRIPT in /etc/cloudvps-boss/post-fail-backup.d/*; do 78 | if [[ ! -d "${SCRIPT}" ]]; then 79 | if [[ -x "${SCRIPT}" ]]; then 80 | "${SCRIPT}" || lerror "Post fail backup script ${SCRIPT} failed." 81 | fi 82 | fi 83 | done 84 | exit 1 85 | fi 86 | 87 | for line in ${DUPLICITY_OUTPUT}; do 88 | lecho "${line}" 89 | done 90 | IFS="${OLD_IFS}" 91 | 92 | echo 93 | lecho "Running post-backup scripts from /etc/cloudvps-boss/post-backup.d/" 94 | for SCRIPT in /etc/cloudvps-boss/post-backup.d/*; do 95 | if [[ ! -d "${SCRIPT}" ]]; then 96 | if [[ -x "${SCRIPT}" ]]; then 97 | "${SCRIPT}" || lerror "Post backup script ${SCRIPT} failed." 98 | fi 99 | fi 100 | done 101 | 102 | echo 103 | lecho "CloudVPS Boss ${VERSION} ended on $(date)." 104 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 1.9.17 4 | 5 | - Update Duplicity to 0.7.17 6 | 7 | ## 1.9.16 8 | 9 | - Add support for Debian 9 10 | 11 | ## 1.9.16 12 | 13 | - Update Duplicity to 0.7.16 14 | 15 | ## 1.9.13 16 | 17 | - Update Duplicity to 0.7.15 18 | - Add manual full backup command 19 | 20 | ## 1.9.12 21 | 22 | - Update Duplicity to 0.7.14 23 | - Remove pygobj dependency 24 | - Don't chmod backup folder 25 | - Remove duplicate log from error email 26 | - Add new object store IP range to firewall 27 | 28 | ## 1.9.11 29 | 30 | - Update Duplicity to 0.7.13 31 | - Remove progress script 32 | 33 | ## 1.9.10 34 | 35 | - Update Duplicity to 0.7.10 36 | - Do not run fail scripts after a failed cleanup. 37 | 38 | ## 1.9.9 39 | 40 | - Update Duplicity to 0.7.09 41 | - Fix typo's 42 | 43 | ## 1.9.8 44 | 45 | - Update Duplicity to 0.7.08 46 | - Remove Arch and Fedora from the installer 47 | - Use pip requirements for install on CentOS 7 & Debian 7 48 | - Fix issue in cleanup script 49 | 50 | 51 | ## 1.9.7 52 | 53 | - Update Duplicity to 0.7.07.1 54 | - Add support for Ubuntu 16.04 55 | - Remove CentOS 5 and Debian 6 from the installer 56 | - Fix issue in cleanup script 57 | 58 | ## 1.9.6 59 | 60 | - Use pip requirements for install on Debian/CentOS 6 61 | - Remove async upload 62 | - Add apt-get update before first package install 63 | 64 | ## 1.9.5 65 | 66 | - Add cleanup command, wrapper around duplicity cleanup. 67 | - Install specific versions of python libraries on Debian 6. Newer won't work with python 2.6. 68 | - Update uninstaller to remove more python libraries. 69 | 70 | ## 1.9.4 71 | 72 | - Fix race condition 73 | 74 | ## 1.9.3 75 | 76 | - Install specific versions of swiftclient, keystoneclient and oslo on CentOS 6. Newer won't work with python 2.6. 77 | 78 | ## 1.9.2 79 | 80 | - Fix install issue with mixed pip/repo packages. 81 | 82 | ## 1.9.1 83 | 84 | - Remove requests workaround for ubuntu 14.04. 85 | - Increase retry time. 86 | - Update Duplicity to 0.7.05 87 | 88 | ## 1.9.0 89 | 90 | - Add package installation on install (curl was unavailable sometimes). 91 | - Add suggestion in error messages on failed install to let user retry the command. 92 | - Retry MySQL credential building when authentication fails, before emailing user. 93 | - Improve lockfile / running backup check. 94 | - Fix large (+250 GB if 25 MB volumes or 2.5 TB if 250 MB volumes) backup sets. Only full backups were made because of https://launchpad.net/+branch/~raymii/duplicity/fix-swiftbackend-max-10000-files-in-list. 95 | - Update some duplicity command options because of deprecation. 96 | - Upgrade Duplicity version to 0.7.04. 97 | 98 | ## 1.8 99 | 100 | - Add send email if MySQL backup fails. 101 | - Change default volsize to 250 MB. 102 | - Upgrade Duplicity version to.0.7.03. 103 | - Upgrade Python to 2.7.10 (on new installs) 104 | 105 | ## 1.7 106 | 107 | - Add check during install/update for already compiled dependencies. 108 | - Add install of base-devel to Arch installer. 109 | - Add more information to failure email script. 110 | - Add lock file checking and handling. 111 | - Add default MySQL data dir to exclude list. 112 | - Change installer download paths to more clean structure. 113 | - Upgrade Duplicity version to 0.7.01. 114 | - Upgrade Duplicity version to 0.7.02. 115 | - Remove overwriting of cronjob during upgrade. 116 | - Fix progress reporting error when df does not support --total. 117 | - Fix rare AUTH_TOKEN not found during install. 118 | - Fix curl dependency check in installer 119 | - Fix curl not found error in credential script 120 | 121 | 122 | 123 | ## 1.6 124 | 125 | - Add view progress command for long running backups. 126 | - Add section for large backups to README. 127 | - Add https download link to README. 128 | - Add bandwidth limit instructions to README. 129 | - Add wget/curl wrapper for remote file downloading. 130 | - Add more clear encryption instructions. 131 | - Add workaround when AUTH_TOKEN during credential setup fails. 132 | - Change default exclude list. 133 | - Change update download link to https. 134 | - Change internal json functions to be more clear. 135 | - Change default retention to 3 months instead of 6. 136 | - Fix installer bug when multiple duplicity source folders exist. 137 | - Fix installer bug when install fails on immutable scripts. 138 | - Remove some non used code. 139 | 140 | ## 1.5 141 | 142 | - Add custom backend support. 143 | - Add custom additional options support. 144 | - Add use of duplicity --name parameter. 145 | - Add documentation on manual cleanup. 146 | - Change auto update to run once a week. 147 | - Fix uninstaller when pip was not named pip. 148 | - Fix some quoting issues. 149 | 150 | ## 1.4 151 | 152 | - Add encryption setup support. 153 | - Add yum clean before update and install 154 | - Add /home/virtfs to exclude list (cpanel) 155 | - Add upload of version number to status folder during updates 156 | - Change more clear documentation on projects and user accounts. 157 | - Fix installation error on CentOS 5 sqlite 158 | - Fix installation error on cpanel + Centos 6. 159 | - Fix no compilation in /tmp (noexec errors) 160 | 161 | ## 1.3 162 | 163 | - Initial public release 164 | -------------------------------------------------------------------------------- /cloudvps-boss/cloudvps-boss.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | VERSION="1.9.17" 22 | TITLE="CloudVPS Boss Backup ${VERSION}" 23 | 24 | if [[ ! -f "/etc/cloudvps-boss/common.sh" ]]; then 25 | lerror "Cannot find /etc/cloudvps-boss/common.sh" 26 | exit 1 27 | fi 28 | source /etc/cloudvps-boss/common.sh 29 | 30 | lecho "${TITLE} started on ${HOSTNAME} at $(date)." 31 | 32 | echo 33 | lecho "Running pre-backup scripts from /etc/cloudvps-boss/pre-backup.d/" 34 | for SCRIPT in /etc/cloudvps-boss/pre-backup.d/*; do 35 | if [[ ! -d "${SCRIPT}" ]]; then 36 | if [[ -x "${SCRIPT}" ]]; then 37 | log "${SCRIPT}" 38 | ionice -c2 nice -n19 "${SCRIPT}" 39 | if [[ $? -ne 0 ]]; then 40 | lerror "Pre backup script ${SCRIPT} failed." 41 | fi 42 | fi 43 | fi 44 | done 45 | 46 | echo 47 | lecho "Create full backup if last full backup is older than: ${FULL_IF_OLDER_THAN} and keep at max ${FULL_TO_KEEP} full backups." 48 | lecho "Starting Duplicity" 49 | 50 | lecho "duplicity --verbosity 9 --log-file /var/log/duplicity.log --volsize ${VOLUME_SIZE} --tempdir=\"${TEMPDIR}\" --file-prefix=\"${HOSTNAME}.\" --name=\"${HOSTNAME}.\" --exclude-device-files --allow-source-mismatch --num-retries 100 --exclude-filelist=/etc/cloudvps-boss/exclude.conf --full-if-older-than=\"${FULL_IF_OLDER_THAN}\" ${ENCRYPTION_OPTIONS} ${CUSTOM_DUPLICITY_OPTIONS} / ${BACKUP_BACKEND}" 51 | 52 | OLD_IFS="${IFS}" 53 | IFS=$'\n' 54 | DUPLICITY_OUTPUT=$(duplicity \ 55 | --verbosity 4 \ 56 | --log-file /var/log/duplicity.log \ 57 | --volsize=${VOLUME_SIZE} \ 58 | --tempdir="${TEMPDIR}" \ 59 | --file-prefix="${HOSTNAME}." \ 60 | --name="${HOSTNAME}." \ 61 | --exclude-device-files \ 62 | --allow-source-mismatch \ 63 | --num-retries 100 \ 64 | --exclude-filelist=/etc/cloudvps-boss/exclude.conf \ 65 | --full-if-older-than="${FULL_IF_OLDER_THAN}" \ 66 | ${ENCRYPTION_OPTIONS} \ 67 | ${CUSTOM_DUPLICITY_OPTIONS} \ 68 | / \ 69 | ${BACKUP_BACKEND} 2>&1 | grep -v -e Warning -e pkg_resources -e oslo -e tar -e attr -e kwargs| sed -n -e '/--------------/,/--------------/ p') 70 | 71 | if [[ $? -ne 0 ]]; then 72 | for line in ${DUPLICITY_OUTPUT}; do 73 | lerror ${line} 74 | done 75 | lerror "CloudVPS Boss Backup to Object Store FAILED!. Please check server ${HOSTNAME}." 76 | lerror "Running post-fail-backup scripts from /etc/cloudvps-boss/post-fail-backup.d/" 77 | for SCRIPT in /etc/cloudvps-boss/post-fail-backup.d/*; do 78 | if [[ ! -d "${SCRIPT}" ]]; then 79 | if [[ -x "${SCRIPT}" ]]; then 80 | "${SCRIPT}" || lerror "Post fail backup script ${SCRIPT} failed." 81 | fi 82 | fi 83 | done 84 | exit 1 85 | fi 86 | 87 | for line in ${DUPLICITY_OUTPUT}; do 88 | lecho "${line}" 89 | done 90 | IFS="${OLD_IFS}" 91 | 92 | echo 93 | lecho "CloudVPS Boss Cleanup ${VERSION} started on $(date). Removing all but ${FULL_TO_KEEP} full backups." 94 | lecho "duplicity --verbosity 9 --log-file /var/log/duplicity.log --file-prefix=\"${HOSTNAME}.\" --name=\"${HOSTNAME}.\" remove-all-but-n-full \"${FULL_TO_KEEP}\" ${ENCRYPTION_OPTIONS} --force ${BACKUP_BACKEND}" 95 | 96 | OLD_IFS="${IFS}" 97 | IFS=$'\n' 98 | DUPLICITY_CLEANUP_OUTPUT=$(duplicity \ 99 | --verbosity 4 \ 100 | --log-file /var/log/duplicity.log \ 101 | --file-prefix="${HOSTNAME}." \ 102 | --name="${HOSTNAME}." \ 103 | remove-all-but-n-full \ 104 | "${FULL_TO_KEEP}" \ 105 | ${ENCRYPTION_OPTIONS} \ 106 | --force \ 107 | ${BACKUP_BACKEND} 2>&1 | grep -v -e Warning -e pkg_resources -e oslo -e attr -e kwargs) 108 | if [[ $? -ne 0 ]]; then 109 | for line in ${DUPLICITY_CLEANUP_OUTPUT}; do 110 | lerror ${line} 111 | done 112 | lerror "CloudVPS Boss Cleanup FAILED!. Please check server ${HOSTNAME}." 113 | fi 114 | 115 | for line in ${DUPLICITY_CLEANUP_OUTPUT}; do 116 | lecho "cleanup: ${line}" 117 | done 118 | IFS="${OLD_IFS}" 119 | 120 | echo 121 | lecho "Running post-backup scripts from /etc/cloudvps-boss/post-backup.d/" 122 | for SCRIPT in /etc/cloudvps-boss/post-backup.d/*; do 123 | if [[ ! -d "${SCRIPT}" ]]; then 124 | if [[ -x "${SCRIPT}" ]]; then 125 | "${SCRIPT}" || lerror "Post backup script ${SCRIPT} failed." 126 | fi 127 | fi 128 | done 129 | 130 | echo 131 | lecho "CloudVPS Boss ${VERSION} ended on $(date)." 132 | -------------------------------------------------------------------------------- /cloudvps-boss/pre-backup.d/11_lockfile_check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | VERSION="1.9.17" 22 | TITLE="CloudVPS Boss Lockfile Check ${VERSION}" 23 | 24 | if [[ ! -f "/etc/cloudvps-boss/common.sh" ]]; then 25 | lerror "Cannot find /etc/cloudvps-boss/common.sh" 26 | exit 1 27 | fi 28 | source /etc/cloudvps-boss/common.sh 29 | 30 | DUPLICITY_LOCKFILE="$(find /root/.cache/duplicity -iname '*.lock' 2>&1 | head -n 1)" 31 | 32 | greater_than_24hour_mail() { 33 | for COMMAND in "mail"; do 34 | command_exists "${COMMAND}" 35 | done 36 | 37 | mail -s "[CLOUDVPS BOSS] ${HOSTNAME}/$(curl -s http://ip.raymii.org): Other backup job still running, more than 24 hours." "${recipient}" <&2 45 | } 46 | 47 | if [[ "${EUID}" -ne 0 ]]; then 48 | lerror "This script must be run as root" 49 | exit 1 50 | fi 51 | 52 | if [[ ! -d "/etc/cloudvps-boss" ]]; then 53 | mkdir -p "/etc/cloudvps-boss" 54 | if [[ $? -ne 0 ]]; then 55 | lerror "Cannot create /etc/cloudvps-boss" 56 | exit 1 57 | fi 58 | fi 59 | 60 | if [[ -f "/etc/boss-backup/auth.conf" ]]; then 61 | lecho "Boss-backup beta auth config found. Copying it." 62 | cp "/etc/boss-backup/auth.conf" "/etc/cloudvps-boss/auth.conf" 63 | fi 64 | 65 | if [[ -f "/etc/swiftbackup/auth.conf" ]]; then 66 | lecho "Swiftbackup beta auth config found. Copying it." 67 | cp "/etc/swiftbackup/auth.conf" "/etc/cloudvps-boss/auth.conf" 68 | fi 69 | 70 | if [[ -f "/etc/cloudvps-boss/auth.conf" ]]; then 71 | lecho "/etc/cloudvps-boss/auth.conf already exists. Not overwriting it" 72 | exit 73 | fi 74 | 75 | if [[ "${1}" == "help" ]]; then 76 | usage 77 | elif [[ -z ${2} || -z ${1} || -z ${3} ]]; then 78 | echo; echo; echo; echo; echo; 79 | read -e -p "Openstack Username (user@example.org): " USERNAME 80 | read -e -s -p "Openstack Password (not shown):" PASSWORD 81 | echo 82 | read -e -p "Openstack Tenant ID: " TENANT_ID 83 | else 84 | USERNAME="${1}" 85 | PASSWORD="${2}" 86 | TENANT_ID="${3}" 87 | fi 88 | 89 | if [[ -z "${USERNAME}" || -z "${PASSWORD}" || -z "${TENANT_ID}" ]]; then 90 | echo 91 | lerror "Need username and password and tenant id." 92 | exit 1 93 | fi 94 | 95 | OS_BASE_AUTH_URL="https://identity.stack.cloudvps.com/v2.0" 96 | OS_AUTH_URL="${OS_BASE_AUTH_URL}/tokens" 97 | OS_TENANTS_URL="${OS_BASE_AUTH_URL}/tenants" 98 | 99 | command -v curl > /dev/null 2>&1 100 | if [[ $? -ne 0 ]]; then 101 | command -v wget > /dev/null 2>&1 102 | if [[ $? -ne 0 ]]; then 103 | lerror "I require curl or wget but none seem to be installed." 104 | lerror "Please install curl or wget" 105 | exit 1 106 | else 107 | AUTH_TOKEN=$(wget -q --header="Content-Type: application/json" --header "Accept: application/json" -O - --post-data='{"auth": {"tenantName": "'${TENANT_ID}'", "passwordCredentials": {"username": "'${USERNAME}'", "password": "'${PASSWORD}'"}}}' "${OS_AUTH_URL}" | grep -o '\"id\": \"[^\"]*\"' | awk -F\" '{print $4}' | sed -n 1p) 108 | fi 109 | else 110 | AUTH_TOKEN=$(curl -s "${OS_AUTH_URL}" -X POST -H "Content-Type: application/json" -H "Accept: application/json" -d '{"auth": {"tenantName": "'${TENANT_ID}'", "passwordCredentials": {"username": "'${USERNAME}'", "password": "'${PASSWORD}'"}}}' | grep -o '\"id\": \"[^\"]*\"' | awk -F\" '{print $4}' | sed -n 1p) 111 | fi 112 | 113 | if [[ -z "${TENANT_ID}" ]]; then 114 | lerror "Tenant ID could not be found. Check username, password or network connectivity." 115 | exit 1 116 | fi 117 | 118 | if [[ -z "${AUTH_TOKEN}" ]]; then 119 | lecho "AUTH_TOKEN empty. Trying again." 120 | sleep 5 121 | command -v curl > /dev/null 2>&1 122 | if [[ $? -ne 0 ]]; then 123 | command -v wget > /dev/null 2>&1 124 | if [[ $? -ne 0 ]]; then 125 | lerror "I require curl or wget but none seem to be installed." 126 | lerror "Please install curl or wget" 127 | exit 1 128 | else 129 | AUTH_TOKEN=$(wget -q --header="Content-Type: application/json" --header "Accept: application/json" -O - --post-data='{"auth": {"tenantName": "'${TENANT_ID}'", "passwordCredentials": {"username": "'${USERNAME}'", "password": "'${PASSWORD}'"}}}' "${OS_AUTH_URL}" | grep -o '\"id\": \"[^\"]*\"' | awk -F\" '{print $4}' | sed -n 1p) 130 | fi 131 | else 132 | AUTH_TOKEN=$(curl -s "${OS_AUTH_URL}" -X POST -H "Content-Type: application/json" -H "Accept: application/json" -d '{"auth": {"tenantName": "'${TENANT_ID}'", "passwordCredentials": {"username": "'${USERNAME}'", "password": "'${PASSWORD}'"}}}' | grep -o '\"id\": \"[^\"]*\"' | awk -F\" '{print $4}' | sed -n 1p) 133 | fi 134 | if [[ -z "${AUTH_TOKEN}" ]]; then 135 | lerror "AUTH_TOKEN could not be found after two tries. Check username, password or network connectivity." 136 | exit 1 137 | fi 138 | fi 139 | 140 | SWIFT_USERNAME="${TENANT_ID}:${USERNAME}" 141 | SWIFT_PASSWORD="${PASSWORD}" 142 | SWIFT_AUTHVERSION="2" 143 | SWIFT_AUTHURL="${OS_BASE_AUTH_URL}" 144 | 145 | if [[ ! -f "/etc/cloudvps-boss/auth.conf" ]]; then 146 | touch "/etc/cloudvps-boss/auth.conf" 147 | chmod 600 "/etc/cloudvps-boss/auth.conf" 148 | cat << EOF > /etc/cloudvps-boss/auth.conf 149 | export SWIFT_USERNAME="${SWIFT_USERNAME}" 150 | export SWIFT_PASSWORD="${SWIFT_PASSWORD}" 151 | export SWIFT_AUTHURL="${SWIFT_AUTHURL}" 152 | export SWIFT_AUTHVERSION="${SWIFT_AUTHVERSION}" 153 | export OS_AUTH_URL="${OS_BASE_AUTH_URL}" 154 | export OS_TENANT_NAME="${TENANT_ID}" 155 | export OS_USERNAME="${USERNAME}" 156 | export OS_PASSWORD="${PASSWORD}" 157 | export OS_TENANT_ID="${TENANT_ID}" 158 | EOF 159 | lecho "Written auth config to /etc/cloudvps-boss/auth.conf." 160 | else 161 | lecho "/etc/cloudvps-boss/auth.conf already exists. Not overwriting it" 162 | fi 163 | 164 | lecho "Username: ${SWIFT_USERNAME}" 165 | lecho "Auth URL: ${SWIFT_AUTHURL}" 166 | lecho "Checking Swift Container for Backups: https://public.objectstore.eu/v1/${TENANT_ID}/cloudvps-boss-backup/" 167 | 168 | curl -s -o /dev/null -X PUT -T "/etc/hosts" --user "${USERNAME}:${PASSWORD}" "https://public.objectstore.eu/v1/${TENANT_ID}/cloudvps-boss-backup/" 169 | if [[ $? == 60 ]]; then 170 | # CentOS 5... 171 | lecho "Curl error Peer certificate cannot be authenticated with known CA certificates." 172 | lecho "This is probably CentOS 5. CentOS 5 is deprecated. Exiting" 173 | exit 1 174 | fi 175 | -------------------------------------------------------------------------------- /cloudvps-boss/common.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | # This file contains common functions used by CloudVPS Boss 22 | 23 | set -o pipefail 24 | 25 | if [[ ${DEBUG} == "1" ]]; then 26 | set -x 27 | fi 28 | 29 | trap ctrl_c INT 30 | 31 | lecho() { 32 | logger -t "cloudvps-boss" -- "$1" 33 | echo "# $1" 34 | } 35 | 36 | log() { 37 | logger -t "cloudvps-boss" -- "$1" 38 | } 39 | 40 | lerror() { 41 | logger -t "cloudvps-boss" -- "ERROR - $1" 42 | echo "$1" 1>&2 43 | } 44 | 45 | PATH=/usr/local/bin:$PATH 46 | PID="$$" 47 | 48 | ## Don't change it to > 80% of minimum available RAM. 49 | VOLUME_SIZE="25" 50 | 51 | # Do not edit. Dirty Workaround for an openstack pbr bug. If not set, everything swift will fail miserably with errors like; Exception: Versioning for this project requires either an sdist tarball, or access to an upstream git repository. Are you sure that git is installed? 52 | # Will be fixed when new pbr version supports the wheel install used by pip. 53 | export PBR_VERSION="0.10.0" 54 | PBR_VERSION="0.10.0" 55 | 56 | if [[ "${EUID}" -ne 0 ]]; then 57 | lerror "This script must be run as root" 58 | exit 1 59 | fi 60 | 61 | if [[ ! -f "/etc/cloudvps-boss/auth.conf" ]]; then 62 | lerror "Cannot find /etc/cloudvps-boss/auth.conf." 63 | exit 1 64 | fi 65 | if [[ ! -f "/etc/cloudvps-boss/backup.conf" ]]; then 66 | lerror "Cannot find /etc/cloudvps-boss/backup.conf." 67 | exit 1 68 | fi 69 | 70 | CONTAINER_NAME="cloudvps-boss-backup" 71 | BACKUP_BACKEND="swift://${CONTAINER_NAME}" 72 | CUSTOM_DUPLICITY_OPTIONS='' 73 | ENCRYPTION_OPTIONS="--no-encryption" 74 | 75 | source /etc/cloudvps-boss/auth.conf 76 | source /etc/cloudvps-boss/backup.conf 77 | 78 | TMP="${TEMPDIR}" 79 | TEMP="${TEMPDIR}" 80 | TMPDIR="${TEMPDIR}" 81 | 82 | if [[ -f "/etc/cloudvps-boss/encryption.conf" ]]; then 83 | source "/etc/cloudvps-boss/encryption.conf" 84 | logger -t "cloudvps-boss" -- "Encryption Configuration Loaded" 85 | fi 86 | 87 | if [[ -f "/etc/cloudvps-boss/custom.conf" ]]; then 88 | source "/etc/cloudvps-boss/custom.conf" 89 | logger -t "cloudvps-boss" -- "Custom Configuration Loaded" 90 | fi 91 | 92 | command_exists() { 93 | command -v "$1" >/dev/null 2>&1 94 | if [[ $? -ne 0 ]]; then 95 | lerror "I require $1 but it's not installed. Aborting." 96 | exit 1 97 | fi 98 | } 99 | 100 | command_exists_non_verbose() { 101 | command -v "$1" >/dev/null 2>&1 102 | if [[ $? -ne 0 ]]; then 103 | exit 1 104 | fi 105 | } 106 | 107 | remove_file() { 108 | if [[ -f "$1" ]]; then 109 | lecho "Removing file $1" 110 | rm "$1" 111 | if [[ "$?" != 0 ]]; then 112 | lerror "Could not remove file $1" 113 | fi 114 | fi 115 | } 116 | 117 | remove_folder() { 118 | if [[ -d "$1" ]]; then 119 | lecho "Removing folder $1" 120 | rm -r "$1" 121 | if [[ "$?" != 0 ]]; then 122 | lerror "Could not remove folder $1" 123 | fi 124 | fi 125 | } 126 | 127 | remove_symlink() { 128 | if [[ -h "$1" ]]; then 129 | lecho "Removing symlink $1" 130 | rm "$1" 131 | if [[ "$?" != 0 ]]; then 132 | lerror "Could not remove symlink $1" 133 | fi 134 | fi 135 | } 136 | 137 | get_hostname() { 138 | HOSTNAME="$(curl -m 3 -s http://169.254.169.254/openstack/latest/meta_data.json | grep -o '\"uuid\": \"[^\"]*\"' | awk -F\" '{print $4}')" 139 | SRV_IP_ADDR="$(curl -q -A CloudVPS-Boss -m 3 -o /dev/null -s https://raymii.org/ >/dev/null 2>/dev/null)" 140 | if [[ -z "${HOSTNAME}" ]]; then 141 | if [[ -f "/var/firstboot/settings" ]]; then 142 | HOSTNAME="$(awk -F= '/hostname/ {print $2}' /var/firstboot/settings)" 143 | else 144 | HOSTNAME="$(uname -n)" 145 | fi 146 | fi 147 | echo "${HOSTNAME}" 148 | } 149 | 150 | ctrl_c() { 151 | lerror "SIGINT received. Exiting." 152 | exit 1 153 | } 154 | 155 | check_choice() { 156 | if [[ -z "${!1}" ]]; then 157 | dialog --title "${TITLE} - Error" --msgbox "${2} must be set. Aborting" 5 50 158 | exit 1 159 | fi 160 | } 161 | 162 | distro_version() { 163 | if [[ -f "/etc/debian_version" ]]; then 164 | NAME="Debian" 165 | VERSION="$(awk -F. '{print $1}' /etc/debian_version)" 166 | fi 167 | if [[ -f "/etc/lsb-release" ]]; then 168 | NAME="$(awk -F= '/DISTRIB_ID/ {print $2}' /etc/lsb-release)" 169 | VERSION="$(awk -F= '/DISTRIB_RELEASE/ {print $2}' /etc/lsb-release)" 170 | fi 171 | if [[ -f "/etc/redhat-release" ]]; then 172 | NAME="$(awk '{ print $1 }' /etc/redhat-release)" 173 | VERSION="$(grep -Eo "[0-9]\.[0-9]" /etc/redhat-release | cut -d . -f 1)" 174 | fi 175 | if [[ "$1" == "name" ]]; then 176 | echo "${NAME}" 177 | fi 178 | if [[ "$1" == "version" ]]; then 179 | echo "${VERSION}" 180 | fi 181 | } 182 | 183 | get_file() { 184 | # Download a file with curl or wget 185 | # get_file SAVE_TO URL 186 | if [[ -n "$1" && -n "$2" ]]; then 187 | command -v curl > /dev/null 2>&1 188 | if [[ $? -ne 0 ]]; then 189 | command -v wget > /dev/null 2>&1 190 | if [[ $? -ne 0 ]]; then 191 | echo "I require curl or wget but none seem to be installed." 192 | echo "Please install curl or wget" 193 | exit 1 194 | else 195 | wget --quiet --output-document "$1" "$2" 196 | fi 197 | else 198 | curl --silent --output "$1" "$2" 199 | fi 200 | else 201 | echo "Not all required parameters received. Usage: get_file SAVE_TO URL" 202 | exit 1 203 | fi 204 | } 205 | 206 | if [[ ! -d "/etc/cloudvps-boss/status/${HOSTNAME}" ]]; then 207 | mkdir -p "/etc/cloudvps-boss/status/${HOSTNAME}" 208 | if [[ $? -ne 0 ]]; then 209 | lerror "Cannot create status folder" 210 | exit 1 211 | fi 212 | fi 213 | 214 | for COMMAND in "awk" "sed" "grep" "tar" "wc" "seq" "gzip" "which" "openssl" "nice" "ionice"; do 215 | command_exists "${COMMAND}" 216 | done 217 | 218 | ACTUAL_HOSTNAME="$(get_hostname)" 219 | logger -t "cloudvps-boss" -- "Configured hostname is ${HOSTNAME}." 220 | logger -t "cloudvps-boss" -- "Actual hostname is ${ACTUAL_HOSTNAME}." 221 | 222 | logger -t "cloudvps-boss" -- "${TITLE} started on ${HOSTNAME} at $(date)." 223 | -------------------------------------------------------------------------------- /cloudvps-boss/pre-backup.d/15-mysql_backup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | 22 | VERSION="1.9.17" 23 | TITLE="CloudVPS Boss MySQL Backup ${VERSION}" 24 | 25 | if [[ ! -f "/etc/cloudvps-boss/common.sh" ]]; then 26 | lerror "Cannot find /etc/cloudvps-boss/common.sh" 27 | exit 1 28 | fi 29 | source /etc/cloudvps-boss/common.sh 30 | 31 | command -v mysql >/dev/null 2>&1 32 | if [[ $? -ne 0 ]]; then 33 | MYSQLD=1 34 | fi 35 | 36 | command -v mysqldump >/dev/null 2>&1 37 | if [[ $? -ne 0 ]]; then 38 | MYSQLD=1 39 | fi 40 | 41 | 42 | failed_mail() { 43 | for COMMAND in "mail"; do 44 | command_exists "${COMMAND}" 45 | done 46 | 47 | mail -s "[CLOUDVPS BOSS] MySQL backup failed on ${HOSTNAME}/$(curl -s http://ip.raymii.org)." "${recipient}" < /root/.my.cnf 145 | [mysqldump] 146 | user=${MYSQL_USER} 147 | password=${MYSQL_PASSWORD} 148 | [mysql] 149 | user=${MYSQL_USER} 150 | password=${MYSQL_PASSWORD} 151 | [client] 152 | user=${MYSQL_USER} 153 | password=${MYSQL_PASSWORD} 154 | EOF 155 | chmod 600 "/root/.my.cnf" 156 | fi 157 | 158 | if [[ -f "/root/.my.cnf" ]]; then 159 | mysql -e 'SHOW FULL PROCESSLIST;' >/dev/null 2>&1 160 | if [[ $? -ne 0 ]]; then 161 | if [[ -f "/etc/cloudvps-boss/mysql_credentials_incorrect" ]]; then 162 | lerror "MySQL credentials incorrect. Please update /root/.my.cnf with the correct credentials. Emailing user." 163 | send_failed_cred_mail 164 | exit 1 165 | else 166 | touch /etc/cloudvps-boss/mysql_credentials_incorrect 167 | lecho "MySQL credentials incorrect. Rebuilding file and retrying." 168 | mv /root/.my.cnf /root/.my.cnf.$$.bak 169 | bash /etc/cloudvps-boss/pre-backup.d/15-mysql_backup.sh 170 | if [[ $? -ne 0 ]]; then 171 | lecho "Rebuild and retry worked." 172 | rm /etc/cloudvps-boss/mysql_credentials_incorrect 173 | exit 0 174 | fi 175 | fi 176 | fi 177 | fi 178 | 179 | 180 | DATABASES="$(mysql -e 'SHOW DATABASES;' | grep -v -e 'Database' -e 'information_schema')" 181 | 182 | if [[ -z "${DATABASES}" ]]; then 183 | lerror "No databases found. Not backing up MySQL" 184 | exit 1 185 | fi 186 | 187 | for DB in ${DATABASES}; do 188 | lecho "Dumping database ${DB} to /var/backups/sql/${DB}.sql.gz" 189 | ionice -c2 nice -n19 mysqldump --opt --lock-all-tables --quick --hex-blob --force "${DB}" | ionice -c2 nice -n19 gzip > "/var/backups/sql/${DB}.sql.gz" 190 | if [[ $? -ne 0 ]]; then 191 | lerror "Database dump ${DB} failed." 192 | else 193 | lecho "Finished dumping database ${DB}"; echo 194 | fi 195 | done 196 | 197 | 198 | -------------------------------------------------------------------------------- /cloudvps-boss/cloudvps-boss-encryption-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | VERSION="1.9.17" 22 | TITLE="CloudVPS Boss Encryption Setup ${VERSION}" 23 | 24 | if [[ ! -f "/etc/cloudvps-boss/common.sh" ]]; then 25 | lerror "Cannot find /etc/cloudvps-boss/common.sh" 26 | exit 1 27 | fi 28 | source /etc/cloudvps-boss/common.sh 29 | 30 | for COMMAND in "gpg"; do 31 | command_exists "${COMMAND}" 32 | done 33 | 34 | if [[ ! -d "/etc/cloudvps-boss/encryption" ]]; then 35 | mkdir -p "/etc/cloudvps-boss/encryption" 36 | chmod 600 "/etc/cloudvps-boss/encryption" 37 | else 38 | if [[ -f "/etc/cloudvps-boss/encryption/setup-finished" ]]; then 39 | lecho "Encryption already set up." 40 | exit 0 41 | fi 42 | fi 43 | 44 | echo "You are going to set up encryption for your backups." 45 | echo "You need to have a good key backup, testing and recovery" 46 | echo "procedure. A lot of information will be shown on the " 47 | echo "screen when the setup is done. Make sure to back that up." 48 | echo "" 49 | echo "Please note that when encryption is set up, support from CloudVPS is not possible." 50 | echo "" 51 | read -e -p "Please type 'I have a good key management procedure and want to set up encryption.': " CONFIRM_ENCRYPTION_SETUP 52 | 53 | if [[ "${CONFIRM_ENCRYPTION_SETUP}" != 'I have a good key management procedure and want to set up encryption.' ]]; then 54 | echo "Input not correct." 55 | exit 1 56 | fi 57 | 58 | 59 | lecho "Placing signing key gpg config" 60 | cat > /etc/cloudvps-boss/encryption/sign-key.gpg.conf < /etc/cloudvps-boss/encryption/enc-key.gpg.conf < /etc/cloudvps-boss/encryption/gen-sign-key.log 2> /etc/cloudvps-boss/encryption/gen-sign-key.error.log 103 | if [[ "$?" -ne 0 ]]; then 104 | lerror "Error generating signing key with gpg" 105 | exit 1 106 | fi 107 | 108 | 109 | lecho "Generating encryption key with gpg." 110 | gpg --batch --gen-key /etc/cloudvps-boss/encryption/enc-key.gpg.conf > /etc/cloudvps-boss/encryption/gen-enc-key.log 2> /etc/cloudvps-boss/encryption/gen-enc-key.error.log 111 | if [[ "$?" -ne 0 ]]; then 112 | lerror "Error generating encryption key with gpg" 113 | exit 1 114 | fi 115 | 116 | touch "/etc/cloudvps-boss/encryption.conf" 117 | chmod 600 "/etc/cloudvps-boss/encryption.conf" 118 | 119 | SIGN_KEY_ID="$(awk '/marked as ultimately trusted/ {print $3}' /etc/cloudvps-boss/encryption/gen-sign-key.error.log)" #| gpg --list-keys --with-colon | awk -F: '/pub/ {print $5}')" 120 | ENC_KEY_ID="$(awk '/marked as ultimately trusted/ {print $3}' /etc/cloudvps-boss/encryption/gen-enc-key.error.log)" # | gpg --list-keys --with-colon | awk -F: '/pub/ {print $5}')" 121 | 122 | echo "# CloudVPS Boss Encryption Config file ${VERSION}" >> /etc/cloudvps-boss/encryption.conf 123 | echo "SIGN_KEY='${SIGN_KEY_ID}'" >> /etc/cloudvps-boss/encryption.conf 124 | echo "ENC_KEY='${ENC_KEY_ID}'" >> /etc/cloudvps-boss/encryption.conf 125 | 126 | echo "ENCRYPTION_OPTIONS=\"--encrypt-key=${ENC_KEY_ID} --sign-key=${SIGN_KEY_ID} \"" >> /etc/cloudvps-boss/encryption.conf 127 | 128 | echo; echo; echo; echo; echo; 129 | read -e -p "Please save the following information. Press ENTER to continue." 130 | 131 | echo "===== BEGIN IMPORTANT RESTORE INFORMATION =====" 132 | echo "Please backup the following in full." 133 | echo "If you don't, or loose it, you will not" 134 | echo "be able to access or restore your backups." 135 | echo "Don't share this with anyone. This information" 136 | echo "gives complete access to this backup." 137 | echo "" 138 | echo "===== SIGNING KEY =====" 139 | echo "GPG Info:" 140 | gpg --list-key "${SIGN_KEY_ID}" 141 | gpg --export-secret-key -a "" 142 | echo "" 143 | echo "===== ENCRYPTION KEY =====" 144 | echo "GPG Info:" 145 | gpg --list-key "${ENC_KEY_ID}" 146 | gpg --export-secret-key -a "" 147 | echo "" 148 | echo "===== GPG Ownertrust =====" 149 | gpg --export-ownertrust 150 | echo "" 151 | echo "===== ENCRYPTION CONFIG =====" 152 | cat /etc/cloudvps-boss/encryption.conf 153 | echo "" 154 | echo "To restore these keys: place the contents" 155 | echo "in 2 files, sign.key and encryption.key." 156 | echo "Execute the following commands as root:" 157 | echo "# gpg --import-key sign.key" 158 | echo "# gpg --import-key encryption.key" 159 | echo "" 160 | echo "Restore the configuration printed above" 161 | echo "to /etc/cloudvps-boss/encryption.conf." 162 | echo "" 163 | echo "Place the ownertrust contents in a file" 164 | echo "named 'ownertrust.gpg' and import it:" 165 | echo "# gpg --import-ownertrust ownertrust.gpg" 166 | echo "" 167 | echo "===== END IMPORTANT RESTORE INFORMATION =====" 168 | echo "" 169 | 170 | lecho "Cleaning up config files." 171 | rm "/etc/cloudvps-boss/encryption/gen-sign-key.error.log" "/etc/cloudvps-boss/encryption/gen-sign-key.log" "/etc/cloudvps-boss/encryption/gen-enc-key.error.log" "/etc/cloudvps-boss/encryption/gen-enc-key.log" "/etc/cloudvps-boss/encryption/enc-key.gpg.conf" "/etc/cloudvps-boss/encryption/sign-key.gpg.conf" 172 | 173 | lecho "Removing secret key used for encryption." 174 | # first get fingerprint, otherwise non-interactive remove fails 175 | ENC_KEY_FINGERPRINT="$(gpg --list-secret-keys --with-colons --fingerprint | grep "${ENC_KEY_ID}" | sed -n 's/^fpr:::::::::\([[:alnum:]]\+\):/\1/p')" 176 | 177 | gpg --batch --yes --delete-secret-key "${ENC_KEY_FINGERPRINT}" 178 | 179 | lecho "Encryption setup done. Please make a backup now, execute 'cloudvps-boss'." 180 | 181 | touch /etc/cloudvps-boss/encryption/setup-finished 182 | if [[ "$?" -ne 0 ]]; then 183 | lerror "Error completing setup, could not touch finish file." 184 | exit 1 185 | fi 186 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | set -o pipefail 22 | 23 | VERSION="1.9.17" 24 | TITLE="CloudVPS Boss Install ${VERSION}" 25 | 26 | if [[ ${DEBUG} == "1" ]]; then 27 | set -x 28 | fi 29 | 30 | lecho() { 31 | logger -t "cloudvps-boss" -- "$1" 32 | echo "# $1" 33 | } 34 | 35 | lerror() { 36 | logger -t "cloudvps-boss" -- "ERROR - $1" 37 | echo "$1" 1>&2 38 | } 39 | 40 | log() { 41 | logger -t "cloudvps-boss" -- "$1" 42 | } 43 | 44 | if [[ "${EUID}" -ne 0 ]]; then 45 | lerror "This script must be run as root." 46 | exit 1 47 | fi 48 | 49 | lecho "${TITLE} started on $(date)." 50 | 51 | usage() { 52 | echo "Usage:" 53 | echo "To install cloudvps-boss with interactive username password question:" 54 | echo "./$0" 55 | echo; echo "To install cloudvps-boss non-interactive:" 56 | echo "./$0 username@domain.tld 'passw0rd' 'tenant id'" 57 | } 58 | 59 | if [[ ! -z "$1" ]]; then 60 | if [[ "$1" == "help" ]]; then 61 | usage 62 | fi 63 | fi 64 | 65 | run_script() { 66 | # check if $1 is a file and execute it with bash. 67 | # log result and exit if script fails. 68 | if [[ -f "$1" ]]; then 69 | log "Starting $1" 70 | bash "$1" "$2" "$3" "$4" 71 | if [[ $? == 0 ]]; then 72 | logger -t "cloudvps-boss" -- "$1 completed." 73 | else 74 | lerror "$1 did not exit cleanly." 75 | exit 1 76 | fi 77 | else 78 | lerror "Cannot find $1." 79 | fi 80 | } 81 | 82 | command_exists() { 83 | # check if command exists and fai otherwise 84 | command -v "$1" >/dev/null 2>&1 85 | if [[ $? -ne 0 ]]; then 86 | lerror "I require $1 but it's not installed. Please install it. I've tried to install it but that failed. Aborting." 87 | exit 1 88 | fi 89 | } 90 | 91 | ## Why all this effort to manually install stuff in three different places you wonder? Because some people refuse to build images with curl included. Why? Who knows... That explains these next 100 or so lines of extra code... 92 | 93 | distro_version() { 94 | if [[ -f "/etc/debian_version" ]]; then 95 | NAME="Debian" 96 | VERSION="$(awk -F. '{print $1}' /etc/debian_version)" 97 | fi 98 | if [[ -f "/etc/lsb-release" ]]; then 99 | NAME="$(awk -F= '/DISTRIB_ID/ {print $2}' /etc/lsb-release)" 100 | VERSION="$(awk -F= '/DISTRIB_RELEASE/ {print $2}' /etc/lsb-release)" 101 | fi 102 | if [[ -f "/etc/redhat-release" ]]; then 103 | NAME="$(awk '{ print $1 }' /etc/redhat-release)" 104 | VERSION="$(grep -Eo "[0-9]\.[0-9]" /etc/redhat-release | cut -d . -f 1)" 105 | fi 106 | if [[ -f "/etc/arch-release" ]]; then 107 | NAME="Arch" 108 | VERSION="Rolling" 109 | fi 110 | if [[ "$1" == "name" ]]; then 111 | echo "${NAME}" 112 | fi 113 | if [[ "$1" == "version" ]]; then 114 | echo "${VERSION}" 115 | fi 116 | } 117 | 118 | get_hostname() { 119 | # Try Openstack Metadata service first 120 | HOSTNAME="$(curl -m 3 -s http://169.254.169.254/openstack/latest/meta_data.json | grep -o '\"uuid\": \"[^\"]*\"' | awk -F\" '{print $4}')" 121 | if [[ -z "${HOSTNAME}" ]]; then 122 | # Otherwise XLS /var/fistboot 123 | if [[ -f "/var/firstboot/settings" ]]; then 124 | HOSTNAME="$(awk -F= '/hostname/ {print $2}' /var/firstboot/settings)" 125 | else 126 | # ask the system if all else fails. 127 | HOSTNAME="$(uname -n)" 128 | fi 129 | fi 130 | 131 | echo "${HOSTNAME}" 132 | } 133 | 134 | install_packages_debian() { 135 | lecho "Installing packages required for installation." 136 | APT_UPDATE="$(apt-get -qq -y --force-yes update > /dev/null 2>&1)" 137 | if [[ "$?" -ne 0 ]]; then 138 | lerror "'apt-get update' failed." 139 | exit 1 140 | fi 141 | for PACKAGE in awk sed grep tar gzip which openssl curl wget screen vim haveged unattended-upgrades; do 142 | /usr/bin/apt-get -qq -y --force-yes -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" install "${PACKAGE}" >/dev/null 2>&1 143 | done 144 | } 145 | 146 | install_packages_centos() { 147 | lecho "Installing packages required for installation." 148 | for PACKAGE in awk sed grep tar gzip which openssl curl wget screen vim haveged yum-cron; do 149 | yum -q -y --disablerepo="*" --disableexcludes=main --enablerepo="base" --enablerepo="updates" install "${PACKAGE}" >/dev/null 2>&1 150 | done 151 | } 152 | 153 | DISTRO_NAME=$(distro_version name) 154 | DISTRO_VERSION=$(distro_version version) 155 | 156 | case "${DISTRO_NAME}" in 157 | "Debian") 158 | lecho "Debian ${DISTRO_VERSION}" 159 | install_packages_debian 160 | ;; 161 | "Ubuntu") 162 | lecho "Ubuntu ${DISTRO_VERSION}" 163 | install_packages_debian 164 | ;; 165 | "CentOS") 166 | lecho "CentOS ${DISTRO_VERSION}" 167 | install_packages_centos 168 | ;; 169 | 170 | *) 171 | lerror "Distro unknown or not supported" 172 | exit 1 173 | ;; 174 | esac 175 | 176 | for COMMAND in "awk" "sed" "grep" "tar" "gzip" "which" "openssl" "curl"; do 177 | command_exists "${COMMAND}" 178 | done 179 | 180 | if [[ -f "/etc/csf/csf.fignore" ]]; then 181 | # Add ourself to the csf file ignore list 182 | # lfd will not scan and mark us suspicious 183 | if [[ ! "$(grep 'cloudvps-boss' /etc/csf/csf.fignore)" ]]; then 184 | lecho "Adding exceptions for lfd." 185 | echo "/tmp/pip-build-root/*" >> /etc/csf/csf.fignore 186 | echo "/tmp/cloudvps-boss/*" >> /etc/csf/csf.fignore 187 | echo "/usr/local/cloudvps-boss/*" >> /etc/csf/csf.fignore 188 | echo "/etc/cloudvps-boss/*" >> /etc/csf/csf.fignore 189 | service lfd restart 2>&1 > /dev/null 190 | fi 191 | if [[ ! "$(grep '89.31.101.64' /etc/csf/csf.allow)" ]]; then 192 | # for regular iptables 193 | #-A OUTPUT -d 89.31.101.64/27 ! -o lo -p tcp -m tcp --dport 443 -j ACCEPT 194 | # Add a rule for the CloudVPS object store to CSF 195 | lecho "Adding exceptions for csf." 196 | csf -a "tcp|out|d=443|d=89.31.101.64/27" "CloudVPS Boss Object Store for backup" 2>&1 > /dev/null 197 | service csf restart 2>&1 > /dev/null 198 | csf -r 2>&1 > /dev/null 199 | fi 200 | if [[ ! "$(grep '31.3.100.121' /etc/csf/csf.allow)" ]]; then 201 | # for regular iptables 202 | #-A OUTPUT -d 31.3.100.121/29 ! -o lo -p tcp -m tcp --dport 443 -j ACCEPT 203 | # Add a rule for the CloudVPS object store to CSF 204 | lecho "Adding exceptions for csf." 205 | csf -a "tcp|out|d=443|d=31.3.100.121/29" "CloudVPS Boss Object Store for backup 2" 2>&1 > /dev/null 206 | service csf restart 2>&1 > /dev/null 207 | csf -r 2>&1 > /dev/null 208 | fi 209 | fi 210 | 211 | if [[ -d "/etc/cloudvps-boss" ]]; then 212 | # check if we already exist, if so, back us up 213 | lecho "Backing up /etc/cloudvps-boss to /var/backups/cloudvps-boss.$$" 214 | if [[ ! -d "/var/backups/cloudvps-boss.$$" ]]; then 215 | mkdir -p "/var/backups/cloudvps-boss.$$" 216 | if [[ "$?" -ne 0 ]]; then 217 | lerror "Cannot create folder /var/backups/cloudvps-boss.$$" 218 | fi 219 | fi 220 | 221 | cp -r "/etc/cloudvps-boss" "/var/backups/cloudvps-boss.$$" 222 | if [[ "$?" -ne 0 ]]; then 223 | lerror "Cannot backup /etc/cloudvps-boss to /var/backups/cloudvps-boss.$$." 224 | exit 1 225 | fi 226 | 227 | if [[ -f "/etc/cron.d/cloudvps-boss" ]]; then 228 | cp -r "/etc/cron.d/cloudvps-boss" "/var/backups/cloudvps-boss.$$/cloudvps-boss.cron.bak" 229 | if [[ "$?" -ne 0 ]]; then 230 | lerror "Cannot backup /etc/cron.d/cloudvps-boss to /var/backups/cloudvps-boss.$$/cloudvps-boss.cron.bak." 231 | exit 1 232 | fi 233 | fi 234 | fi 235 | 236 | for FOLDER in "/etc/cloudvps-boss/pre-backup.d" "/etc/cloudvps-boss/post-backup.d" "/etc/cloudvps-boss/post-fail-backup.d"; do 237 | # create a few required folders 238 | if [[ ! -d "${FOLDER}" ]]; then 239 | mkdir -p "${FOLDER}" 240 | if [[ $? -ne 0 ]]; then 241 | lerror "Cannot create ${FOLDER}" 242 | exit 1 243 | fi 244 | fi 245 | done 246 | 247 | log "Extracting to /etc/cloudvps-boss/" 248 | # we copy all the things manually because 249 | # some users do a chattr +i on stuff they don't want 250 | # overwritten. A cp -r fails and leaves inconsistent state, 251 | # a manual copy only fails the chattr'd things. 252 | for COPY_FILE in "README.md" "LICENSE.md" "CHANGELOG.md"; do 253 | cp "${COPY_FILE}" "/etc/cloudvps-boss/${COPY_FILE}" 254 | if [[ "$?" -ne 0 ]]; then 255 | lerror "Cannot copy ${COPY_FILE} to /etc/cloudvps-boss/${COPY_FILE}." 256 | fi 257 | done 258 | 259 | for COPY_FILE in "cloudvps-boss.cron" "backup.conf" "cloudvps-boss-encryption-setup.sh" "cloudvps-boss-list-current-files.sh" "cloudvps-boss-verify.sh" "cloudvps-boss-cleanup.sh" "cloudvps-boss-restore.sh" "cloudvps-boss.sh" "cloudvps-boss-stats.sh" "cloudvps-boss-manual-full.sh" "cloudvps-boss-update.sh" "common.sh" "exclude.conf" "uninstall.sh"; do 260 | cp "cloudvps-boss/${COPY_FILE}" "/etc/cloudvps-boss/${COPY_FILE}" 261 | if [[ "$?" -ne 0 ]]; then 262 | lerror "Cannot copy cloudvps-boss/${COPY_FILE} to /etc/cloudvps-boss/${COPY_FILE}." 263 | fi 264 | done 265 | 266 | for COPY_FILE in "10-upload-starting-status.sh" "11_lockfile_check.sh" "15-mysql_backup.sh" "15-postgresql_backup.sh"; do 267 | cp "cloudvps-boss/pre-backup.d/${COPY_FILE}" "/etc/cloudvps-boss/pre-backup.d/${COPY_FILE}" 268 | if [[ "$?" -ne 0 ]]; then 269 | lerror "Cannot copy cloudvps-boss/${COPY_FILE} to /etc/cloudvps-boss/pre-backup.d/${COPY_FILE}." 270 | fi 271 | done 272 | 273 | for COPY_FILE in "10-upload-completed-status.sh"; do 274 | cp "cloudvps-boss/post-backup.d/${COPY_FILE}" "/etc/cloudvps-boss/post-backup.d/${COPY_FILE}" 275 | if [[ "$?" -ne 0 ]]; then 276 | lerror "Cannot copy cloudvps-boss/${COPY_FILE} to /etc/cloudvps-boss/post-backup.d/${COPY_FILE}." 277 | fi 278 | done 279 | 280 | for COPY_FILE in "10-upload-fail-status.sh" "20-failure-notify.sh"; do 281 | cp "cloudvps-boss/post-fail-backup.d/${COPY_FILE}" "/etc/cloudvps-boss/post-fail-backup.d/${COPY_FILE}" 282 | if [[ "$?" -ne 0 ]]; then 283 | lerror "Cannot copy cloudvps-boss/${COPY_FILE} to /etc/cloudvps-boss/post-fail-backup.d/${COPY_FILE}." 284 | fi 285 | done 286 | 287 | # See if we are upgrading and if so 288 | # place back the important config files 289 | for CONF_FILE in "auth.conf" "email.conf" "backup.conf" "custom.conf" "exclude.conf" "encryption.conf"; do 290 | if [[ -f "/var/backups/cloudvps-boss.$$/cloudvps-boss/${CONF_FILE}" ]]; then 291 | lecho "Update detected. Placing back file ${CONF_FILE}." 292 | cp -r "/var/backups/cloudvps-boss.$$/cloudvps-boss/${CONF_FILE}" "/etc/cloudvps-boss/${CONF_FILE}" 293 | fi 294 | done 295 | 296 | # complicated loop to run the installer and the credentials script 297 | # with the correct parameters. 298 | for SCRIPT in "install_duplicity.sh" "credentials.sh"; do 299 | if [[ "${SCRIPT}" == "credentials.sh" ]]; then 300 | if [[ ! -z "$1" ]]; then 301 | if [[ ! -z "$2" ]]; then 302 | if [[ ! -z "$3" ]]; then 303 | run_script "${SCRIPT}" "$1" "$2" "$3" 304 | else 305 | run_script "${SCRIPT}" 306 | fi 307 | else 308 | run_script "${SCRIPT}" 309 | fi 310 | else 311 | run_script "${SCRIPT}" 312 | fi 313 | else 314 | run_script "${SCRIPT}" 315 | fi 316 | done 317 | 318 | # hostname is used by Duplicity... 319 | HOSTNAME="$(get_hostname)" 320 | # get and set the hostname in the config. Fails if config is chattr +i. 321 | sed -i "s/replace_me/${HOSTNAME}/g" /etc/cloudvps-boss/backup.conf 322 | 323 | if [[ ! -d "/etc/cron.d" ]]; then 324 | mkdir -p "/etc/cron.d" 325 | fi 326 | 327 | if [[ ! -f "/etc/cron.d/cloudvps-boss" ]]; then 328 | mv "/etc/cloudvps-boss/cloudvps-boss.cron" "/etc/cron.d/cloudvps-boss" 329 | if [[ "$?" -ne 0 ]]; then 330 | lerror "Cannot place cronjob in /etc/cron.d." 331 | fi 332 | # use awk to get a number between 0 and 6 for the hour 333 | RANDH="$(awk 'BEGIN{srand();print int(rand()*(0-6))+6 }')" 334 | RANDM="$(awk 'BEGIN{srand();print int(rand()*(0-59))+59 }')" 335 | # and 0 to 59 for the minutes. Then place it in the cronjob. 336 | sed -i -e "s/RANDH/${RANDH}/g" -e "s/RANDM/${RANDM}/g" /etc/cron.d/cloudvps-boss 337 | # and show the user 338 | lecho "Randomized cronjob time, will run on ${RANDH}:${RANDM}." 339 | fi 340 | 341 | if [[ ! -d "/usr/local/bin" ]]; then 342 | mkdir -p "/usr/local/bin" 343 | fi 344 | 345 | for COMMAND in "cloudvps-boss.sh" "cloudvps-boss-restore.sh" "cloudvps-boss-stats.sh" "cloudvps-boss-update.sh" "cloudvps-boss-list-current-files.sh" "cloudvps-boss-manual-full.sh"; do 346 | log "Creating symlink for /etc/cloudvps-boss/${COMMAND} in /usr/local/bin/${COMMAND%.sh}." 347 | chmod +x "/etc/cloudvps-boss/${COMMAND}" 348 | ln -fs "/etc/cloudvps-boss/${COMMAND}" "/usr/local/bin/${COMMAND%.sh}" 349 | done 350 | 351 | for FILE in "pre-backup.d/15-mysql_backup.sh" "pre-backup.d/15-postgresql_backup.sh" "post-backup.d/10-upload-completed-status.sh" "pre-backup.d/10-upload-starting-status.sh" "pre-backup.d/11_lockfile_check.sh" "post-fail-backup.d/10-upload-fail-status.sh" "post-fail-backup.d/20-failure-notify.sh"; do 352 | # make sure all files are executable 353 | chmod +x "/etc/cloudvps-boss/${FILE}" 354 | done 355 | 356 | for FILE in "cloudvps-boss-progress.sh" "post-backup.d/15-clean-verbose-log.sh"; do 357 | if [[ -f "${FILE}" ]]; then 358 | # progress script is broken. 359 | # log cleaning should be done by logrotate. 360 | rm "/etc/cloudvps-boss/${FILE}" 361 | fi 362 | done 363 | 364 | for FILE in "cloudvps-boss-progress"; do 365 | if [[ -L "${FILE}" ]]; then 366 | # also remove symlink 367 | rm "/usr/local/bin/${FILE}" 368 | fi 369 | done 370 | 371 | if [[ -f "/usr/local/share/man/man1/duplicity.1" ]]; then 372 | # otherwise, /usr/bin/mandb: can't open /usr/local/share/man/man1/duplicity.1: Permission denied 373 | chmod 755 /usr/local/share/man/man1/duplicity.1 374 | fi 375 | 376 | if [[ -f "/usr/local/share/man/man1/rdiffdir.1" ]]; then 377 | chmod 755 /usr/local/share/man/man1/rdiffdir.1 378 | fi 379 | 380 | echo 381 | lecho "If you want to receive email notifications of issues, please install" 382 | lecho "a mailserver and add email addresses, one per line, to the following" 383 | lecho "file: /etc/cloudvps-boss/email.conf" 384 | echo 385 | lecho "If you want to set up encryption, please execute the following command:" 386 | lecho "bash /etc/cloudvps-boss/cloudvps-boss-encryption-setup.sh" 387 | echo 388 | lecho "CloudVPS Boss installation completed." 389 | echo 390 | -------------------------------------------------------------------------------- /cloudvps-boss/cloudvps-boss-restore.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | VERSION="1.9.17" 22 | TITLE="CloudVPS Boss Recovery ${VERSION}" 23 | 24 | if [[ ! -f "/etc/cloudvps-boss/common.sh" ]]; then 25 | lerror "Cannot find /etc/cloudvps-boss/common.sh" 26 | exit 1 27 | fi 28 | source /etc/cloudvps-boss/common.sh 29 | 30 | lecho "${TITLE} started on ${HOSTNAME} at $(date)." 31 | 32 | for COMMAND in "duplicity" "rsync" "gzip" "sed" "grep" "tar" "which" "dialog" "openssl"; do 33 | command_exists "${COMMAND}" 34 | done 35 | 36 | DIALOG_1_MESSAGE="Hello. This is the CloudVPS Boss Restore script. \n\nIt will restore either files or a database from your Object Store Backup. The script will ask the following questions before restoring:\n\n - Hostname \n - Type (File/Database)\n - Path/Database Name \n - Restore from date/time\n\nPlease press return to continue." 37 | 38 | DIALOG_2_MESSAGE="Please enter the hostname of the backup you want to restore. \n\nIt is pre-filled with the currently configured hostname (from backup.conf which was set during installation). If this is not equal to when the backups were made, the restore will fail. \nDo not enter the domain name or a dot at the end. For tank.example.org just enter tank" 39 | 40 | DIALOG_3_MESSAGE="What type of restore do you want to do?" 41 | 42 | DIALOG_4_MESSAGE="Please enter the full path to the file or folder you want to backup. \n\nIf you want to restore the folder '/home/user/test' then enter '/home/user/test/'. The next question will ask you if you want to restore the backup to its original location or to /var/restore." 43 | 44 | DIALOG_11_MESSAGE="Do you wan to restore the folder to its original location or to /var/restore.${PID}/ ?\nIf you restore a file/folder to it's original location it will overwrite *any* files/folders that already exist both there and in the backup with files from the backup. \nIf you restore a folder to it's original location, it does not alter or remove any files that are in the folder but not in the backup.\nIf you restore a folder to /var/restore.${PID}/ you will find the original contents there and you can move it to another location yourself." 45 | 46 | DIALOG_5_MESSAGE="Please enter the MySQL database name.\n\nIf it exists in the backups it will be restored, overwriting any databases with the same name. \nMake sure MySQL superuser credentials are set in /root/.my.cnf, otherwise the restore will fail. See documentation for more info. \nAlso make sure the database server is running." 47 | 48 | DIALOG_6_MESSAGE="Please enter the PostgreSQL database name.\n\nIf it exists in the backups it will be restored, overwriting any databases with the same name. \nMake sure PostgreSQL system user postgres exists, otherwise the restore will fail. See documentation for more info. \nAlso make sure the database server is running." 49 | 50 | DIALOG_7_MESSAGE="Please enter the restore time. \n\nThis can be a relative date like 3D (for three days ago) or 2W (for two weeks ago) - (s=seconds, m=minutes, h=hours, D=days, M=months, W=weeks, Y=years). Also accepted are w3 datetime strings like '2017-06-25T07:00:00+02:00' which means 25'th of June, 2017, 07:00 +2 UTC. YYYY/MM/DD, YYYY-MM-DD, MM/DD/YYYY, or MM-DD-YYYY are also accepted as day formats. Please read the Duplicity Man page, section Time Formats for more info." 51 | 52 | dialog --title "${TITLE} - Introduction" --msgbox "${DIALOG_1_MESSAGE}" 20 65 53 | 54 | dialog --title "${TITLE} - Hostname" --inputbox "${DIALOG_2_MESSAGE}" 0 0 "${HOSTNAME}" 2> "/tmp/${PID}.hostname" 55 | 56 | HOSTNAME=$(cat /tmp/${PID}.hostname && rm /tmp/${PID}.hostname) 57 | 58 | check_choice HOSTNAME "Hostname" 59 | 60 | dialog --title "${TITLE} - Restore Type" --menu "${DIALOG_3_MESSAGE}" 0 0 0 "1" "File/folder restore" "2" "MySQL Database" "3" "PostgreSQL Database" 2> "/tmp/${PID}.restore-type" 61 | 62 | RESTORE_TYPE="$(cat /tmp/${PID}.restore-type && rm /tmp/${PID}.restore-type)" 63 | 64 | check_choice RESTORE_TYPE "Restore type" 65 | 66 | 67 | if [[ "${RESTORE_TYPE}" == 1 ]]; then 68 | 69 | dialog --title "${TITLE} - Original Path" --inputbox "${DIALOG_4_MESSAGE}" 0 0 2> "/tmp/${PID}.original-path" 70 | 71 | ORIGINAL_PATH=$(cat /tmp/${PID}.original-path && rm /tmp/${PID}.original-path) 72 | 73 | check_choice ORIGINAL_PATH "Original path" 74 | 75 | dialog --title "${TITLE} - Restore Path" --menu "${DIALOG_11_MESSAGE}" 0 0 0 "1" "Original location (${ORIGINAL_PATH})" "2" "/var/restore.${PID}/" 2> "/tmp/${PID}.restore-path-choice" 76 | 77 | RESTORE_PATH_CHOICE=$(cat /tmp/${PID}.restore-path-choice && rm /tmp/${PID}.restore-path-choice) 78 | 79 | check_choice RESTORE_PATH_CHOICE "Restore path" 80 | 81 | if [[ "${RESTORE_PATH_CHOICE}" == 1 ]]; then 82 | RESTORE_PATH="${ORIGINAL_PATH}" 83 | fi 84 | if [[ "${RESTORE_PATH_CHOICE}" == 2 ]]; then 85 | RESTORE_PATH="/var/restore.${PID}/" 86 | fi 87 | 88 | dialog --title "${TITLE} - Restore from Date" --inputbox "${DIALOG_7_MESSAGE}" 0 0 2> "/tmp/${PID}.restore-datetime" 89 | 90 | RESTORE_DATETIME=$(cat /tmp/${PID}.restore-datetime && rm /tmp/${PID}.restore-datetime) 91 | 92 | check_choice RESTORE_DATETIME "Restore date/time" 93 | 94 | DIALOG_8_MESSAGE="Restoring file/folder \"$(dirname ${ORIGINAL_PATH})/$(basename ${ORIGINAL_PATH})\" from time ${RESTORE_DATETIME} for host ${HOSTNAME}.\n\nIt will be restored to ${RESTORE_PATH} .\nIf you want to cancel, press CTRL+C now. \nOtherwise, press return to continue." 95 | 96 | dialog --title "${TITLE} - Restore" --msgbox "${DIALOG_8_MESSAGE}" 20 70 97 | 98 | echo; echo; echo; echo; echo; echo; 99 | lecho "Restoring file/folder \"$(dirname ${ORIGINAL_PATH})/$(basename ${ORIGINAL_PATH})\" from time ${RESTORE_DATETIME} for host ${HOSTNAME}. It will be restored to ${RESTORE_PATH}. Date: $(date)." 100 | 101 | RELATIVE_PATH="${ORIGINAL_PATH:1}" 102 | 103 | lecho "duplicity --file-prefix=\"${HOSTNAME}.\" --name=\"${HOSTNAME}.\" ${ENCRYPTION_OPTIONS} ${CUSTOM_DUPLICITY_OPTIONS} --allow-source-mismatch --num-retries 100 --tempdir \"${TEMPDIR}\" -t ${RESTORE_DATETIME} --file-to-restore ${RELATIVE_PATH} ${BACKUP_BACKEND} \"/var/restore.${PID}\"" 104 | 105 | OLD_IFS="${IFS}" 106 | IFS=$'\n' 107 | RESTORE_OUTPUT=$(duplicity \ 108 | --file-prefix="${HOSTNAME}." \ 109 | --name="${HOSTNAME}." \ 110 | ${ENCRYPTION_OPTIONS} \ 111 | ${CUSTOM_DUPLICITY_OPTIONS} \ 112 | --allow-source-mismatch \ 113 | --num-retries 100 \ 114 | --tempdir "${TEMPDIR}" \ 115 | -t ${RESTORE_DATETIME} \ 116 | --file-to-restore ${RELATIVE_PATH} \ 117 | ${BACKUP_BACKEND} "/var/restore.${PID}" 2>&1 | grep -v -e Warning -e pkg_resources -e oslo) 118 | if [[ $? -ne 0 ]]; then 119 | for line in ${RESTORE_OUTPUT}; do 120 | lerror ${line} 121 | done 122 | lerror "Restore FAILED. Please check logging, path name and network connectivity." 123 | exit 1 124 | fi 125 | for line in ${RESTORE_OUTPUT}; do 126 | lecho ${line} 127 | done 128 | IFS="${OLD_IFS}" 129 | 130 | if [[ "${RESTORE_PATH_CHOICE}" == 1 ]]; then 131 | if [[ ! -d "$(dirname ${RESTORE_PATH})" ]]; then 132 | mkdir -p "$(dirname ${RESTORE_PATH})" 133 | fi 134 | lecho "Moving /var/restore.${PID} to ${RESTORE_PATH}" 135 | if [[ -f "/var/restore.${PID}" ]]; then 136 | TYPEA="File" 137 | logger -t "cloudvps-boss" -- "FILE RESTORE TO ORIGINAL PATH" 138 | logger -t "cloudvps-boss" -- "rsync -azq \"/var/restore.${PID}\" \"${RESTORE_PATH}\"" 139 | rsync -azq "/var/restore.${PID}" "${RESTORE_PATH}" 140 | if [[ $? -ne 0 ]]; then 141 | echo "File Restore unsuccessful. Please check logging, path name and network connectivity." 142 | exit 1 143 | fi 144 | fi 145 | if [[ -d "/var/restore.${PID}" ]]; then 146 | TYPEA="Folder" 147 | logger -t "cloudvps-boss" -- "DIRECTORY RESTORE TO ORIGINAL PATH" 148 | logger -t "cloudvps-boss" -- "rsync -azq \"/var/restore.${PID}/\" \"${RESTORE_PATH}\"" 149 | rsync -azq "/var/restore.${PID}/" "${RESTORE_PATH}" 150 | if [[ $? -ne 0 ]]; then 151 | echo "Folder Restore unsuccessful. Please check logging, path name and network connectivity." 152 | exit 1 153 | fi 154 | fi 155 | fi 156 | lecho "${TYPEA} restore successfull." 157 | fi 158 | 159 | if [[ "${RESTORE_TYPE}" == 2 ]]; then 160 | 161 | for COMMAND in "mysql"; do 162 | command_exists "${COMMAND}" 163 | done 164 | 165 | dialog --title "${TITLE} - MySQL Database Name" --inputbox "${DIALOG_5_MESSAGE}" 0 0 2> "/tmp/${PID}.mysql-db-name" 166 | 167 | MYSQL_DB_NAME="$(cat /tmp/${PID}.mysql-db-name && rm /tmp/${PID}.mysql-db-name)" 168 | 169 | check_choice MYSQL_DB_NAME "MySQL Database Name" 170 | 171 | dialog --title "${TITLE} - Restore from Date" --inputbox "${DIALOG_7_MESSAGE}" 0 0 2> "/tmp/${PID}.restore-datetime" 172 | 173 | RESTORE_DATETIME="$(cat /tmp/${PID}.restore-datetime && rm /tmp/${PID}.restore-datetime)" 174 | 175 | check_choice RESTORE_DATETIME "Restore date/time" 176 | 177 | DIALOG_9_MESSAGE="Restoring MySQL database ${MYSQL_DB_NAME} from time ${RESTORE_DATETIME} for host ${HOSTNAME}. \nIt will be restored to /var/restore.${PID} and then placed back in the MySQL server. \nIf you want to cancel, press CTRL+C now. \nOtherwise, press return to continue." 178 | 179 | dialog --title "${TITLE} - Restore" --msgbox "${DIALOG_9_MESSAGE}" 10 70 180 | 181 | echo; echo; echo; echo; echo; echo; 182 | lecho "Restoring MySQL database ${MYSQL_DB_NAME} from time ${RESTORE_DATETIME} for host ${HOSTNAME}. It will be restored to /var/restore.${PID} and then placed back in the MySQL server. Date: $(date)." 183 | 184 | lecho "duplicity --file-prefix=\"${HOSTNAME}.\" --name=\"${HOSTNAME}.\" ${ENCRYPTION_OPTIONS} ${CUSTOM_DUPLICITY_OPTIONS} --allow-source-mismatch --num-retries 100 --tempdir=\"${TEMPDIR}\" -t ${RESTORE_DATETIME} --file-to-restore var/backups/sql/${MYSQL_DB_NAME}.sql.gz ${BACKUP_BACKEND} \"/var/restore.${PID}.gz\"" 185 | 186 | OLD_IFS="${IFS}" 187 | IFS=$'\n' 188 | RESTORE_OUTPUT=$(duplicity \ 189 | --file-prefix="${HOSTNAME}." \ 190 | --name="${HOSTNAME}." \ 191 | --allow-source-mismatch \ 192 | --num-retries 100 \ 193 | ${ENCRYPTION_OPTIONS} \ 194 | ${CUSTOM_DUPLICITY_OPTIONS} \ 195 | --tempdir "${TEMPDIR}" \ 196 | -t ${RESTORE_DATETIME} \ 197 | --file-to-restore var/backups/sql/${MYSQL_DB_NAME}.sql.gz \ 198 | ${BACKUP_BACKEND} "/var/restore.${PID}.gz" 2>&1 | grep -v -e Warning -e pkg_resources -e oslo) 199 | if [[ $? -ne 0 ]]; then 200 | for line in ${RESTORE_OUTPUT}; do 201 | lerror ${line} 202 | done 203 | lerror "Restore FAILED. Please check logging, path name and network connectivity." 204 | exit 1 205 | fi 206 | for line in ${RESTORE_OUTPUT}; do 207 | lecho ${line} 208 | done 209 | IFS="${OLD_IFS}" 210 | 211 | gzip -f -d "/var/restore.${PID}.gz" 212 | if [[ "$?" != 0 ]]; then 213 | echo "Gunzip unsuccessful. Please check logging, path name and network connectivity." 214 | exit 1 215 | fi 216 | 217 | DATABASE_EXISTS=$(mysql -e 'show databases;' | grep "${MYSQL_DB_NAME}") 218 | 219 | if [[ -z "${DATABASE_EXISTS}" ]]; then 220 | lecho "MySQL Database does not exist. Creating db ${MYSQL_DB_NAME}." 221 | CREATE_NON_EXIST_DB=$(mysql -e "create database ${MYSQL_DB_NAME};") 222 | if [[ "$?" != 0 ]]; then 223 | echo "Database Import unsuccessful. Please check logging, path name and network connectivity." 224 | exit 1 225 | fi 226 | fi 227 | 228 | mysql "${MYSQL_DB_NAME}" < "/var/restore.${PID}" 229 | if [[ "$?" != 0 ]]; then 230 | echo "Database Import unsuccessful. Please check logging, path name and network connectivity." 231 | exit 1 232 | fi 233 | lecho "MySQL restore successfull." 234 | 235 | fi 236 | 237 | if [[ "${RESTORE_TYPE}" == 3 ]]; then 238 | for COMMAND in "psql"; do 239 | command_exists "${COMMAND}" 240 | done 241 | 242 | dialog --title "${TITLE} - Postgresql Database Name" --inputbox "${DIALOG_6_MESSAGE}" 0 0 2> "/tmp/${PID}.psql-db-name" 243 | 244 | PSQL_DB_NAME=$(cat /tmp/${PID}.psql-db-name && rm /tmp/${PID}.psql-db-name) 245 | 246 | check_choice PSQL_DB_NAME "PSQL Database Name" 247 | 248 | dialog --title "${TITLE} - Restore from Date" --inputbox "${DIALOG_7_MESSAGE}" 0 0 2> "/tmp/${PID}.restore-datetime" 249 | 250 | RESTORE_DATETIME=$(cat /tmp/${PID}.restore-datetime && rm /tmp/${PID}.restore-datetime) 251 | 252 | check_choice RESTORE_DATETIME "Restore date/time" 253 | 254 | DIALOG_10_MESSAGE="Restoring PostgreSQL database ${PSQL_DB_NAME} from time ${RESTORE_DATETIME} for host ${HOSTNAME}. \nIt will be restored to /var/restore.${PID} and then placed back in the PostgreSQL server. \nIf you want to cancel, press CTRL+C now. \nOtherwise, press return to continue." 255 | 256 | dialog --title "${TITLE} - Restore" --msgbox "${DIALOG_10_MESSAGE}" 10 70 257 | 258 | echo; echo; echo; echo; echo; echo; 259 | lecho "Restoring PostgreSQL database ${PSQL_DB_NAME} from time ${RESTORE_DATETIME} for host ${HOSTNAME}. It will be restored to /var/restore.${PID} and then placed back in the PostgreSQL server. Date: $(date)." 260 | 261 | lecho "duplicity --file-prefix=\"${HOSTNAME}.\" --name=\"${HOSTNAME}.\" ${ENCRYPTION_OPTIONS} ${CUSTOM_DUPLICITY_OPTIONS} --allow-source-mismatch --num-retries 5 --tempdir=\"${TEMPDIR}\" -t ${RESTORE_DATETIME} --file-to-restore var/backups/sql/${PSQL_DB_NAME}.psql.gz ${BACKUP_BACKEND} \"/var/restore.${PID}.gz\"" 262 | 263 | OLD_IFS="${IFS}" 264 | IFS=$'\n' 265 | RESTORE_OUTPUT=$(duplicity \ 266 | --file-prefix="${HOSTNAME}." \ 267 | --name="${HOSTNAME}." \ 268 | ${ENCRYPTION_OPTIONS} \ 269 | ${CUSTOM_DUPLICITY_OPTIONS} \ 270 | --allow-source-mismatch \ 271 | --num-retries 5 \ 272 | --tempdir "${TEMPDIR}" \ 273 | -t ${RESTORE_DATETIME} \ 274 | --file-to-restore var/backups/sql/${PSQL_DB_NAME}.psql.gz \ 275 | ${BACKUP_BACKEND} "/var/restore.${PID}.gz" 2>&1 | grep -v -e Warning -e pkg_resources -e oslo) 276 | if [[ $? -ne 0 ]]; then 277 | for line in ${RESTORE_OUTPUT}; do 278 | lerror ${line} 279 | done 280 | lerror "Restore FAILED. Please check logging, path name and network connectivity." 281 | exit 1 282 | fi 283 | for line in ${RESTORE_OUTPUT}; do 284 | lecho ${line} 285 | done 286 | IFS="${OLD_IFS}" 287 | 288 | gzip -f -d "/var/restore.${PID}.gz" 289 | if [[ $? -ne 0 ]]; then 290 | echo "Gunzip unsuccessful. Please check logging, path name and network connectivity." 291 | exit 1 292 | fi 293 | 294 | chmod 777 /var/restore.${PID} 295 | if [[ -z "$(su postgres -c "psql -l | awk '{print $1}' | grep \"${PSQL_DB_NAME}\"")" ]]; then 296 | lecho "Database not found, creating empty DB" 297 | su postgres -c "createdb ${PSQL_DB_NAME}" 298 | fi 299 | OLD_IFS="${IFS}" 300 | IFS=$'\n' 301 | PG_RESTORE_OUTPUT=$(su postgres -c "psql \"${PSQL_DB_NAME}\" < \"/var/restore.${PID}\"") 302 | if [[ $? -ne 0 ]]; then 303 | for line in ${PG_RESTORE_OUTPUT}; do 304 | lerror ${line} 305 | done 306 | lerror "Database Import FAILED. Please check logging, path name and network connectivity." 307 | exit 1 308 | fi 309 | for line in ${PG_RESTORE_OUTPUT}; do 310 | lecho ${line} 311 | done 312 | IFS="${OLD_IFS}" 313 | lecho "PostgresSQL restore successfull." 314 | 315 | fi 316 | 317 | lecho "${TITLE} ended on ${HOSTNAME} at $(date)." 318 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | CloudVPS Boss (CloudVPS Backup to Object Store Script) - Duplicity 2 | wrapper to back up to OpenStack Swift, Object Store. Copyright (C) 3 | 2016 CloudVPS. 4 | Author: Remy van Elst, https://raymii.org 5 | 6 | This program is free software; you can redistribute it and/or modify it 7 | under the terms of the GNU General Public License as published by the 8 | Free Software Foundation; either version 2 of the License, or (at your 9 | option) any later version. 10 | 11 | This program is distributed in the hope that it will be useful, but 12 | WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | General Public License for more details. 15 | 16 | You should have received a copy of the GNU General Public License along 17 | with this program; if not, write to the Free Software Foundation, Inc., 18 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | 20 | ---------------------------------------------------------------------- 21 | GNU GENERAL PUBLIC LICENSE 22 | 23 | Version 2, June 1991 24 | 25 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 26 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 27 | Everyone is permitted to copy and distribute verbatim copies 28 | of this license document, but changing it is not allowed. 29 | 30 | Preamble 31 | 32 | The licenses for most software are designed to take away your 33 | freedom to share and change it. By contrast, the GNU General Public 34 | License is intended to guarantee your freedom to share and change free 35 | software--to make sure the software is free for all its users. This 36 | General Public License applies to most of the Free Software 37 | Foundation's software and to any other program whose authors commit to 38 | using it. (Some other Free Software Foundation software is covered by 39 | the GNU Lesser General Public License instead.) You can apply it to 40 | your programs, too. 41 | 42 | When we speak of free software, we are referring to freedom, not 43 | price. Our General Public Licenses are designed to make sure that you 44 | have the freedom to distribute copies of free software (and charge for 45 | this service if you wish), that you receive source code or can get it 46 | if you want it, that you can change the software or use pieces of it 47 | in new free programs; and that you know you can do these things. 48 | 49 | To protect your rights, we need to make restrictions that forbid 50 | anyone to deny you these rights or to ask you to surrender the rights. 51 | These restrictions translate to certain responsibilities for you if you 52 | distribute copies of the software, or if you modify it. 53 | 54 | For example, if you distribute copies of such a program, whether 55 | gratis or for a fee, you must give the recipients all the rights that 56 | you have. You must make sure that they, too, receive or can get the 57 | source code. And you must show them these terms so they know their 58 | rights. 59 | 60 | We protect your rights with two steps: (1) copyright the software, and 61 | (2) offer you this license which gives you legal permission to copy, 62 | distribute and/or modify the software. 63 | 64 | Also, for each author's protection and ours, we want to make certain 65 | that everyone understands that there is no warranty for this free 66 | software. If the software is modified by someone else and passed on, we 67 | want its recipients to know that what they have is not the original, so 68 | that any problems introduced by others will not reflect on the original 69 | authors' reputations. 70 | 71 | Finally, any free program is threatened constantly by software 72 | patents. We wish to avoid the danger that redistributors of a free 73 | program will individually obtain patent licenses, in effect making the 74 | program proprietary. To prevent this, we have made it clear that any 75 | patent must be licensed for everyone's free use or not licensed at all. 76 | 77 | The precise terms and conditions for copying, distribution and 78 | modification follow. 79 | 80 | GNU GENERAL PUBLIC LICENSE 81 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 82 | 83 | 0. This License applies to any program or other work which contains 84 | a notice placed by the copyright holder saying it may be distributed 85 | under the terms of this General Public License. The "Program", below, 86 | refers to any such program or work, and a "work based on the Program" 87 | means either the Program or any derivative work under copyright law: 88 | that is to say, a work containing the Program or a portion of it, 89 | either verbatim or with modifications and/or translated into another 90 | language. (Hereinafter, translation is included without limitation in 91 | the term "modification".) Each licensee is addressed as "you". 92 | 93 | Activities other than copying, distribution and modification are not 94 | covered by this License; they are outside its scope. The act of 95 | running the Program is not restricted, and the output from the Program 96 | is covered only if its contents constitute a work based on the 97 | Program (independent of having been made by running the Program). 98 | Whether that is true depends on what the Program does. 99 | 100 | 1. You may copy and distribute verbatim copies of the Program's 101 | source code as you receive it, in any medium, provided that you 102 | conspicuously and appropriately publish on each copy an appropriate 103 | copyright notice and disclaimer of warranty; keep intact all the 104 | notices that refer to this License and to the absence of any warranty; 105 | and give any other recipients of the Program a copy of this License 106 | along with the Program. 107 | 108 | You may charge a fee for the physical act of transferring a copy, and 109 | you may at your option offer warranty protection in exchange for a fee. 110 | 111 | 2. You may modify your copy or copies of the Program or any portion 112 | of it, thus forming a work based on the Program, and copy and 113 | distribute such modifications or work under the terms of Section 1 114 | above, provided that you also meet all of these conditions: 115 | 116 | a) You must cause the modified files to carry prominent notices 117 | stating that you changed the files and the date of any change. 118 | 119 | b) You must cause any work that you distribute or publish, that in 120 | whole or in part contains or is derived from the Program or any 121 | part thereof, to be licensed as a whole at no charge to all third 122 | parties under the terms of this License. 123 | 124 | c) If the modified program normally reads commands interactively 125 | when run, you must cause it, when started running for such 126 | interactive use in the most ordinary way, to print or display an 127 | announcement including an appropriate copyright notice and a 128 | notice that there is no warranty (or else, saying that you provide 129 | a warranty) and that users may redistribute the program under 130 | these conditions, and telling the user how to view a copy of this 131 | License. (Exception: if the Program itself is interactive but 132 | does not normally print such an announcement, your work based on 133 | the Program is not required to print an announcement.) 134 | 135 | These requirements apply to the modified work as a whole. If 136 | identifiable sections of that work are not derived from the Program, 137 | and can be reasonably considered independent and separate works in 138 | themselves, then this License, and its terms, do not apply to those 139 | sections when you distribute them as separate works. But when you 140 | distribute the same sections as part of a whole which is a work based 141 | on the Program, the distribution of the whole must be on the terms of 142 | this License, whose permissions for other licensees extend to the 143 | entire whole, and thus to each and every part regardless of who wrote it. 144 | 145 | Thus, it is not the intent of this section to claim rights or contest 146 | your rights to work written entirely by you; rather, the intent is to 147 | exercise the right to control the distribution of derivative or 148 | collective works based on the Program. 149 | 150 | In addition, mere aggregation of another work not based on the Program 151 | with the Program (or with a work based on the Program) on a volume of 152 | a storage or distribution medium does not bring the other work under 153 | the scope of this License. 154 | 155 | 3. You may copy and distribute the Program (or a work based on it, 156 | under Section 2) in object code or executable form under the terms of 157 | Sections 1 and 2 above provided that you also do one of the following: 158 | 159 | a) Accompany it with the complete corresponding machine-readable 160 | source code, which must be distributed under the terms of Sections 161 | 1 and 2 above on a medium customarily used for software interchange; or, 162 | 163 | b) Accompany it with a written offer, valid for at least three 164 | years, to give any third party, for a charge no more than your 165 | cost of physically performing source distribution, a complete 166 | machine-readable copy of the corresponding source code, to be 167 | distributed under the terms of Sections 1 and 2 above on a medium 168 | customarily used for software interchange; or, 169 | 170 | c) Accompany it with the information you received as to the offer 171 | to distribute corresponding source code. (This alternative is 172 | allowed only for noncommercial distribution and only if you 173 | received the program in object code or executable form with such 174 | an offer, in accord with Subsection b above.) 175 | 176 | The source code for a work means the preferred form of the work for 177 | making modifications to it. For an executable work, complete source 178 | code means all the source code for all modules it contains, plus any 179 | associated interface definition files, plus the scripts used to 180 | control compilation and installation of the executable. However, as a 181 | special exception, the source code distributed need not include 182 | anything that is normally distributed (in either source or binary 183 | form) with the major components (compiler, kernel, and so on) of the 184 | operating system on which the executable runs, unless that component 185 | itself accompanies the executable. 186 | 187 | If distribution of executable or object code is made by offering 188 | access to copy from a designated place, then offering equivalent 189 | access to copy the source code from the same place counts as 190 | distribution of the source code, even though third parties are not 191 | compelled to copy the source along with the object code. 192 | 193 | 4. You may not copy, modify, sublicense, or distribute the Program 194 | except as expressly provided under this License. Any attempt 195 | otherwise to copy, modify, sublicense or distribute the Program is 196 | void, and will automatically terminate your rights under this License. 197 | However, parties who have received copies, or rights, from you under 198 | this License will not have their licenses terminated so long as such 199 | parties remain in full compliance. 200 | 201 | 5. You are not required to accept this License, since you have not 202 | signed it. However, nothing else grants you permission to modify or 203 | distribute the Program or its derivative works. These actions are 204 | prohibited by law if you do not accept this License. Therefore, by 205 | modifying or distributing the Program (or any work based on the 206 | Program), you indicate your acceptance of this License to do so, and 207 | all its terms and conditions for copying, distributing or modifying 208 | the Program or works based on it. 209 | 210 | 6. Each time you redistribute the Program (or any work based on the 211 | Program), the recipient automatically receives a license from the 212 | original licensor to copy, distribute or modify the Program subject to 213 | these terms and conditions. You may not impose any further 214 | restrictions on the recipients' exercise of the rights granted herein. 215 | You are not responsible for enforcing compliance by third parties to 216 | this License. 217 | 218 | 7. If, as a consequence of a court judgment or allegation of patent 219 | infringement or for any other reason (not limited to patent issues), 220 | conditions are imposed on you (whether by court order, agreement or 221 | otherwise) that contradict the conditions of this License, they do not 222 | excuse you from the conditions of this License. If you cannot 223 | distribute so as to satisfy simultaneously your obligations under this 224 | License and any other pertinent obligations, then as a consequence you 225 | may not distribute the Program at all. For example, if a patent 226 | license would not permit royalty-free redistribution of the Program by 227 | all those who receive copies directly or indirectly through you, then 228 | the only way you could satisfy both it and this License would be to 229 | refrain entirely from distribution of the Program. 230 | 231 | If any portion of this section is held invalid or unenforceable under 232 | any particular circumstance, the balance of the section is intended to 233 | apply and the section as a whole is intended to apply in other 234 | circumstances. 235 | 236 | It is not the purpose of this section to induce you to infringe any 237 | patents or other property right claims or to contest validity of any 238 | such claims; this section has the sole purpose of protecting the 239 | integrity of the free software distribution system, which is 240 | implemented by public license practices. Many people have made 241 | generous contributions to the wide range of software distributed 242 | through that system in reliance on consistent application of that 243 | system; it is up to the author/donor to decide if he or she is willing 244 | to distribute software through any other system and a licensee cannot 245 | impose that choice. 246 | 247 | This section is intended to make thoroughly clear what is believed to 248 | be a consequence of the rest of this License. 249 | 250 | 8. If the distribution and/or use of the Program is restricted in 251 | certain countries either by patents or by copyrighted interfaces, the 252 | original copyright holder who places the Program under this License 253 | may add an explicit geographical distribution limitation excluding 254 | those countries, so that distribution is permitted only in or among 255 | countries not thus excluded. In such case, this License incorporates 256 | the limitation as if written in the body of this License. 257 | 258 | 9. The Free Software Foundation may publish revised and/or new versions 259 | of the General Public License from time to time. Such new versions will 260 | be similar in spirit to the present version, but may differ in detail to 261 | address new problems or concerns. 262 | 263 | Each version is given a distinguishing version number. If the Program 264 | specifies a version number of this License which applies to it and "any 265 | later version", you have the option of following the terms and conditions 266 | either of that version or of any later version published by the Free 267 | Software Foundation. If the Program does not specify a version number of 268 | this License, you may choose any version ever published by the Free Software 269 | Foundation. 270 | 271 | 10. If you wish to incorporate parts of the Program into other free 272 | programs whose distribution conditions are different, write to the author 273 | to ask for permission. For software which is copyrighted by the Free 274 | Software Foundation, write to the Free Software Foundation; we sometimes 275 | make exceptions for this. Our decision will be guided by the two goals 276 | of preserving the free status of all derivatives of our free software and 277 | of promoting the sharing and reuse of software generally. 278 | 279 | NO WARRANTY 280 | 281 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 282 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 283 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 284 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 285 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 286 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 287 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 288 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 289 | REPAIR OR CORRECTION. 290 | 291 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 292 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 293 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 294 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 295 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 296 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 297 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 298 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 299 | POSSIBILITY OF SUCH DAMAGES. 300 | 301 | END OF TERMS AND CONDITIONS 302 | 303 | How to Apply These Terms to Your New Programs 304 | 305 | If you develop a new program, and you want it to be of the greatest 306 | possible use to the public, the best way to achieve this is to make it 307 | free software which everyone can redistribute and change under these terms. 308 | 309 | To do so, attach the following notices to the program. It is safest 310 | to attach them to the start of each source file to most effectively 311 | convey the exclusion of warranty; and each file should have at least 312 | the "copyright" line and a pointer to where the full notice is found. 313 | 314 | 315 | Copyright (C) 316 | 317 | This program is free software; you can redistribute it and/or modify 318 | it under the terms of the GNU General Public License as published by 319 | the Free Software Foundation; either version 2 of the License, or 320 | (at your option) any later version. 321 | 322 | This program is distributed in the hope that it will be useful, 323 | but WITHOUT ANY WARRANTY; without even the implied warranty of 324 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 325 | GNU General Public License for more details. 326 | 327 | You should have received a copy of the GNU General Public License along 328 | with this program; if not, write to the Free Software Foundation, Inc., 329 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 330 | 331 | Also add information on how to contact you by electronic and paper mail. 332 | 333 | If the program is interactive, make it output a short notice like this 334 | when it starts in an interactive mode: 335 | 336 | Gnomovision version 69, Copyright (C) year name of author 337 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 338 | This is free software, and you are welcome to redistribute it 339 | under certain conditions; type `show c' for details. 340 | 341 | The hypothetical commands `show w' and `show c' should show the appropriate 342 | parts of the General Public License. Of course, the commands you use may 343 | be called something other than `show w' and `show c'; they could even be 344 | mouse-clicks or menu items--whatever suits your program. 345 | 346 | You should also get your employer (if you work as a programmer) or your 347 | school, if any, to sign a "copyright disclaimer" for the program, if 348 | necessary. Here is a sample; alter the names: 349 | 350 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 351 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 352 | 353 | , 1 April 1989 354 | Ty Coon, President of Vice 355 | 356 | This General Public License does not permit incorporating your program into 357 | proprietary programs. If your program is a subroutine library, you may 358 | consider it more useful to permit linking proprietary applications with the 359 | library. If this is what you want to do, use the GNU Lesser General 360 | Public License instead of this License. 361 | 362 | -------------------------------------------------------------------------------- /install_duplicity.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift 3 | # Copyright (C) 2018 Remy van Elst. (CloudVPS Backup to Object Store Script) 4 | # Author: Remy van Elst, https://raymii.org 5 | # 6 | # This program is free software; you can redistribute it and/or modify it 7 | # under the terms of the GNU General Public License as published by the 8 | # Free Software Foundation; either version 2 of the License, or (at your 9 | # option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, but 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 | # General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License along 17 | # with this program; if not, write to the Free Software Foundation, Inc., 18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 | # 20 | 21 | set -o pipefail 22 | 23 | VERSION="1.9.17" 24 | DUPLICITY_VERSION="0.7.17" 25 | TITLE="CloudVPS Boss Duplicity Installer ${VERSION}" 26 | DL_SRV="https://download.cloudvps.com/cloudvps-boss" # no ending slash (/) 27 | 28 | if [[ ${DEBUG} == "1" ]]; then 29 | set -x 30 | fi 31 | 32 | lecho() { 33 | logger -t "cloudvps-boss" -- "$1" 34 | echo "# $1" 35 | } 36 | 37 | lerror() { 38 | logger -t "cloudvps-boss" -- "ERROR - $1" 39 | echo "$1" 1>&2 40 | } 41 | 42 | log() { 43 | logger -t "cloudvps-boss" -- "$1" 44 | } 45 | 46 | pushd () { 47 | command pushd "$@" > /dev/null 48 | } 49 | 50 | popd () { 51 | command popd "$@" > /dev/null 52 | } 53 | 54 | get_file() { 55 | # Download a file with curl or wget 56 | # get_file SAVE_TO URL 57 | if [[ -n "$1" && -n "$2" ]]; then 58 | command -v curl > /dev/null 2>&1 59 | if [[ $? -ne 0 ]]; then 60 | command -v wget > /dev/null 2>&1 61 | if [[ $? -ne 0 ]]; then 62 | lerror "I require curl or wget but none seem to be installed." 63 | lerror "Please install curl or wget" 64 | exit 1 65 | else 66 | wget --quiet --output-document "$1" "$2" 67 | fi 68 | else 69 | curl --silent --output "$1" "$2" 70 | fi 71 | else 72 | lerror "Not all required parameters received. Usage: get_file SAVE_TO URL" 73 | exit 1 74 | fi 75 | } 76 | 77 | if [[ "${EUID}" -ne 0 ]]; then 78 | lerror "This script must be run as root" 79 | exit 1 80 | fi 81 | 82 | log "${TITLE} started on $(date)." 83 | 84 | if [[ -f "/etc/cloudvps-boss/duplicity_${DUPLICITY_VERSION}_installed" ]]; then 85 | lecho "Duplicity ${DUPLICITY_VERSION} already compiled and installed." 86 | exit 0 87 | fi 88 | 89 | cd /tmp/ 90 | 91 | distro_version() { 92 | if [[ -f "/etc/debian_version" ]]; then 93 | NAME="Debian" 94 | VERSION="$(awk -F. '{print $1}' /etc/debian_version)" 95 | fi 96 | if [[ -f "/etc/lsb-release" ]]; then 97 | NAME="$(awk -F= '/DISTRIB_ID/ {print $2}' /etc/lsb-release)" 98 | VERSION="$(awk -F= '/DISTRIB_RELEASE/ {print $2}' /etc/lsb-release)" 99 | fi 100 | if [[ -f "/etc/redhat-release" ]]; then 101 | NAME="$(awk '{ print $1 }' /etc/redhat-release)" 102 | VERSION="$(grep -Eo "[0-9]\.[0-9]" /etc/redhat-release | cut -d . -f 1)" 103 | fi 104 | if [[ -f "/etc/arch-release" ]]; then 105 | NAME="Arch" 106 | VERSION="Rolling" 107 | fi 108 | if [[ "$1" == "name" ]]; then 109 | echo "${NAME}" 110 | fi 111 | if [[ "$1" == "version" ]]; then 112 | echo "${VERSION}" 113 | fi 114 | } 115 | 116 | install_duplicity_debian_7() { 117 | 118 | APT_UPDATE="$(apt-get -qq -y --force-yes update > /dev/null 2>&1)" 119 | if [[ "$?" -ne 0 ]]; then 120 | lerror "'apt-get update' failed." 121 | exit 1 122 | fi 123 | 124 | APT_INSTALL="$(apt-get -qq -y --force-yes install util-linux wget dialog libc6 python build-essential libxslt1-dev libxml2-dev librsync-dev git-core python-dev python-setuptools librsync1 python-lockfile python-pip 2>&1)" 125 | if [[ "$?" -ne 0 ]]; then 126 | lerror "'apt-get install util-linux wget dialog libc6 python build-essential libxslt1-dev libxml2-dev librsync-dev git-core python-dev python-setuptools librsync1 python-lockfile python-pip' failed." 127 | exit 1 128 | fi 129 | 130 | mkdir -p '/usr/local/cloudvps-boss/source/duplicity' 131 | if [[ "$?" -ne 0 ]]; then 132 | lerror "'mkdir -p /usr/local/cloudvps-boss/source/duplicity' failed." 133 | exit 1 134 | fi 135 | 136 | touch "/usr/local/cloudvps-boss/requirements.txt" 137 | chmod 600 "/usr/local/cloudvps-boss/requirements.txt" 138 | cat << EOF > /usr/local/cloudvps-boss/requirements.txt 139 | Babel==2.2.0 140 | M2Crypto==0.21.1 141 | PyYAML==3.10 142 | argparse==1.2.1 143 | boto==2.25.0 144 | debtcollector==1.2.0 145 | distribute==0.6.10 146 | funcsigs==0.4 147 | futures==3.0.4 148 | iniparse==0.3.1 149 | iso8601==0.1.11 150 | lockfile==0.8 151 | lxml==3.3.5 152 | monotonic==0.6 153 | msgpack-python==0.4.7 154 | netaddr==0.7.18 155 | netifaces==0.10.4 156 | oslo.config==2.7.0 157 | oslo.i18n==2.7.0 158 | oslo.serialization==2.2.0 159 | oslo.utils==2.7.0 160 | paramiko==1.10.1 161 | pbr==3.0.1 162 | prettytable==0.7.2 163 | pycrypto==2.6 164 | pycurl==7.19.0 165 | pyserial==2.5 166 | python-apt==0.8.8.2 167 | python-keystoneclient==2.3.1 168 | python-swiftclient==3.0.0 169 | pytz==2015.7 170 | requests==2.9.1 171 | six==1.10.0 172 | stevedore==1.10.0 173 | urlgrabber==3.9.1 174 | wrapt==1.10.6 175 | wsgiref==0.1.2 176 | yolk==0.4.3 177 | fasteners==0.14.1 178 | keystoneauth1==2.9.0 179 | EOF 180 | 181 | PIP_REQ="$(pip install --upgrade --requirement /usr/local/cloudvps-boss/requirements.txt 2>&1)" 182 | if [[ "$?" -ne 0 ]]; then 183 | lerror "Error installing dependencies with pip: 'pip install --upgrade --requirement /usr/local/cloudvps-boss/requirements.txt' failed." 184 | exit 1 185 | fi 186 | 187 | if [[ ! -d "/usr/local/cloudvps-boss/duplicity" ]]; then 188 | mkdir -p "/usr/local/cloudvps-boss/duplicity" 189 | if [[ $? -ne 0 ]]; then 190 | lerror "'mkdir -p /usr/local/cloudvps-boss/duplicity' failed." 191 | exit 1 192 | fi 193 | fi 194 | 195 | get_file "/usr/local/cloudvps-boss/duplicity.tar.gz" "${DL_SRV}/duplicity/duplicity-${DUPLICITY_VERSION}.tar.gz" 2>&1 196 | if [[ "$?" -ne 0 ]]; then 197 | lerror "'Downloading ${DL_SRV}/duplicity/duplicity-${DUPLICITY_VERSION}.tar.gz to /usr/local/cloudvps-boss/duplicity.tar.gz failed." 198 | exit 1 199 | fi 200 | 201 | tar --extract --file="/usr/local/cloudvps-boss/duplicity.tar.gz" --directory="/usr/local/cloudvps-boss/duplicity/" 2>&1 202 | if [[ "$?" -ne 0 ]]; then 203 | lerror "'tar --extract --file=\"/usr/local/cloudvps-boss/duplicity.tar.gz\" --directory=\"/usr/local/cloudvps-boss/duplicity/\"' failed." 204 | exit 1 205 | fi 206 | 207 | DUPLICITY_SOURCE_FOLDER=$(find /usr/local/cloudvps-boss/duplicity/ -maxdepth 1 -iname 'duplicity-*' -type d | sort -n | tail -n 1) 208 | if [[ "$?" -ne 0 ]]; then 209 | lerror "Source folder in /usr/local/cloudvps-boss/duplicity/ not found." 210 | exit 1 211 | fi 212 | 213 | pushd "${DUPLICITY_SOURCE_FOLDER}" 214 | 215 | SETUP_INSTALL="$(python2 setup.py install 2>&1)" 216 | if [[ "$?" -ne 0 ]]; then 217 | lerror "Error installing Duplicity: 'python2 setup.py install' failed." 218 | exit 1 219 | fi 220 | popd 221 | 222 | } 223 | 224 | install_duplicity_debian_8() { 225 | ## keystoneclient and swiftclient are in the repo's, no more pip. 226 | 227 | #swiftclient is not in precise :( 228 | if [[ "${DISTRO_VERSION}" == "12.04" ]]; then 229 | APT_UPDATE="$(apt-get -qq -y --force-yes update > /dev/null 2>&1)" 230 | if [[ "$?" -ne 0 ]]; then 231 | lerror "'apt-get update' failed." 232 | exit 1 233 | fi 234 | 235 | APT_INSTALL_SOFTPROP="$(apt-get -qq -y --force-yes install software-properties-common python-software-properties > /dev/null 2>&1)" 236 | if [[ "$?" -ne 0 ]]; then 237 | lerror "'apt-get install software-properties-common python-software-properties' failed." 238 | exit 1 239 | fi 240 | 241 | APT_ADD_REPO_STACK="$(add-apt-repository --yes cloud-archive:icehouse > /dev/null 2>&1)" 242 | if [[ "$?" -ne 0 ]]; then 243 | lerror "'add-apt-repository cloud-archive:icehouse' failed." 244 | exit 1 245 | fi 246 | 247 | mkdir -p '/usr/local/cloudvps-boss/source/duplicity' 248 | if [[ "$?" -ne 0 ]]; then 249 | lerror "'mkdir -p /usr/local/cloudvps-boss/source/duplicity' failed." 250 | exit 1 251 | fi 252 | 253 | touch "/usr/local/cloudvps-boss/requirements.txt" 254 | chmod 600 "/usr/local/cloudvps-boss/requirements.txt" 255 | cat << EOF > /usr/local/cloudvps-boss/requirements.txt 256 | fasteners==0.14.1 257 | EOF 258 | 259 | PIP_REQ="$(pip install --upgrade --requirement /usr/local/cloudvps-boss/requirements.txt 2>&1)" 260 | if [[ "$?" -ne 0 ]]; then 261 | lerror "Error installing dependencies with pip: 'pip install --upgrade --requirement /usr/local/cloudvps-boss/requirements.txt' failed." 262 | exit 1 263 | fi 264 | 265 | fi 266 | 267 | #fasteners is not in 14.04 268 | if [[ "${DISTRO_VERSION}" == "14.04" ]]; then 269 | mkdir -p '/usr/local/cloudvps-boss/source/duplicity' 270 | if [[ "$?" -ne 0 ]]; then 271 | lerror "'mkdir -p /usr/local/cloudvps-boss/source/duplicity' failed." 272 | exit 1 273 | fi 274 | 275 | touch "/usr/local/cloudvps-boss/requirements.txt" 276 | chmod 600 "/usr/local/cloudvps-boss/requirements.txt" 277 | cat << EOF > /usr/local/cloudvps-boss/requirements.txt 278 | fasteners==0.14.1 279 | EOF 280 | 281 | PIP_REQ="$(pip install --upgrade --requirement /usr/local/cloudvps-boss/requirements.txt 2>&1)" 282 | if [[ "$?" -ne 0 ]]; then 283 | lerror "Error installing dependencies with pip: 'pip install --upgrade --requirement /usr/local/cloudvps-boss/requirements.txt' failed." 284 | exit 1 285 | fi 286 | fi 287 | 288 | if [[ "${DISTRO_VERSION}" == "16.04" || "${DISTRO_VERSION}" == "15.04" || "${DISTRO_VERSION}" == "15.10" || "${DISTRO_VERSION}" == "16.10" || "${DISTRO_VERSION}" == "17.04" || "${DISTRO_VERSION}" == "17.10" || "${DISTRO_VERSION}" == "9" || "${DISTRO_VERSION}" == "18.04" ]]; then 289 | APT_INSTALL="$(apt-get -qq -y --force-yes install python-fasteners 2>&1)" 290 | if [[ "$?" -ne 0 ]]; then 291 | lerror "'apt-get install python-fasteners' failed." 292 | exit 1 293 | fi 294 | fi 295 | 296 | APT_UPDATE="$(apt-get -qq -y --force-yes update > /dev/null 2>&1)" 297 | if [[ "$?" -ne 0 ]]; then 298 | lerror "'apt-get update' failed." 299 | exit 1 300 | fi 301 | 302 | APT_INSTALL="$(apt-get -qq -y --force-yes install util-linux wget dialog libc6 python build-essential libxslt1-dev libxml2-dev librsync-dev git-core python-dev python-setuptools librsync1 python-lockfile python-pip python-keystoneclient python-swiftclient 2>&1)" 303 | if [[ "$?" -ne 0 ]]; then 304 | lerror "'apt-get install util-linux wget dialog libc6 python build-essential libxslt1-dev libxml2-dev librsync-dev git-core python-dev python-setuptools librsync1 python-lockfile python-pip python-keystoneclient python-swiftclient' failed." 305 | exit 1 306 | fi 307 | 308 | mkdir -p '/usr/local/cloudvps-boss/source/duplicity' 309 | if [[ "$?" -ne 0 ]]; then 310 | lerror "'mkdir -p /usr/local/cloudvps-boss/source/duplicity' failed." 311 | exit 1 312 | fi 313 | 314 | if [[ ! -d "/usr/local/cloudvps-boss/duplicity" ]]; then 315 | mkdir -p "/usr/local/cloudvps-boss/duplicity" 316 | if [[ $? -ne 0 ]]; then 317 | lerror "'mkdir -p /usr/local/cloudvps-boss/duplicity' failed." 318 | exit 1 319 | fi 320 | fi 321 | 322 | get_file "/usr/local/cloudvps-boss/duplicity.tar.gz" "${DL_SRV}/duplicity/duplicity-${DUPLICITY_VERSION}.tar.gz" 2>&1 323 | if [[ "$?" -ne 0 ]]; then 324 | lerror "'Downloading ${DL_SRV}/duplicity/duplicity-${DUPLICITY_VERSION}.tar.gz to /usr/local/cloudvps-boss/duplicity.tar.gz failed." 325 | exit 1 326 | fi 327 | 328 | tar --extract --file="/usr/local/cloudvps-boss/duplicity.tar.gz" --directory="/usr/local/cloudvps-boss/duplicity/" 2>&1 329 | if [[ "$?" -ne 0 ]]; then 330 | lerror "'tar --extract --file=\"/usr/local/cloudvps-boss/duplicity.tar.gz\" --directory=\"/usr/local/cloudvps-boss/duplicity/\"' failed." 331 | exit 1 332 | fi 333 | 334 | DUPLICITY_SOURCE_FOLDER=$(find /usr/local/cloudvps-boss/duplicity/ -maxdepth 1 -iname 'duplicity-*' -type d | sort -n | tail -n 1) 335 | if [[ "$?" -ne 0 ]]; then 336 | lerror "Source folder in /usr/local/cloudvps-boss/duplicity/ not found." 337 | exit 1 338 | fi 339 | 340 | pushd "${DUPLICITY_SOURCE_FOLDER}" 341 | 342 | SETUP_INSTALL="$(python2 setup.py install 2>&1)" 343 | if [[ "$?" -ne 0 ]]; then 344 | lerror "Error installing Duplicity: 'python2 setup.py install' failed." 345 | exit 1 346 | fi 347 | popd 348 | 349 | } 350 | 351 | install_duplicity_centos_6() { 352 | 353 | YUM_CLEAN="$(yum -q -y clean all 2>&1)" 354 | if [[ "$?" -ne 0 ]]; then 355 | lerror "'yum clean all' failed." 356 | exit 1 357 | fi 358 | 359 | YUM_UPDATE="$(yum -q -y update 2>&1)" 360 | if [[ "$?" -ne 0 ]]; then 361 | lerror "'yum update' failed." 362 | exit 1 363 | fi 364 | 365 | YUM_DEVEL_OUTPUT=$(yum -q -y --disablerepo="*" --disableexcludes=main --enablerepo="base" --enablerepo="updates" groupinstall "Development Tools" 2>&1) 366 | if [[ "$?" -ne 0 ]]; then 367 | lerror "'yum --disablerepo=\"*\" --disableexcludes=main --enablerepo=\"base\" --enablerepo=\"updates\" groupinstall \"Development Tools\"' failed." 368 | exit 1 369 | fi 370 | 371 | YUM_INSTALL_BASE_OUTPUT="$(yum -q -y --disablerepo="*" --disableexcludes=main --enablerepo="base" --enablerepo="updates" install rpm-build gettext screen libxslt-python libxslt-devel python-devel python-setuptools python python-lxml wget git dialog 2>&1)" 372 | if [[ "$?" -ne 0 ]]; then 373 | lerror "'yum -q -y --disablerepo=\"*\" --disableexcludes=main --enablerepo=\"base\" --enablerepo=\"updates\" install rpm-build gettext screen libxslt-python libxslt-devel python-devel python-setuptools python python-lxml wget git dialog' failed." 374 | exit 1 375 | fi 376 | 377 | YUM_INSTALL_EPEL2_OUTPUT="$(yum -q -y --disablerepo="*" --disableexcludes=main --enablerepo="epel" install librsync-devel librsync python-lockfile python-pip 2>&1)" 378 | if [[ "$?" -ne 0 ]]; then 379 | lerror "'yum -q -y --disablerepo=\"*\" --disableexcludes=main --enablerepo=\"epel\" install librsync-devel librsync python-lockfile python-pip' failed." 380 | exit 1 381 | fi 382 | 383 | mkdir -p '/usr/local/cloudvps-boss/duplicity' 384 | if [[ "$?" -ne 0 ]]; then 385 | lerror "Error creating Duplicity source folder." 386 | exit 1 387 | fi 388 | 389 | touch "/usr/local/cloudvps-boss/requirements.txt" 390 | chmod 600 "/usr/local/cloudvps-boss/requirements.txt" 391 | cat << EOF > /usr/local/cloudvps-boss/requirements.txt 392 | argparse==1.4.0 393 | babel==2.2.0 394 | debtcollector==1.2.0 395 | distribute==0.6.10 396 | funcsigs==0.4 397 | futures==3.0.4 398 | importlib==1.0.1 399 | iniparse==0.3.1 400 | iso8601==0.1.11 401 | lockfile==0.8 402 | lxml==3.3.5 403 | monotonic==0.6 404 | msgpack-python==0.4.7 405 | netaddr==0.7.18 406 | netifaces==0.10.4 407 | ordereddict==1.2 408 | oslo.config==2.7.0 409 | oslo.i18n==2.7.0 410 | oslo.serialization==2.2.0 411 | oslo.utils==2.7.0 412 | pbr==1.8.1 413 | prettytable==0.7.2 414 | pycurl==7.19.0 415 | pygpgme==0.1 416 | python-keystoneclient==1.7.0 417 | python-swiftclient==2.5.0 418 | pytz==2015.7 419 | requests==2.9.1 420 | six==1.10.0 421 | stevedore==1.10.0 422 | urlgrabber==3.9.1 423 | wrapt==1.10.6 424 | fasteners==0.14.1 425 | EOF 426 | 427 | 428 | PIP_REQ="$(pip install --upgrade --requirement /usr/local/cloudvps-boss/requirements.txt 2>&1)" 429 | if [[ "$?" -ne 0 ]]; then 430 | lerror "Error installing dependencies with pip: 'pip install --upgrade --requirement /usr/local/cloudvps-boss/requirements.txt' failed." 431 | exit 1 432 | fi 433 | 434 | if [[ ! -d "/usr/local/cloudvps-boss/duplicity" ]]; then 435 | mkdir -p "/usr/local/cloudvps-boss/duplicity" 436 | if [[ $? -ne 0 ]]; then 437 | lerror "Cannot create /usr/local/cloudvps-boss/duplicity" 438 | exit 1 439 | fi 440 | fi 441 | 442 | get_file "/usr/local/cloudvps-boss/duplicity.tar.gz" "${DL_SRV}/duplicity/duplicity-${DUPLICITY_VERSION}.tar.gz" 2>&1 443 | if [[ "$?" -ne 0 ]]; then 444 | lerror "downloading ${DL_SRV}/duplicity/duplicity-${DUPLICITY_VERSION}.tar.gz to /usr/local/cloudvps-boss/duplicity.tar.gz failed" 445 | exit 1 446 | fi 447 | 448 | tar --extract --file="/usr/local/cloudvps-boss/duplicity.tar.gz" --directory="/usr/local/cloudvps-boss/duplicity/" 2>&1 449 | if [[ "$?" -ne 0 ]]; then 450 | lerror "Error extracting Duplicity source" 451 | exit 1 452 | fi 453 | 454 | DUPLICITY_SOURCE_FOLDER="$(find /usr/local/cloudvps-boss/duplicity/ -maxdepth 1 -iname 'duplicity-*' -type d | sort -n | tail -n 1)" 455 | if [[ "$?" -ne 0 ]]; then 456 | lerror "Error locating Duplicity source folder." 457 | exit 1 458 | fi 459 | 460 | pushd "${DUPLICITY_SOURCE_FOLDER}" 461 | 462 | SETUP_INSTALL="$(python2 setup.py install 2>&1)" 463 | if [[ "$?" -ne 0 ]]; then 464 | lerror "Error installing Duplicity." 465 | exit 1 466 | fi 467 | 468 | popd 469 | 470 | if [[ ! -f "/etc/profile.d/duplicity" ]]; then 471 | touch "/etc/profile.d/duplicity" 472 | echo "PATH=/usr/local/bin:$PATH" > "/etc/profile.d/duplicity" 473 | fi 474 | } 475 | 476 | install_duplicity_centos_7() { 477 | 478 | YUM_CLEAN="$(yum -q -y clean all 2>&1)" 479 | if [[ "$?" -ne 0 ]]; then 480 | lerror "Error cleaning yum." 481 | exit 1 482 | fi 483 | 484 | YUM_UPDATE="$(yum -q -y update 2>&1)" 485 | if [[ "$?" -ne 0 ]]; then 486 | lerror "Error updating packages." 487 | exit 1 488 | fi 489 | 490 | YUM_DEVEL_OUTPUT=$(yum -q -y --disablerepo="*" --disableexcludes=main --enablerepo="base" --enablerepo="updates" groupinstall "Development Tools" 2>&1) 491 | if [[ "$?" -ne 0 ]]; then 492 | lerror "Error installing development tools. Make sure base repository is enabled." 493 | exit 1 494 | fi 495 | 496 | YUM_INSTALL_BASE_OUTPUT="$(yum -q -y --disablerepo="*" --disableexcludes=main --enablerepo="base" --enablerepo="updates" install screen rpm-build gettext libxslt-python libxslt-devel python-lxml wget git dialog python-devel 2>&1)" 497 | if [[ "$?" -ne 0 ]]; then 498 | lerror "Error installing required packages from base." 499 | exit 1 500 | fi 501 | 502 | YUM_INSTALL_EPEL_OUTPUT="$(yum -q -y --disablerepo="*" --disableexcludes=main --enablerepo="epel" install librsync-devel librsync python-lockfile python-pip 2>&1)" 503 | if [[ "$?" -ne 0 ]]; then 504 | lerror "Error installing required packages from epel." 505 | exit 1 506 | fi 507 | 508 | if [[ ! -d "/usr/local/cloudvps-boss/duplicity" ]]; then 509 | mkdir -p "/usr/local/cloudvps-boss/duplicity" 510 | if [[ $? -ne 0 ]]; then 511 | lerror "Cannot create /usr/local/cloudvps-boss/duplicity" 512 | exit 1 513 | fi 514 | fi 515 | 516 | touch "/usr/local/cloudvps-boss/requirements.txt" 517 | chmod 600 "/usr/local/cloudvps-boss/requirements.txt" 518 | cat << EOF > /usr/local/cloudvps-boss/requirements.txt 519 | Babel==2.3.4 520 | backports.ssl-match-hostname==3.4.0.2 521 | configobj==4.7.2 522 | debtcollector==1.5.0 523 | decorator==3.4.0 524 | funcsigs==1.0.2 525 | futures==3.0.5 526 | iniparse==0.4 527 | IPy==0.75 528 | iso8601==0.1.11 529 | keystoneauth1==2.8.0 530 | lockfile==0.9.1 531 | lxml==3.2.1 532 | monotonic==1.1 533 | msgpack-python==0.4.7 534 | netaddr==0.7.18 535 | netifaces==0.10.4 536 | oslo.config==3.12.0 537 | oslo.i18n==3.7.0 538 | oslo.serialization==2.10.0 539 | oslo.utils==3.14.0 540 | pbr==1.10.0 541 | prettytable==0.7.2 542 | pycurl==7.19.0 543 | pygpgme==0.3 544 | pyliblzma==0.5.3 545 | pyparsing==1.5.6 546 | python-keystoneclient==3.1.0 547 | python-swiftclient==3.0.0 548 | pytz==2016.4 549 | pyudev==0.15 550 | pyxattr==0.5.1 551 | requests==2.10.0 552 | rfc3986==0.3.1 553 | six==1.10.0 554 | stevedore==1.15.0 555 | urlgrabber==3.10 556 | wrapt==1.10.8 557 | fasteners==0.14.1 558 | EOF 559 | 560 | PIP_REQ="$(pip install --upgrade --requirement /usr/local/cloudvps-boss/requirements.txt 2>&1)" 561 | if [[ "$?" -ne 0 ]]; then 562 | lerror "Error installing dependencies with pip: 'pip install --upgrade --requirement /usr/local/cloudvps-boss/requirements.txt' failed." 563 | exit 1 564 | fi 565 | 566 | get_file "/usr/local/cloudvps-boss/duplicity.tar.gz" "${DL_SRV}/duplicity/duplicity-${DUPLICITY_VERSION}.tar.gz" 2>&1 567 | if [[ "$?" -ne 0 ]]; then 568 | lerror "downloading ${DL_SRV}/duplicity/duplicity-${DUPLICITY_VERSION}.tar.gz to /usr/local/cloudvps-boss/duplicity.tar.gz failed." 569 | exit 1 570 | fi 571 | 572 | tar --extract --file="/usr/local/cloudvps-boss/duplicity.tar.gz" --directory="/usr/local/cloudvps-boss/duplicity/" 2>&1 573 | if [[ "$?" -ne 0 ]]; then 574 | lerror "Error extracting Duplicity source" 575 | exit 1 576 | fi 577 | 578 | DUPLICITY_SOURCE_FOLDER="$(find /usr/local/cloudvps-boss/duplicity/ -maxdepth 1 -iname 'duplicity-*' -type d | sort -n | tail -n 1)" 579 | if [[ "$?" -ne 0 ]]; then 580 | lerror "Error locating Duplicity source folder." 581 | exit 1 582 | fi 583 | 584 | pushd "${DUPLICITY_SOURCE_FOLDER}" 585 | 586 | SETUP_INSTALL="$(python2 setup.py install 2>&1)" 587 | if [[ "$?" -ne 0 ]]; then 588 | lerror "Error installing Duplicity." 589 | exit 1 590 | fi 591 | 592 | popd 593 | 594 | if [[ ! -f "/etc/profile.d/duplicity" ]]; then 595 | touch "/etc/profile.d/duplicity" 596 | echo "PATH=/usr/local/bin:$PATH" > "/etc/profile.d/duplicity" 597 | fi 598 | } 599 | 600 | install_epel_7() { 601 | EPEL_INSTALL="$(rpm -Uvh ${DL_SRV}/utils/epel-release-7.noarch.rpm 2>&1 > /dev/null)" 602 | } 603 | 604 | install_epel_6() { 605 | EPEL_INSTALL="$(rpm -Uvh ${DL_SRV}/utils/epel-release-6.noarch.rpm 2>&1 > /dev/null)" 606 | } 607 | 608 | if [[ ! -f "$(which python2)" ]]; then 609 | if [[ ! -L "$(which python2)" ]]; then 610 | if [[ $(python -V 2>&1 | cut -d " " -f 2 | cut -d . -f 1) == "2" ]]; then 611 | ln -s "$(which python)" "$(which python)2" 612 | fi 613 | fi 614 | fi 615 | 616 | DISTRO_NAME=$(distro_version name) 617 | DISTRO_VERSION=$(distro_version version) 618 | 619 | lecho "Compiling Duplicity ${DUPLICITY_VERSION}." 620 | 621 | case "${DISTRO_NAME}" in 622 | 623 | "Debian") 624 | case "${DISTRO_VERSION}" in 625 | 9) 626 | lecho "Debian 9" 627 | install_duplicity_debian_8 628 | ;; 629 | 8) 630 | lecho "Debian 8" 631 | install_duplicity_debian_8 632 | ;; 633 | 7) 634 | lecho "Debian 7" 635 | install_duplicity_debian_7 636 | ;; 637 | *) 638 | lerror "Distro unknown or not supported" 639 | exit 1 640 | ;; 641 | esac 642 | ;; 643 | "Ubuntu") 644 | # ubuntu has keystoneclient and swiftclient in the repo's. 645 | lecho "Ubuntu ${DISTRO_VERSION}" 646 | install_duplicity_debian_8 647 | ;; 648 | "CentOS") 649 | case "${DISTRO_VERSION}" in 650 | 7) 651 | lecho "CentOS 7" 652 | install_epel_7 653 | install_duplicity_centos_7 654 | exit 655 | ;; 656 | 6) 657 | lecho "CentOS 6" 658 | install_epel_6 659 | install_duplicity_centos_6 660 | ;; 661 | *) 662 | lerror "Distro unknown or not supported" 663 | exit 1 664 | ;; 665 | esac 666 | ;; 667 | 668 | *) 669 | lerror "Distro unknown or not supported" 670 | exit 1 671 | ;; 672 | esac 673 | 674 | touch "/etc/cloudvps-boss/duplicity_${DUPLICITY_VERSION}_installed" 675 | 676 | lecho "${TITLE} ended on $(date)." 677 | 678 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ### Introduction 4 | 5 | CloudVPS Boss is a backup suite for Linux that utilises Duplicity to backup and restore files, folders and databases from and to Openstack Swift with optional encryption, in our case the CloudVPS Object Store. 6 | 7 | Backing up to this medium means that your data is stored three times over two different datacenters for a very low price. The name means “CloudVPS Backup to Object Store Script” and was chosen by our marketing team (but the development team obviously loves it). 8 | 9 | The code is open source so you can use it on non CloudVPS object stores in the unlikely case that you would want to do that. 10 | 11 | ### Table of Contents 12 | 13 |
    14 |
  1. Introduction
    2. 15 |
  2. Requirements
    3. 16 |
  3. Installation
    4. 17 | 22 |
  4. Configuration
    5. 23 |
      24 |
    • Custom Configuration
      • 25 |
    • Bandwidth limits
      • 26 |
    27 |
  5. Backup
    6. 28 | 32 |
  6. Restore
    7. 33 | 42 |
  7. Information and Statistics
    8. 43 | 47 |
  8. Notifications and Monitoring
    9. 48 |
  9. Uninstall
    10. 49 |
  10. Duplicity Encryption
    11. 50 | 53 |
  11. Other Edge Cases
    12. 54 | 65 |
  12. License
    13. 66 |
67 | 68 | 69 | 70 | ### Requirements 71 | 72 | CloudVPS Boss is tested and should work on and with the following distributions and software: 73 | 74 | - Ubuntu 12.04 and up 75 | - Debian 7, 8 76 | - CentOS 6, 7 77 | 78 | - MySQL/MariaDB 5.0 - 5.6 79 | - PostgreSQL 9.0 - 9.4 80 | 81 | The following distributions are NOT supported. That means, if something breaks, you're on your own: 82 | 83 | - Ubuntu < 12.04 84 | - Debian 6 85 | - CentOS 5 86 | 87 | 88 | 89 | ### Installation 90 | 91 | #### Object Store user 92 | 93 | Make sure you have an Openstack or Object Store project. If not, please contact sales or order one via the website. 94 | 95 | The software needs a tenant ID, username and password of an Object Store account. A tenant ID is the same as the Project ID, not the user ID. 96 | 97 | If you back up multiple servers to the same container, they are able to access all the backups. If you do not want that machines can see each others backup you need to create a seperate project and user in that project per server you want to backup. Or, you can setup encryption of the backups, see the "Encryption" section for more info on that. 98 | 99 | Please do not use your main account, but create a new user under your projects: 100 | 101 | - Login to the interface over at https://interface.cloudvps.com. 102 | 103 | - Navigate to "User Manager" --> "User Manager" 104 | 105 | - Click the "+" and fill in the required data. Note down the password, you will need it later. The image below is an example, please use a meaningfull username like `backups@mycompany.nl`. 106 | 107 | ![](http://i.imgur.com/rxVnCd6.png) 108 | 109 | - Add the user to your openstack project. Click the "+" and select the correct project. Give it only "swiftoperator" permissions. 110 | . 111 | 112 | - Note down the Project ID. This is the same as the Tenant ID. 113 | 114 | This account will be used to place the backup in the object store. If you want, you can create an account for every server you want to backup with different credentials. Make sure the username is descriptive. 115 | 116 | The object store account can already contain data. The backups are written to the container `cloudvps-boss-backup`. This container is allowed to have pre-existing data, but it is not recommended. The container name cannot be changed. Multiple servers can be backed up to the container, as long as the hostname configured is different. 117 | 118 | #### Installation 119 | 120 | You need to run the installer as the `root` user, or as a user with an `EUID` equal to 0. All the cloudvps-boss tools will check if they are executed as root and will fail otherwise. 121 | 122 | Download the .tar.gz file containing the latest version of the script: 123 | 124 | wget -O cloudvps-boss-latest.tar.gz https://download.cloudvps.com/cloudvps-boss/cloudvps-boss_latest.tar.gz 125 | 126 | Extract it: 127 | 128 | tar -xf cloudvps-boss_latest.tar.gz 129 | 130 | Go in the folder and start the installer: 131 | 132 | cd cloudvps-boss 133 | bash install.sh 134 | 135 | If you want to start the installation unattended, give the username and password as options: 136 | 137 | cd cloudvps-boss 138 | bash install.sh "username@example.org "p@ssw0rd" "tenant id" 139 | 140 | A one liner, if you need to do multiple installations: 141 | 142 | rm -rf /tmp/cloudvps*; wget -O /tmp/cloudvps-boss_latest.tar.gz https://download.cloudvps.com/cloudvps-boss/cloudvps-boss_latest.tar.gz && cd /tmp && tar -xf cloudvps-boss_latest.tar.gz && cd /tmp/cloudvps-boss && bash install.sh 143 | 144 | The installer will ask you for your Openstack Username, Password and Tenant ID. Your pasword will not be shown when you type it, you will also not see stars or some other masking. 145 | 146 | A version of Duplicity with support for OpenStack Swift will be installed, as well as the required build tools, and the necessary openstack clients. 147 | 148 | If the csf or lfd firewall applications are detected, IP ranges related to the CloudVPS Object Store will be configured as exception rules. 149 | 150 | After that it will create or update the container `cloudvps-boss-backup`. It will also place the required config files and such. 151 | 152 | Example install output: 153 | 154 | # CloudVPS Boss Install 1.6 started on Sun Aug 24 08:56:14 CEST 2014. 155 | # Adding exceptions for lfd. 156 | # Adding exceptions for csf. 157 | # Backing up /etc/cloudvps-boss to /var/backups/cloudvps-boss.17118 158 | # Starting install_duplicity.sh 159 | # CloudVPS Boss Duplicity Install 0.6.24 started on Sun Jul 27 10:35:04 CEST 2014. 160 | # Ubuntu 12.04 161 | /usr/local/cloudvps-boss/duplicity/duplicity-0.6.24 /tmp 162 | /tmp 163 | # Duplicity installation script completed 164 | # install_duplicity.sh completed. 165 | # Starting credentials.sh 166 | # CloudVPS Boss Credentials Config 1.6 started on Sun Aug 24 08:56:14 CEST 2014. 167 | 168 | Openstack Username (user@example.org): user@example.org 169 | Openstack Password (not shown): 170 | Openstack Tenant ID : 22[...]8a 171 | 172 | # Username: Example Tenant:user@example.org 173 | # Auth URL: https://identity.stack.cloudvps.com/v2.0 174 | # Written auth config to /etc/cloudvps-boss/auth.conf. 175 | # Checking Swift Container for Backups: https://public.objectstore.eu/v1/22[...]8a/cloudvps-boss-backup/ 176 | # credentials.sh completed. 177 | # Randomized cronjob time, will run on 4:36. 178 | # Creating symlink for backup script in /usr/local/bin/cloudvps-boss. 179 | # Creating symlink for recovery script in /usr/local/bin/cloudvps-boss-restore. 180 | # Creating symlink for status script in /usr/local/bin/cloudvps-boss-stats. 181 | # Creating symlink for update script in /usr/local/bin/cloudvps-boss-update. 182 | 183 | # CloudVPS Boss installation completed. 184 | 185 | 186 | If you upgrade the script or by accident run the installer again it will backup all the config it can find and then place new ones. It will backup and place back the auth, backup and exclude config. Updating can be done at any time with the following command: 187 | 188 | cloudvps-boss-update 189 | 190 | The cronjob placed will auto update the script every month on the first day of the month. 191 | 192 | 193 | 194 | #### Manual Installation 195 | 196 | If you are on a unsupported distribution you need to to the following things to get cloudvps-boss working. 197 | 198 | - Install python 2.7. 199 | - Install python-swiftclient and python-keystoneclient. 200 | - Make sure wget, curl, openssl, awk, which, sed and grep are available. 201 | - If you want encryption, also install gpg. 202 | - Install/Compile Duplicity, a version higher than 0.6.22. 203 | - Create the folder /etc/cloudvps-boss 204 | - Create the pre-backup.d, post-fail-backup.d and post-backup.d folders in there 205 | - Place the required files there, like so: 206 | 207 | /etc/cloudvps-boss/ 208 | |-- auth.conf 209 | |-- backup.conf 210 | |-- cloudvps-boss-encryption-setup.sh 211 | |-- cloudvps-boss-restore.sh 212 | |-- cloudvps-boss-stats.sh 213 | |-- cloudvps-boss-list-current-files.sh 214 | |-- cloudvps-boss-update.sh 215 | |-- cloudvps-boss.sh 216 | |-- common.sh 217 | |-- exclude.conf 218 | |-- post-backup.d 219 | | `-- 10-upload-completed-status.sh 220 | |-- post-fail-backup.d 221 | | |-- 10-upload-fail-status.sh 222 | | `-- 20-failure-notify.sh 223 | |-- pre-backup.d 224 | | |-- 10-upload-starting-status.sh 225 | | |-- 15-mysql_backup.sh 226 | | `-- 15-postgresql_backup.sh 227 | `-- uninstall.sh 228 | 229 | 230 | - Create symlinks in /usr/local/bin to cloudvps-boss, cloudvps-boss-restore, cloudvps-boss-update, cloudvps-list-current-files and cloudvps-boss-stats. 231 | - Place the cloudvps-boss.cron file in /etc/cron.d/ 232 | - Configure auth.conf and backup.conf. (See below, section Configuration) 233 | - Optional, set up encryption. 234 | 235 | 236 | 237 | #### Notes on CentOS 238 | 239 | On CentOS 5 we also compile and install a newer version of sqlite, setuptools and Python version 2.7. We don't touch the system Python 2.4, so Yum still works. 240 | 241 | Another step we take on this distro, if needed, is updating the system root CA certicicates file, making a safety copy of the original. This is necessary for the swift client, to allow it to verify the CloudVPS SSL credentials correctly. 242 | 243 | For all CentOS releases, we disable all the repositories except `base` during installation. The script installs the `epel` repository and uses this for the python packages. If you run RHEL, Scientific Linux or Oracle Linux, make sure you either enable or rename the repository which contains base packages to `base` otherwise the installation will fail. 244 | 245 | 246 | 247 | #### Notes Arch Linux 248 | 249 | Arch Linux is not supported, but we do provide a (as of August 2014) working installer. It however fails if you have gcc-multilib installed. Install the required packages manually then. 250 | 251 | 252 | 253 | ### Configuration 254 | 255 | The install script creates a default configuration for the backup process, with these settings in it: 256 | 257 | - Create a full backup every 14 days 258 | - Keep at max 6 full backups 259 | - Create an incremental backup on the other days, once a day. 260 | 261 | This gives you 3 months of retention. The settings can be found in `/etc/cloudvps-boss/backup.conf`: 262 | 263 | # Server hostname. Will be replaced during install. Must be unique among backuped servers. 264 | HOSTNAME="replace_me" 265 | # Create a full backup if the last is older than 14 days. 266 | FULL_IF_OLDER_THAN="14D" 267 | # Keep at max 6 full backups. 268 | FULL_TO_KEEP="6" 269 | # Only change this if your tmp folder is to small. See README 270 | TEMPDIR="/tmp" 271 | 272 | If you want more or less retention, change the variables. For one week of retention, create a full backup if the other full is older than 7 days and keep at max 1 full backup. If you want a month of retention, create a full backup if the other full is older than 7 days and keep at max 4 full backups. 273 | 274 | You can increase the amount of days before a full backup is created. Duplicity will create more incremental backups then. 275 | 276 | Note that full backups have a positive effect on restore speeds, but a negative effect on the amount of storage needed. Increase the relative number of full backups for better speed, or lower it for less storage. 277 | 278 | To get 6 months of retention with just one full backup, create a full backup if the other full is older than 6 months (6M) and keep at max 1 full backup. 279 | 280 | The auth.conf file has the credentials needed for Swift and Openstack authentication: 281 | 282 | SWIFT_USERNAME="TENANT-ID:USERNAME" 283 | SWIFT_PASSWORD="super_secure_password" 284 | SWIFT_AUTHURL="https://identity.stack.cloudvps.com/v2.0" 285 | SWIFT_AUTHVERSION="2" 286 | OS_AUTH_URL="https://identity.stack.cloudvps.com/v2.0" 287 | OS_TENANT_NAME="TENANT-ID" 288 | OS_USERNAME="USERNAME" 289 | OS_PASSWORD="super_secure_password" 290 | OS_TENANT_ID="TENANT-ID" 291 | 292 | Make sure the tenant has access to swift, the swiftoperator role. Compute is not required. The tenant id is the same as the project id. 293 | 294 | If a backup runs longer than 24 hours, and if an email addresses and mailing is configured, you will receive an email about this. If backups regularly take longer than 24 hours you might need to investigate why that is the case. 295 | 296 | 297 | 298 | #### Custom Configuration 299 | 300 | If you want to use a custom backend or extra Duplicity options you can configure those in the following file: 301 | 302 | /etc/cloudvps-boss/custom.conf 303 | 304 | This file is not there by default, you should create it. 305 | 306 | The following options are supported: 307 | 308 | BACKUP_BACKEND='' 309 | CONTAINER_NAME='' 310 | CUSTOM_DUPLICITY_OPTIONS='' 311 | 312 | If you want to use a container other than the default 'cloudvps-boss-backup' you can change CONTAINER_NAME. 313 | 314 | You can specify a custom backend here, this will be used instead of our Object Store. You can also specify custom options which will be added to every Duplicity command. For example, to use an FTP server: 315 | 316 | BACKUP_BACKEND='ftp://backup@example.com:Passw0rd@example.com' 317 | 318 | For FTP you need to install `ncftp`. 319 | 320 | To use an SSH/SFTP server with a custom SSH key: 321 | 322 | CUSTOM_DUPLICITY_OPTIONS='--ssh-options=-oIdentityFile=/root/.ssh/backup' 323 | BACKUP_BACKEND='ssh://backup@example.com:22/backups/' 324 | 325 | For SFTP/SSH you need to install `paramiko`. 326 | 327 | Please read the [Duplicity man page](duplicity.nongnu.org/duplicity.1.html) to see which backends and custom options are supported. 328 | 329 | Do note that we do not support custom configuration and / or custom backends. 330 | 331 | 332 | 333 | ### Bandwidth Limit 334 | 335 | By default CloudVPS Boss will use all available bandwidth. If you have a 100 Mbit uplink it will use all the bandwidth it can get. This might be an issue on some systems. You can use the traffic shaping program `trickle` to limit the bandwidth of the backup. 336 | 337 | Install it for your distribution. Debian/Ubuntu: 338 | 339 | apt-get install trickle 340 | 341 | CentOS/RHEL (via EPEL): 342 | 343 | yum install trickle 344 | 345 | Add trickle to the cronjob: 346 | 347 | vim /etc/cron.d/cloudvps-boss 348 | 349 | Change the default command: 350 | 351 | # CloudVPS Boss Backup 352 | 52 6 * * * root /usr/local/bin/cloudvps-boss 353 | 354 | To the following: 355 | 356 | # CloudVPS Boss Backup 357 | 52 6 * * * root /usr/bin/trickle -s -u 250 /usr/local/bin/cloudvps-boss 358 | 359 | The `-u 250` is the max upload speed in KB/s. The above command line will effectifly limit CloudVPS Boss to max 250KB/s upload (2 Mbit/s). 360 | 361 | Change the cronjob file to be immutable. Otherwise the change will be overwritten next update: 362 | 363 | chattr +i /etc/cron.d/cloudvps-boss 364 | 365 | 366 | 367 | 368 | ### Backup 369 | 370 | By default a cronjob is placed in `/etc/cron.d/cloudvps-boss` with a randomized time between 00:00 and 06:59 (AM) to run the backup once a day. You can edit that file to run a backup more or less often. If you want to back up every hour you should change the cronjob to something like this: 371 | 372 | 1 * * * * root /usr/local/bin/cloudvps-boss 373 | 374 | Do note that the cron file will be overwritten during the update once a month. If you don't want that, change the permissions to `immutable`: 375 | 376 | chattr +i /etc/cron.d/cloudvps-boss 377 | 378 | Before editing it, remove the immutable flag using the same command, just with `-i` instead of `+i`. 379 | 380 | You can also just copy the cronjob file to another file in `/etc/cron.d`. 381 | 382 | If you want to run a backup manually, use the command `cloudvps-boss`. An example backup for a server with CentOS and MySQL looks like this: 383 | 384 | # CloudVPS Boss Backup 1.6 started on hostname at Sun Aug 24 09:08:16 CEST 2014. 385 | 386 | # Running pre-backup scripts from /etc/cloudvps-boss/pre-backup.d/ 387 | 388 | # Dumping database mysql to /var/backups/sql/mysql.sql.gz 389 | -- Warning: Skipping the data of table mysql.event. Specify the --events option explicitly. 390 | # Finished dumping database mysql 391 | 392 | # Dumping database ttrss to /var/backups/sql/ttrss.sql.gz 393 | # Finished dumping database ttrss 394 | 395 | # Create full backup if last full backup is older than: 14D and keep at max 6 full backups. 396 | # Starting Duplicity 397 | # ionice -c2 nice -n19 duplicity --asynchronous-upload --volsize 25 --tempdir="" --file-prefix="hostname." --exclude-device-files --exclude-globbing-filelist=/etc/cloudvps-boss/exclude.conf --full-if-older-than="14D" --no-encryption / swift://cloudvps-boss-backup 398 | # Reading globbing filelist /etc/cloudvps-boss/exclude.conf 399 | # Local and Remote metadata are synchronized, no sync needed. 400 | # Last full backup date: Mon Aug 11 06:54:04 2014 401 | # --------------[ Backup Statistics ]-------------- 402 | # StartTime 1408864103.07 (Sun Aug 24 09:08:23 2014) 403 | # EndTime 1408864132.40 (Sun Aug 24 09:08:52 2014) 404 | # ElapsedTime 29.33 (29.33 seconds) 405 | # SourceFiles 79330 406 | # SourceFileSize 1406281961 (1.31 GB) 407 | # NewFiles 801 408 | # NewFileSize 4827146 (4.60 MB) 409 | # DeletedFiles 237 410 | # ChangedFiles 124 411 | # ChangedFileSize 94015369 (89.7 MB) 412 | # ChangedDeltaSize 0 (0 bytes) 413 | # DeltaEntries 1162 414 | # RawDeltaSize 29241572 (27.9 MB) 415 | # TotalDestinationSizeChange 28268116 (27.0 MB) 416 | # Errors 0 417 | # ------------------------------------------------- 418 | 419 | # CloudVPS Boss Cleanup 1.6 started on Sun Aug 24 09:08:53 CEST 2014. Removing all but 6 full backups. 420 | # ionice -c2 nice -n19 duplicity --file-prefix="hostname." remove-all-but-n-full "6" --force --no-encryption swift://cloudvps-boss-backup 421 | # Local and Remote metadata are synchronized, no sync needed. 422 | # Last full backup date: Mon Aug 11 06:54:04 2014 423 | # No old backup sets found, nothing deleted. 424 | 425 | # Running post-backup scripts from /etc/cloudvps-boss/post-backup.d/ 426 | 427 | # CloudVPS Boss Completed Status Upload 1.6 ended on hostname at Sun Aug 24 09:08:58 CEST 2014. 428 | 429 | # CloudVPS Boss 1.6 ended on Sun Aug 24 09:08:58 CEST 2014. 430 | 431 | 432 | If the cleanup action also runs you will have extra output like this: 433 | 434 | # CloudVPS Boss Cleanup 1.6 started on Tue Jul 15 03:31:12 CEST 2014. Removing all but 6 full backups. 435 | # Local and Remote metadata are synchronized, no sync needed. 436 | # Last full backup date: Tue Jul 15 03:21:08 2014 437 | # Deleting backup chain at time: 438 | # Sun Jul 13 03:21:10 2014 439 | # Deleting complete signature chain Sun Jul 13 03:21:10 2014 440 | # Deleting complete signature chain Sun Jul 13 03:21:10 2014 441 | # Deleting complete backup chain Sun Jul 13 03:21:10 2014 442 | 443 | 444 | 445 | #### A note on Database Dumps 446 | 447 | CloudVPS Boss detects if there is a MySQL/MariaDB or a PostgreSQL server running or available during every backup. If you first install the backup scripts and configure a database server later on, you don't have to configure anything extra (sometimes the credentials if they are not detected). The MySQL/MariaDB backup script detects the following control panels or other credential files: 448 | 449 | - DirectAdmin 450 | - Plesk 451 | - OpenPanel 452 | - cPanel/WHM 453 | - Debian sys-maint user 454 | 455 | If you run a bare CentOS/Fedora install and want to have automated MySQL/MariaDB backups you should edit the file `/root/.my.cnf` with the correct username and password (with all privileges): 456 | 457 | [client] 458 | user=none 459 | password=none 460 | 461 | Make sure you do not remove the `/root/.my.cnf` file. This file is what gives CloudVPS Boss access to the database. If you change the root password, also change it here. 462 | 463 | For Postgres it detects if there is a `postgres` system user and that peer authentication is allowed for the domain socket and localhost. If so, it will use that to dump all databases (one by one using `pg_dump`). It also dumps the global database with the roles and such using `pg_dumpall -g`. If you have a special PostgreSQL setup which does not use peer authentication or the `postgres` system user you should manually update the script placed in `/etc/cloudvps-boss/pre-backup.d/`. 464 | 465 | 466 | ##### Disabling database backups 467 | 468 | If you want to disable database dumps, remove the mysql and postgresql scripts in `/etc/cloudvps-boss/pre-backup.d/`. These files are placed back during an update. If you don't want that, make the files empty and set the `immutable` flag on them. Scroll above a bit to find out how to set the flag on a file in the Cron section. 469 | 470 | 471 | 472 | #### View backup progress 473 | 474 | Introduced in version 1.6 is a simple backup progress reporting tool. Duplicity does not natively support reporting progress, the option that is in there does not work. By using a combination of verbose logging and some math we can get the backup progress. 475 | 476 | The command works best for a long running full backup. 477 | 478 | The command is: `cloudvps-boss-progress`. 479 | Example output: 480 | 481 | # Diskspace used: 1580589 MiB / 1543 GiB 482 | # Duplicity volume: 41 483 | # Amount uploaded: 41984 MiB / 41 GiB. 484 | # Duplicity running for 68 minutes / 1 hour. 485 | # Speed: 82 Mbps / 10 MBps 486 | # 487 | # Full backup only: 488 | # Estimated 2564 minutes / 42 hours to complete 489 | # 2.66% done, 1502 GiB left to upload of 1543 GiB. 490 | [##________________________________________________] 491 | 492 | Do note that the reported data is an estimate, reality may differ. 493 | 494 | Duplicity needs to finish at least one volume, otherwise the progress report fails: 495 | 496 | Error reading current volume. Please let duplicity finish at least 1 volume. 497 | 498 | If you are having trouble with a large backup set or a long running one, please read the section on very large backups. 499 | 500 | 501 | 502 | ### Recovery 503 | 504 | ![](http://i.imgur.com/LHA9lc1.png) 505 | 506 | To recover a file or a database, use the `cloudvps-boss-restore` command. It is installed together with the script during installation. It is a dialog based script which makes it easy to restore a file, folder or database. 507 | 508 | The script consists of te following steps: 509 | 510 | - Hostname 511 | 512 | ![](http://i.imgur.com/TInVrFz.png) 513 | 514 | This is pre-filled with the current configured hostname (from backup.conf which was set during installation). If this is not equal to when the backups were made, the restore will fail. To restore from another machine, enter that hostname here. 515 | 516 | - Type 517 | 518 | ![](http://i.imgur.com/cbTVXgt.png) 519 | 520 | Choose File/folder, MySQL database or PostgreSQL database. 521 | 522 | - File/Folder/DB Name 523 | 524 | ![](http://i.imgur.com/Pqc7ms6.png) 525 | 526 | Either the full path to the file or folder you want to restore, or the name of the database you want to restore. 527 | 528 | If you want to restore the folder '/home/user/test' then enter '/home/user/test/'. If you select file, there will follow another question asking if you want to restore the file to its original location or to /var/restore. If you restore the file to its original location it will overwrite *any* files/folders that already exist both there and in the backup with files from the backup. If you restore a folder, it does not alter or remove any files that are in the folder but not in the backup. 529 | 530 | If the database exists in the backups it will be restored, overwriting any databases with the same name. Make sure MySQL superuser credentials are set in /root/.my.cnf, otherwise the restore will fail. Make sure PostgreSQL system user `postgres` exists, otherwise the restore will fail. Also make sure the database server is running. 531 | 532 | - Restore Location 533 | 534 | ![](http://i.imgur.com/oGutcVr.png) 535 | 536 | If you want to restore a file/folder this question will ask you if you want to restore it to its original location or restore it to /var/restore/. If you restore the file to its original location it will overwrite *any* files/folders that already exist both there and in the backup with files from the backup. If you restore a folder, it does not alter or remove any files that are in the folder but not in the backup. If you restore to /var/restore you can move the files/folders to another location yourself. 537 | 538 | If you want to restore a database to another database you need to do that manually, by restoring the database dump (from /var/backups/sql) and them importing that to a new database with the respective tools. 539 | 540 | - Restore date/time 541 | 542 | ![](http://i.imgur.com/ItK20EP.png) 543 | 544 | Provide the time/date from when you want to restore a backup. This can be a relative date like 3D (for three days ago) or 2W (for two weeks ago) 545 | 546 | You can use the `cloudvps-boss-stats` command to see which backup sets, dates and times are available. See below for more info on `cloudvps-boss-stats`. 547 | 548 | `(s=seconds, m=minutes, h=hours, D=days, M=months, W=weeks, Y=years).` 549 | 550 | Also accepted are w3 datetime strings like '2014-06-25T07:00:00+02:00' which means 25'th of June, 2014, 07:00 +2 UTC. YYYY/MM/DD, YYYY-MM-DD, MM/DD/YYYY, or MM-DD-YYYY are also accepted as day formats. 551 | 552 | Please read the Duplicity Man page, section Time Formats for more info. 553 | 554 | - Confirmation 555 | 556 | ![](http://i.imgur.com/K4tNVO5.png) 557 | 558 | Provides an overview of what we are going to do and the last option to cancel it. Press Enter to start the restore. It will take a while, there is no progress output. 559 | 560 | 561 | 562 | #### Restoring Encrypted Backups 563 | 564 | If you want to restore an encrypted backup you need to place back the encryption key you saved when setting up the encryption. Place the contents of the `===== ENCRYPTION KEY =====` part in a file named `encryption.key`. 565 | 566 | Import this file in the GPG keyring, as root: 567 | 568 | gpg --import-key encryption.key 569 | 570 | Now you can use the normal `cloudvps-boss-restore` command. 571 | 572 | If you are asked for a GPG passphrase, just press return since there is no passphrase on the key. 573 | 574 | If you get an error message like below: 575 | 576 | GPGError: GPG Failed, see log below: 577 | ===== Begin GnuPG log ===== 578 | gpg: encrypted with 2048-bit ELG-E key, ID BE56538D, created 2014-10-31 579 | "CloudVPS Boss Backup Encryption Key for hostname " 580 | gpg: decryption failed: secret key not available 581 | ===== End GnuPG log ===== 582 | Restore FAILED. Please check logging, path name and network connectivity. 583 | 584 | Then you have not placed the (correct) private encryption key back. 585 | 586 | Don't forget to remove the key when you are done restoring a file. Get the GPG key ID for the `Encryption` key: 587 | 588 | gpg --list-secret-keys 589 | 590 | Output: 591 | 592 | /root/.gnupg/secring.gpg 593 | ------------------------ 594 | sec 4096R/1A49C5EF 2014-10-31 595 | uid CloudVPS Boss Backup Signing Key for hostname 596 | ssb 2048g/1E0D9239 2014-10-31 597 | 598 | sec 4096R/7EDFFCF4 2014-10-31 599 | uid CloudVPS Boss Backup Encryption Key for hostname 600 | ssb 2048g/BE56538D 2014-10-31 601 | 602 | Copy the key ID for the `Encryption` key and delete it: 603 | 604 | gpg --delete-secret-key 7EDFFCF4 605 | 606 | Output: 607 | 608 | gpg (GnuPG) 1.4.11; Copyright (C) 2010 Free Software Foundation, Inc. 609 | This is free software: you are free to change and redistribute it. 610 | There is NO WARRANTY, to the extent permitted by law. 611 | 612 | sec 4096R/7EDFFCF4 2014-10-31 CloudVPS Boss Backup Encryption Key for hostname 613 | 614 | Delete this key from the keyring? (y/N) y 615 | This is a secret key! - really delete? (y/N) y 616 | 617 | Press `y` and return twice to confirm the deletion. 618 | 619 | You need to delete the encryption private key because only the public encryption key is required to create a backup. The public key is used to encrypt data, the private key is used to decrypt data. 620 | 621 | If the encryption config or the keys are not available (you are restoring on another machine) you will need to follow the instructions in section "Restore keys and encryption config". When that is restored, you can follow the normal procedure for restoring a backup. 622 | 623 | 624 | 625 | #### How do I restore a full backup 626 | 627 | The restore script currently has issues with a wildcard like `*`. To restore a full system backup, first use the command `cloudvps-boss-list-current-files` to see all the files and folders in the backup. Take all the folders in the `/` folder and manually restore those. Most of the time you want to restore the following folders: 628 | 629 | - bin 630 | - boot 631 | - etc 632 | - home 633 | - lib/lib64 634 | - opt 635 | - root 636 | - sbin 637 | - srv 638 | - usr 639 | - var 640 | 641 | 642 | 643 | #### Where are my files/databases restored? 644 | 645 | The file / folder is restored on the original location. All exisiting files already available on the filesystem and also in the backup will be overwritten with files/folders from the backup. If you restore a folder, if it exists on the filesytem, the entire folder will be overwritten. 646 | 647 | For MySQL and Postgres databases, if the database already exists, all data will be overwritten with data from the backup. If the database does not exist, it will be created. 648 | 649 | 650 | 651 | #### Recover from another host 652 | 653 | To recover data from another host (for example, after a reinstall or crash) you can follow the steps above. However, you must make sure the hostname given in to the restore script is the same as the hostname of the other machine. If you are going to restore a database you must make sure that a database server is available, running and accessible to the restore script. 654 | 655 | 656 | 657 | #### Errors during recovery 658 | 659 | You might receive errors during recovery. Most of the time this means you provided a wrong parameter: 660 | 661 | - The file/database might not be available in the backup. 662 | - The file/database might not be available in the given timeframe. (30M instead of 30m (months over minutes)) 663 | - The database server might not be accessible. 664 | - The file/folder/partition might not be writable. 665 | - The local disk might be full or does not have enough free space. 666 | 667 | Please make sure all things are correct and that there is network connectivity and enough free space. 668 | 669 | 670 | 671 | #### Manual Recovery 672 | 673 | You can use regular duplicity to manually restore your data. TL;DR, use `--file-prefix`, `--name` and `--no-encryption` or the right gpg key id's. Read the below secton for more info on possible errors and such. 674 | 675 | If you get an error like this: 676 | 677 | Another instance is already running with this archive directory 678 | If you are sure that this is the only instance running you may delete 679 | the following lockfile and run the command again : 680 | /root/.cache/duplicity/0f8c1d052364ab963f59e5e6b3539b40/lockfile.lock 681 | 682 | Use the command `ps auxf` to check if there is still an instance running and if you are sure you are not running it in a screen session or something else, kill it with `pkill duplicity`. 683 | 684 | If you try to restore manually (`duplicity --no-encryption list-current-files swift://cloudvps-boss-backup 685 | `) and you do not give a hostname prefix you will get the following error: 686 | 687 | duplicity --no-encryption list-current-files swift://cloudvps-boss-backup 688 | Local and Remote metadata are synchronized, no sync needed. 689 | Last full backup date: none 690 | Traceback (most recent call last): 691 | File "/usr/local/bin/duplicity", line 1502, in 692 | with_tempdir(main) 693 | File "/usr/local/bin/duplicity", line 1496, in with_tempdir 694 | fn() 695 | File "/usr/local/bin/duplicity", line 1345, in main 696 | do_backup(action) 697 | File "/usr/local/bin/duplicity", line 1434, in do_backup 698 | list_current(col_stats) 699 | File "/usr/local/bin/duplicity", line 667, in list_current 700 | sig_chain = col_stats.get_signature_chain_at_time(time) 701 | File "/usr/local/lib/python2.7/dist-packages/duplicity/collections.py", line 977, in get_signature_chain_at_time 702 | raise CollectionsError("No signature chains found") 703 | CollectionsError: No signature chains found 704 | 705 | You must give a hostname, like so: 706 | 707 | duplicity --file-prefix="my-host." --name="my-host." --no-encryption list-current-files swift://cloudvps-boss-backup 708 | 709 | Note the DOT at the end of the prefix. If you are restoring on the same machine (or one with exactly the same hostname) you can use the following command options: 710 | 711 | duplicity --file-prefix="$(uname -n)." --name="$(uname -n)." --no-encryption list-current-files swift://cloudvps-boss-backup 712 | 713 | You also need to specify the relative path to the backup path. In the standard configuration the backup path is "`/`". Therefore, if we want to restore the folder `/etc/apache2/sites-enabled/` we should use the following command to restore it in the `/root/restore` folder: 714 | 715 | duplicity --file-prefix="$(uname -n)." --name="$(uname -n)." --no-encryption -t 3D --file-to-restore etc/apache2/sites-enabled swift://cloudvps-boss-backup ./restore 716 | 717 | Result: 718 | 719 | Local and Remote metadata are synchronized, no sync needed. 720 | Last full backup date: Sun Jun 29 19:39:26 2014 721 | 722 | If we don't specify either the hostname prefix with the dot or the relative path we get the following error: 723 | 724 | duplicity --file-prefix="$(uname -n)." --no-encryption -t 3D --file-to-restore /etc/apache2/sites-enabled swift://cloudvps-boss-backup ./restore 725 | Local and Remote metadata are synchronized, no sync needed. 726 | Last full backup date: Sun Jun 29 19:37:26 2014 727 | /etc/hosts not found in archive, no files restored. 728 | 729 | If you have encryption set up you need to remove the `--no-encryption` part and specify the correct GPG keys like so: `--encrypt-key=B5E3AEA9B --sign-key=8CEA5A683`. These keys are the ones you've backed up when you set up encryption. Remember to place back the private key. See "Restoring Encrypted Backups" on that part. 730 | 731 | 732 | 733 | #### A note on database restores 734 | 735 | ##### MySQL 736 | 737 | MySQL dumps are restored to the locally accessible database server defined in `/root/.my.cnf`. If the database to restore does not exist, it is created. If it exists, it is overwritten. 738 | 739 | MySQL users are also backed up, because the database `mysql` is backed up. This, however, is not automatically restored. You must either create a user yourself or explicitly restore the database `mysql`. 740 | 741 | ##### PostgreSQL 742 | 743 | The PostgreSQL database dumps are restored to the local postgresql server, using the `postgres` user. Make sure `peer` authentication is allowed. Roles and other data is backed up as well, using the `pg_dumpall -g` command. These are not restored automatically. You can do this yourself, or create a new role. To restore the role/global settings, you must use the `pg_restore` command, you cannot just restore the SQL dumped. 744 | 745 | To restore the users/roles, start the restore script and restore the file "/var/backups/sql/pg_global_data.psql.gz". Then manually import this dump with the `pg_restore` command. 746 | 747 | 748 | 749 | ### Information and Statistics 750 | 751 | CloudVPS Boss provides a simple statistics and information program, `cloudvps-boss-stats`. It shows you parts of the confuration and the status of your backup and backupsets. You can run it manually from the command line: 752 | 753 | cloudvps-boss-stats 754 | 755 | Example output: 756 | 757 | ========================================= 758 | # Start of CloudVPS Boss Status 759 | # Hostname: 1204-test-postgres-base 760 | # Username: Example Tenant:user@example.org 761 | # Storage used: 1971 MB 762 | # Full backups to keep: 6 763 | # Create full backup if last full backup is older than: 14D 764 | ----------------------------------------- 765 | # Duplicity collection status: 766 | # Local and Remote metadata are synchronized, no sync needed. 767 | # Last full backup date: Sun Jun 29 19:39:26 2014 768 | # Collection Status 769 | # ----------------- 770 | # Connecting with backend: SwiftBackend 771 | # Archive dir: /root/.cache/duplicity/0f8c1d052364ab963f59e5e6b3539b40 772 | # Found 0 secondary backup chains. 773 | # Found primary backup chain with matching signature chain: 774 | # ------------------------- 775 | # Chain start time: Sun Jun 29 19:39:26 2014 776 | # Chain end time: Thu Jul 3 13:51:50 2014 777 | # Number of contained backup sets: 16 778 | # Total number of contained volumes: 18 779 | # Type of backup set: Time: Num volumes: 780 | # Full Sun Jun 29 19:39:26 2014 2 781 | # Incremental Sun Jun 29 20:06:43 2014 1 782 | # Incremental Sun Jun 29 20:44:04 2014 1 783 | # Incremental Sun Jun 29 21:38:12 2014 1 784 | # Incremental Mon Jun 30 12:06:27 2014 1 785 | # Incremental Mon Jun 30 12:46:50 2014 1 786 | # Incremental Mon Jun 30 13:11:20 2014 1 787 | # Incremental Mon Jun 30 14:18:41 2014 1 788 | # Incremental Tue Jul 1 07:52:02 2014 1 789 | # Incremental Tue Jul 1 08:07:35 2014 1 790 | # Incremental Tue Jul 1 08:08:38 2014 1 791 | # Incremental Tue Jul 1 08:16:18 2014 1 792 | # Incremental Tue Jul 1 08:21:14 2014 1 793 | # Incremental Thu Jul 3 05:32:08 2014 1 794 | # Incremental Thu Jul 3 10:09:46 2014 1 795 | # Incremental Thu Jul 3 13:51:50 2014 2 796 | # ------------------------- 797 | # No orphaned or incomplete backup sets found. 798 | # End of CloudVPS Boss Status 799 | ========================================= 800 | 801 | If you want to email this to yourself on a regular base, create a cronjob for it: 802 | 803 | # /etc/cron.d/cloudvps-boss-stats - weekly email 804 | MAILTO="user@example.org" 805 | 0 0 * * 0 root /usr/local/bin/cloudvps-boss-stats 806 | 807 | 808 | 809 | 810 | #### How can I view the full contents of a backup? 811 | 812 | You can execute the following command: 813 | 814 | cloudvps-boss-list-current-files 815 | 816 | It will use the duplicity option `list-current-files` to show all the files in the latest backup. 817 | 818 | Example output: 819 | 820 | Sat May 3 00:16:01 2014 home/admin/domains/example.org/public_html/modules_v3/fact_sources/report.xml 821 | Sat May 3 00:16:01 2014 home/admin/domains/example.org/public_html/modules_v3/families 822 | Sat May 3 00:16:01 2014 home/admin/domains/example.org/public_html/modules_v3/families/module.php 823 | Sat May 3 00:16:01 2014 home/admin/domains/example.org/public_html/modules_v3/family_group_report 824 | 825 | It gives a lot of output, you might want to view it through `less` or pipe the output to a text file: 826 | 827 | cloudvps-boss-list-current-files > files-in-backup.txt 828 | vim files-in-backup.txt 829 | 830 | You can specify a time of a backup of which you want to view the files: 831 | 832 | cloudvps-boss-list-current-files 3D # for the backup from three days ago. 833 | 834 | The time format is the same as which Duplicity supports. [See the man page for more info](http://duplicity.nongnu.org/duplicity.1.html#sect9) 835 | 836 | If you want to know if a specific file or folder is in the backup, you can use the `grep` command: 837 | 838 | cloudvps-boss-list-current-files 3D | grep "home/user/important-folder" 839 | 840 | If that gives no output, it is not in the backup. Try another timeframe. 841 | 842 | 843 | 844 | ### How can I manually clean up / remove backups? 845 | 846 | To remove backups outside of Duplicity we can use the `swift` command line to select the correct files and remove them. 847 | 848 | *Do note, this will break your backups. Only use this if you have a corrupt backup, be carefull.* 849 | 850 | Let's say your backup is corrupt and gives an AssertionError or some other one. You need to remove all the backups and start over again. 851 | 852 | Or you just want to clean up and start over, thus loosing all your backup history. 853 | 854 | First source the authentication and common files: 855 | 856 | source /etc/cloudvps-boss/auth.conf 857 | source /etc/cloudvps-boss/common.sh 858 | 859 | Get a list of all the backup related files from `swift`: 860 | 861 | swift list cloudvps-boss-backup --prefix HOSTNAME. 862 | 863 | Where you replase HOSTNAME with your server hostname. 864 | 865 | Example output: 866 | 867 | HOSTNAME.duplicity-full.20140827T01.9.5Z.vol9.difftar.gz 868 | HOSTNAME.duplicity-full.20140910T230407Z.manifest 869 | HOSTNAME.duplicity-new-signatures.20140831T01.9.5Z.to.20140901T012204Z.sigtar.gz 870 | 871 | Remove the Duplicity cache files: 872 | 873 | rm -rf /root/.cache/duplicity 874 | 875 | Now remove all the backup files from the Object Store: 876 | 877 | swift list cloudvps-boss-backup --prefix HOSTNAME. | xargs -L 1 -I % swift delete cloudvps-boss-backup % 878 | 879 | This will delete all files one by one from the object store. Don't forget to create a new backup when this is done. 880 | 881 | To see all the hosts you have backed up in one container, use the following command: 882 | 883 | swift list cloudvps-boss-backup | grep -v -e status | awk -F"duplicity" '{print $1}' | sort -u 884 | 885 | 886 | 887 | ### Notifications and Monitoring 888 | 889 | If you want to receive email notifications of *failed* backups you need to install and configure a mailserver (postfix, sendmail, exim etc.). On a controlpanel VPS this is already done. You should also create the file `/etc/cloudvps-boss/email.conf`. That file should have one email address per line. All those email addresses will receive an email when a backup fails. 890 | 891 | If you want to receive the output of the backup via mail every time it runs, change the following line in the file `/etc/cron.d/cloudvps-boss`: 892 | 893 | MAILTO="root" 894 | 895 | Replace `root` with your email address. You still need to set up a mailserver. 896 | 897 | **If you do not set up a mailserver and configure that file with at least one email address you will NOT receive notifications of failed backups.** 898 | 899 | 900 | Before, after and after a failed backup we upload a status file to the object store. This is placed in `/etc/cloudvps-boss/status/$HOSTNAME/` and can be named `started`, `completed` or `failed`. You can use Nagios or any other monitoring system to monitor the age of these files and give you alerts when they are older than X amount of time. Nagios has the file_age check for this. Below is an `NRPE` example: 901 | 902 | command[check_cloudvps-boss]=/usr/lib64/nagios/plugins/check_file_age -w 86400 -c 129600 -f /etc/cloudvps-boss/status/**/completed 903 | 904 | You can also write a simple script which talks to a web service or MQTT broker and place that in either `/etc/cloudvps-boss/post-fail-backup.d/` or `/etc/cloudvps-boss/post-backup.d/`. This way you can implement your own calls or actions when a backup fails or succeeds. 905 | 906 | 907 | 908 | ### Uninstall 909 | 910 | You can use the uninstall script to remove all of CloudVPS Boss. It does not touch your backups, it only removes CloudVPS Boss and related scripts/config. Run it like so: 911 | 912 | bash /etc/cloudvps-boss/uninstall.sh 913 | 914 | Example output: 915 | 916 | Would you like to completely remove CloudVPS Boss? Your backups will NOT be removed. [y/N]? y 917 | # Completely removing all of CloudVPS Boss 918 | # Removing folder /etc/cloudvps-boss/ 919 | # Removing symlink /usr/local/bin/cloudvps-boss 920 | # Removing symlink /usr/local/bin/cloudvps-boss-restore 921 | # Removing symlink /usr/local/bin/cloudvps-boss-stats 922 | # Uninstalling python-swiftclient with pip. 923 | # Uninstalling python-keystoneclient with pip. 924 | 925 | 926 | 927 | ### Duplicity Encryption 928 | 929 | Before version 1.4 CloudVPS Boss did not support encryption. To use Duplicity's encryption features, make sure you have a version of CloudVPS Boss higher than 1.4. 930 | 931 | CloudVPS Boss support Duplicity's encryption features. It has a command that can be used to enable encryption. It will generate 2 GPG keys and place an extra config file. 932 | 933 | When using encryption it is **important** that you back up your keys, test if they work and repeat this on a regular basis. 934 | 935 | **If you do not backup your keys or loose them and the backups of your keys, all your backups are unaccessible!** CloudVPS is not able to access or recover your backups if you have lost the keys. 936 | 937 | If you understand what this means then please read on. If you don't, read up on encryption and gpg or ask someone else for help. 938 | 939 | 940 | 941 | #### Why Encryption? 942 | 943 | There are several reasons to use Encryption for your backups. The most important are: 944 | 945 | - Nobody else (without the keys) can access your backed up data. 946 | - You can backup multiple servers to the same container without one being able to read anohter ones backup data. (Deletion is still possible) 947 | 948 | We cannot see the data in your backups if they are encrypted. Nobody else can. If we are forced to hand over your backed up data we can only give out encrypted blobs. 949 | 950 | There are also some disadvantages to enabling encryption. 951 | 952 | - Encryption requires a lot of CPU. The VPS will be slower when the backup is made. 953 | - Restores take longer because of the increased CPU power needed. 954 | - Backups will be a bit larger in size (about 10 to 25 percent). 955 | - Restores require extra steps, the keying material needs to be restored first. 956 | 957 | 958 | 959 | #### Set up Encryption 960 | 961 | To set up encryption you first need to make sure GPG is installed. On debian/ubuntu: 962 | 963 | apt-get install gnupg2 964 | 965 | On CentOS/RHEL/Fedora: 966 | 967 | yum install gnupg2 968 | 969 | CloudVPS Boss provides an easy setup of all the required keys and config. After running that all the CloudVPS Boss tools will use encryption, transparantly. 970 | 971 | The program also prints out all the information you need for restoring the keys. You need to back that up to a different place. 972 | 973 | If you do not back this data up and the keys are lost, you will not be able to decrypt your backed up data. 974 | 975 | To start the encryption setup, execute the following command: 976 | 977 | bash /etc/cloudvps-boss/cloudvps-boss-encryption-setup.sh 978 | 979 | You are then given some information and asked to confirm that you will backup the keying materials: 980 | 981 | You are going to set up encryption for your backups. 982 | You need to have a good key backup, testing and recovery 983 | procedure. A lot of information will be shown on the 984 | screen when the setup is done. Make sure to back that up. 985 | 986 | Please type 'I have a good key management procedure and want to set up encryption.': 987 | 988 | Key generation can take a long time because of the lack of randomness on a virtual machine. The [haveged](http://www.issihosts.com/haveged/) daemon is an easy-to-use, unpredictable random number generator based upon an adaptation of the [HAVEGE](http://www.irisa.fr/caps/projects/hipsor/) algorithm. If you want to speed up the key generation, install `haveged` with your package manager. For Ubuntu/Debian: 989 | 990 | apt-get install haveged 991 | 992 | CentOS/Red Hat: 993 | 994 | yum install haveged 995 | 996 | Then it will create the actual keys using GPG: 997 | 998 | # Placing signing key gpg config 999 | # Placing encryption key gpg config 1000 | # Generating signing key with gpg. 1001 | # Generating encryption key with gpg. 1002 | 1003 | It will then pause and say the following: 1004 | 1005 | Please save the following information. Press ENTER to continue. 1006 | 1007 | After you press enter you will receive all the information you need to restore the data. This includes the GPG keys, and the backup config. Print that out, back that up elsewhere, manage it. That data are the keys to the kingdom. 1008 | 1009 | It looks like this: 1010 | 1011 | ===== BEGIN IMPORTANT RESTORE INFORMATION ===== 1012 | Please backup the following in full. 1013 | If you don't, or loose it, you will not 1014 | be able to access or restore your backups. 1015 | Don't share this with anyone. This information 1016 | gives complete access to this backup. 1017 | 1018 | 1019 | ===== SIGNING KEY ===== 1020 | GPG Info: 1021 | gpg: checking the trustdb 1022 | gpg: 3 marginal(s) needed, 1 complete(s) needed, PGP trust model 1023 | gpg: depth: 0 valid: 2 signed: 0 trust: 0-, 0q, 0n, 0m, 0f, 2u 1024 | pub 4096R/68BD06B3 2014-08-24 1025 | uid CloudVPS Boss Backup Signing Key for hostname 1026 | sub 2048g/6549E14F 2014-08-24 1027 | 1028 | -----BEGIN PGP PRIVATE KEY BLOCK----- 1029 | Version: GnuPG v2.0.14 (GNU/Linux) 1030 | 1031 | lQc+BFP5jhkBEADSdgzAD7YWlqe7xTW8ftxt5Ujd9y6bmfMXaqrbR4xClNHH1u1O 1032 | LfTySta0v2AZNpKD5Xi8yOjE4JMUypzj3YQRY2e52i8r7fY8tB6aeV7clZ/aM7BU 1033 | [...] 1034 | TrVzxer0kOxGVt57CsMR++u7I2LIaE6W9XOi7r7D8ruiTxi6hgEDK6GiOaSrB0sI 1035 | 3pvuCP5RarO9dFcVAEIAVzOlzl6g 1036 | =Mq9b 1037 | -----END PGP PRIVATE KEY BLOCK----- 1038 | 1039 | ===== ENCRYPTION KEY ===== 1040 | GPG Info: 1041 | pub 4096R/64DDBC7D 2014-08-24 1042 | uid CloudVPS Boss Backup Encryption Key for hostname 1043 | sub 2048g/507ACE44 2014-08-24 1044 | 1045 | -----BEGIN PGP PRIVATE KEY BLOCK----- 1046 | Version: GnuPG v2.0.14 (GNU/Linux) 1047 | 1048 | lQc+BFP5jh0BEAC65qgr8mEYAbQaB0esRMC0K74r2EWJoMVar9NAmRYjCOVeJlqQ 1049 | r5n1TSgibEZBR1xV+EfGgOfsJhXqsH1rMPoyIcmwi85JJ5xYChOgzIGXAm/WXyXc 1050 | [...] 1051 | SXVMaAMqSRbtZXKBNTejFckIx3w67fJw27s57n3a6yC41Iwm2Tje7jXSkukGI2Pg 1052 | CFk1FLi8KirQiN+usoL9rleg08ScgWgRPLXf 1053 | =spVj 1054 | -----END PGP PRIVATE KEY BLOCK----- 1055 | 1056 | ===== GPG Ownertrust ===== 1057 | # List of assigned trustvalues, created Sun 24 Aug 2014 10:55:13 AM CEST 1058 | # (Use "gpg --import-ownertrust" to restore them) 1059 | DE26F4EE4258911CA84DB3B1FF47966CC3C2722A:6: 1060 | 6D8FA327F0E32E105CF172AE278E482DE3AA7B82:6: 1061 | 1062 | ===== ENCRYPTION CONFIG ===== 1063 | # CloudVPS Boss Encryption Config file 1.4 1064 | SIGN_KEY="68BD06B3" 1065 | ENC_KEY="64DDBC7D" 1066 | ENCRYPTION_OPTIONS="--encrypt-key=64DDBC7D --sign-key=68BD06B3 " 1067 | 1068 | To restore these keys: place the contents 1069 | in 2 files, sign.key and encryption.key. 1070 | Execute the following commands as root: 1071 | # gpg --import-key sign.key 1072 | # gpg --import-key encryption.key 1073 | 1074 | Restore the configuration printed above 1075 | to /etc/cloudvps-boss/encryption.conf. 1076 | 1077 | Place the ownertrust contents in a file" 1078 | named 'ownertrust.gpg' and import it:" 1079 | # gpg --import-ownertrust ownertrust.gpg" 1080 | 1081 | ===== END IMPORTANT RESTORE INFORMATION ===== 1082 | # Cleaning up config files. 1083 | # Removing secret key used for encryption. 1084 | # Encryption setup done. Please make a backup now, execute 'cloudvps-boss'. 1085 | 1086 | We cannot stress enough how important it is you back this data up. Please, please do so. 1087 | 1088 | Do not share this with anyone. If you give out the above information people will be able to access your backups. 1089 | 1090 | The encryption is now setup. Please make a backup. You can see the difference between non-encrypted backups and encrypted backups by 2 ways. The first is the command which is printed before the backup. Non-encrypted backups look like this: 1091 | 1092 | # ionice -c2 nice -n19 duplicity --asynchronous-upload --volsize 25 --tempdir="" --file-prefix="HOSTNAME." --exclude-device-files --exclude-globbing-filelist=/etc/cloudvps-boss/exclude.conf --full-if-older-than="14D" --no-encryption / swift://cloudvps-boss-backup 1093 | 1094 | Encrypted backups do not have the `--no-encryption` option but specify which keys are used to sign and encrypt (`--encrypt-key=64DDBC7D --sign-key=68BD06B3`): 1095 | 1096 | # ionice -c2 nice -n19 duplicity --asynchronous-upload --volsize 25 --tempdir="" --file-prefix="hostname." --exclude-device-files --exclude-globbing-filelist=/etc/cloudvps-boss/exclude.conf --full-if-older-than="14D" --encrypt-key=64DDBC7D --sign-key=68BD06B3 / swift://cloudvps-boss-backup 1097 | 1098 | The second way is to look at the files in the object store. These will now have the `.gpg` extension instead of `volX.difftar` or `.sigtar`. 1099 | 1100 | 1101 | ### I want a passphrase on my key 1102 | 1103 | You can set a password for the encryption key by first importing the secret key. Then execute the following command: 1104 | 1105 | gpg --edit-key ENCRYPTION_KEY_ID 1106 | 1107 | Give the following command: 1108 | 1109 | gpg> passwd 1110 | This key is not protected. 1111 | Enter the new passphrase for this secret key. 1112 | 1113 | Enter a passphrase twice. Then save and quit: 1114 | 1115 | gpg> quit 1116 | Save changes? (y/N) y 1117 | 1118 | 1119 | Example output 1120 | 1121 | 1122 | 1123 | #### Restore keys and encryption config 1124 | 1125 | To restore data from an encrypted backup the encryption private key needs to be restored. See the section "Restoring Encrypted Backups" to find out how to do that. 1126 | 1127 | If the encryption config or the keys are not available you will need to follow the below procedure. This might happen when you want to restore a backup to another machine. 1128 | 1129 | It is important to test the backups of every machine when they are encrypted. Please test the encrypted backups, on a new machine, every month. This includes restoring the keying material. 1130 | 1131 | When the encryption was set up, you backed up the important information containing the keys. Find it for this host. 1132 | 1133 | Make sure GPG and haveged are installed. On debian/ubuntu: 1134 | 1135 | apt-get install gnupg2 haveged 1136 | 1137 | On CentOS/RHEL/Fedora: 1138 | 1139 | yum install gnupg2 haveged 1140 | 1141 | Edit or create the following file: 1142 | 1143 | vim /etc/cloudvps-boss/encryption.conf 1144 | 1145 | Place the below contents in it, filling in the key ID's and passphrases with your data from your back up: 1146 | 1147 | # CloudVPS Boss Encryption Config file 1.4 1148 | SIGN_KEY="" 1149 | ENC_KEY="" 1150 | ENCRYPTION_OPTIONS="--encrypt-key="" --sign-key="" " 1151 | 1152 | Place the encryption key in a file named `encryption.key`. In your backed up data it was displayed like this: 1153 | 1154 | ===== ENCRYPTION KEY ===== 1155 | GPG Info: 1156 | pub 4096R/64DDBC7D 2014-08-24 1157 | uid CloudVPS Boss Backup Encryption Key for hostname 1158 | sub 2048g/507ACE44 2014-08-24 1159 | 1160 | -----BEGIN PGP PRIVATE KEY BLOCK----- 1161 | Version: GnuPG v2.0.14 (GNU/Linux) 1162 | 1163 | lQc+BFP5jh0BEAC65qgr8mEYAbQaB0esRMC0K74r2EWJoMVar9NAmRYjCOVeJlqQ 1164 | r5n1TSgibEZBR1xV+EfGgOfsJhXqsH1rMPoyIcmwi85JJ5xYChOgzIGXAm/WXyXc 1165 | [...] 1166 | SXVMaAMqSRbtZXKBNTejFckIx3w67fJw27s57n3a6yC41Iwm2Tje7jXSkukGI2Pg 1167 | CFk1FLi8KirQiN+usoL9rleg08ScgWgRPLXf 1168 | =spVj 1169 | -----END PGP PRIVATE KEY BLOCK----- 1170 | 1171 | You need the part between `-----BEGIN PGP PRIVATE KEY BLOCK-----` and `-----END PGP PRIVATE KEY BLOCK-----`. 1172 | 1173 | Repeat this for the signing key. Place it in a file named `sign.key`. 1174 | 1175 | Import these keys into the GPG database using the following command: 1176 | 1177 | gpg --import sign.key 1178 | 1179 | The output looks like this: 1180 | 1181 | 1182 | gpg: directory `/root/.gnupg' created 1183 | gpg: new configuration file `/root/.gnupg/gpg.conf' created 1184 | gpg: WARNING: options in `/root/.gnupg/gpg.conf' are not yet active during this run 1185 | gpg: keyring `/root/.gnupg/secring.gpg' created 1186 | gpg: keyring `/root/.gnupg/pubring.gpg' created 1187 | gpg: key 8C12A683: secret key imported 1188 | gpg: /root/.gnupg/trustdb.gpg: trustdb created 1189 | gpg: key 8C12A683: public key "CloudVPS Boss Backup Signing Key for hostname " imported 1190 | gpg: Total number processed: 1 1191 | gpg: imported: 1 (RSA: 1) 1192 | gpg: secret keys read: 1 1193 | gpg: secret keys imported: 1 1194 | 1195 | Do the same for the encryption key: 1196 | 1197 | gpg --import encryption.key 1198 | 1199 | Example output: 1200 | 1201 | gpg: key B5EBDA9B: secret key imported 1202 | gpg: key B5EBDA9B: public key "CloudVPS Boss Backup Encryption Key for hostname " imported 1203 | gpg: Total number processed: 1 1204 | gpg: imported: 1 (RSA: 1) 1205 | gpg: secret keys read: 1 1206 | gpg: secret keys imported: 1 1207 | 1208 | Now we need to mark the keys as trusted. If you have backed up the Owner Trust you can restore it. Place the contents in a file named `ownertrust.gpg` and import it with the following command: 1209 | 1210 | gpg --import-ownertrust ownertrust.gpg 1211 | 1212 | You can also do it manually. Execute the following command, replacing the key ID with your key ID: 1213 | 1214 | gpg --edit-key 8C12A683 trust 1215 | 1216 | Example output: 1217 | 1218 | gpg (GnuPG) 1.4.11; Copyright (C) 2010 Free Software Foundation, Inc. 1219 | This is free software: you are free to change and redistribute it. 1220 | There is NO WARRANTY, to the extent permitted by law. 1221 | 1222 | Secret key is available. 1223 | 1224 | pub 4096R/8C12A683 created: 2014-08-23 expires: never usage: SCEA 1225 | trust: unknown validity: unknown 1226 | sub 2048g/C99A7CDB created: 2014-08-23 expires: never usage: E 1227 | [ unknown] (1). CloudVPS Boss Backup Signing Key for hostname 1228 | 1229 | pub 4096R/8C12A683 created: 2014-08-23 expires: never usage: SCEA 1230 | trust: unknown validity: unknown 1231 | sub 2048g/C99A7CDB created: 2014-08-23 expires: never usage: E 1232 | [ unknown] (1). CloudVPS Boss Backup Signing Key for hostname 1233 | 1234 | Please decide how far you trust this user to correctly verify other users' keys 1235 | (by looking at passports, checking fingerprints from different sources, etc.) 1236 | 1237 | 1 = I don't know or won't say 1238 | 2 = I do NOT trust 1239 | 3 = I trust marginally 1240 | 4 = I trust fully 1241 | 5 = I trust ultimately 1242 | m = back to the main menu 1243 | 1244 | We have to type `5` here: 1245 | 1246 | Your decision? 5 1247 | 1248 | Then type `y`: 1249 | 1250 | Do you really want to set this key to ultimate trust? (y/N) y 1251 | 1252 | pub 4096R/8C12A683 created: 2014-08-23 expires: never usage: SCEA 1253 | trust: ultimate validity: unknown 1254 | sub 2048g/C99A7CDB created: 2014-08-23 expires: never usage: E 1255 | [ unknown] (1). CloudVPS Boss Backup Signing Key for hostname 1256 | Please note that the shown key validity is not necessarily correct 1257 | unless you restart the program. 1258 | 1259 | gpg> quit 1260 | 1261 | Repeat this for the encryption key: 1262 | 1263 | gpg --edit-key B5EBDA9B trust 1264 | 1265 | You can now install CloudVPS Boss and restore an encrypted backup. Make sure to use the correct hostname. 1266 | 1267 | 1268 | 1269 | ### Other Edge Cases 1270 | 1271 | This section provides more info on other edge cases. 1272 | 1273 | 1274 | 1275 | #### What if my container is deleted? 1276 | 1277 | If your container with all the backups is deleted from the oject store you've lost all you backups. New backups will work without issues, except they might log lines like these for the first time: 1278 | 1279 | Deleting local /root/.cache/duplicity/0f8c1d052364ab963f59e5e6b3539b40/example.duplicity-new-signatures.20140704T143110Z.to.20140705T053010Z.sigtar.gz (not authoritative at backend). 1280 | 1281 | 1282 | 1283 | #### What if my password changed? 1284 | 1285 | You will need to edit `/etc/cloudvps-boss/auth.conf` and place your new password there. Ohterwise you will get errors like these: 1286 | 1287 | Unauthorised. Check username, password and tenant name/id 1288 | cloudvps-boss: ERROR - Could not upload status 1289 | Connection failed: ClientException Unauthorised. Check username, password and tenant name/id 1290 | Error trying to create container 'cloudvps-boss-backup': 0 1291 | 1292 | 1293 | 1294 | #### What if the Object Store is Down? 1295 | 1296 | The backups will fail. If the object store is up again they will continue the next time the cronjob runs. 1297 | 1298 | 1299 | 1300 | #### What happens if a backup is still running when a new one starts? 1301 | 1302 | The running backup will continue. The new backup will give an error like this: 1303 | 1304 | cloudvps-boss: ERROR - Reading globbing filelist /etc/cloudvps-boss/exclude.conf 1305 | cloudvps-boss: ERROR - Another instance is already running with this archive directory 1306 | cloudvps-boss: ERROR - If you are sure that this is the only instance running you may delete 1307 | cloudvps-boss: ERROR - the following lockfile and run the command again : 1308 | cloudvps-boss: ERROR - /root/.cache/duplicity/0f8c1d052364ab963f59e5e6b3539b40/lockfile.lock 1309 | CloudVPS Boss to Object Store FAILED!. Please check server example.org. 1310 | Running post-fail-backup scripts from /etc/cloudvps-boss/post-fail-backup.d/ 1311 | 1312 | If you've configured email support you will receive an email with the error messages. 1313 | 1314 | If this happens multiple times you need to check if the backup is still running. If there is no rsync or Duplicity process, remove the lockfile and manually try again. 1315 | 1316 | 1317 | 1318 | #### Not enough free space 1319 | 1320 | If you get errors like these: 1321 | 1322 | cloudvps-boss: ERROR - Temp space has 242413568 available, backup needs approx 602931200. 1323 | 1324 | Your `/tmp/` folder does not have enough space. Change it in `/etc/cloudvps-boss/backup.conf` to a folder which does have enough space: 1325 | 1326 | mkdir /root/tmp 1327 | TEMPDIR="/root/tmp" 1328 | 1329 | If you still receive the error message you need to clean up some space so that Duplicity can do it's work. See the [Duplicity Man Page](http://duplicity.nongnu.org/duplicity.1.html#sect7) for more info. 1330 | 1331 | 1332 | 1333 | #### Unsupported Backup Scheme 1334 | 1335 | If you get the following error: 1336 | 1337 | UnsupportedBackendScheme: scheme not supported in url: swift://cloudvps-boss-backup 1338 | 1339 | You need to upgrade Duplicity. The version you have is to old. Swift support was added in 0.6.22. Remove the system package and install it again via the installer from cloudvps-boss. 1340 | 1341 | 1342 | 1343 | #### logger: command not found 1344 | 1345 | If you get errors like the following: 1346 | 1347 | install.sh: line 32: logger: command not found 1348 | 1349 | You are probably on a very minimal image. Please install the logger command, in Ubuntu/Debian it can be found in the `bsdutils` package: 1350 | 1351 | apt-get --reinstall install bsdutils 1352 | 1353 | 1354 | 1355 | #### Duplicity is using 100% of all resources 1356 | 1357 | The script utilizes aggressive `nice` and `ionice` parameters (`ionice -c2 nice -n19 $commnad`). This means that if your server is doing stuff it will not steal resources from other processes. If your server is doing nothing or not much, it will use the resources it can use. Unused resources are a waste, so we use all we can get. However, the nice settings should make sure that when you have a busy server only a small amount of capacity is used so that other processes do not suffer from the backup. 1358 | 1359 | 1360 | 1361 | #### A note on huge files / large backups 1362 | 1363 | If you have more than 500 GB to backup, or have large files changing fast you need to tweak a few settings to get the backup to run fast enough. 1364 | 1365 | 1. Make sure there is enough temp space. The `TMPDIR` (/tmp by default) and the `/root/.cache/` folder. The latter one is because of this [bug](https://bugs.launchpad.net/duplicity/+bug/385495) in Duplicity, it does not split the signature files so they can become huge. If your root partition is smaal it might fill up, so symlink in to somewhere with more space. If your home partition is large, change the `TMPDIR` and create a symlink for the cache folder: 1366 | 1367 | mkdir -p /home/tmp/cache 1368 | TMPDIR=/home/tmp 1369 | ln -s /home/tmp/cache /root/.cache 1370 | 1371 | 2. Change the volume size from the default 25 MB to something larger, like 250 or 512 MB. Make sure the volumes are not bigger than the available RAM, otherwise restoring will fail. To change the default volume size, add the following to your `custom.conf` file: 1372 | 1373 | VOLUME_SIZE="250" 1374 | 1375 | Where 250 is the size in megabytes. 1376 | 1377 | 3. Change the `--max-blocksize` option. 1378 | 1379 | max_blocksize determines the size of the block examined for changes during the diff process. For files < 1MB the blocksize is a constant of 512. For files over 1MB the size is given by: 1380 | 1381 | file_blocksize = int((file_len / (2000 * 512)) * 512) 1382 | return min(file_blocksize, globals.max_blocksize) 1383 | 1384 | where globals.max_blocksize is defaulted to 2048. If you specify a larger max_blocksize, your difftar files will be larger, but your sigtar files will be smaller. If you specify a smaller max_blocksize, the reverse occurs. The --max-blocksize option should be in multiples of 512. 1385 | 1386 | ([via Kenneth Loafman](http://lists.nongnu.org/archive/html/duplicity-talk/2014-10/msg00053.html)) 1387 | 1388 | Add the following to `custom.conf`: 1389 | 1390 | CUSTOM_DUPLICITY_OPTIONS='--max-blocksize=16777216' 1391 | 1392 | Do note that the above parameters also need to be set when doing a restore on another machine. 1393 | 1394 | 1395 | 1396 | #### Restore looks 'weird' in a screen session 1397 | 1398 | ![](http://i.imgur.com/uCj4bcE.png) 1399 | 1400 | If the restore script looks weird like above and you are running in a `screen` session you can do either one of the following things to fix it. The issue is either with your locale settings. 1401 | 1402 | - Add `cjkwidth off` to your `.screenrc`. 1403 | - Run the screen command prefixed with `LANG= `. (Example, `LANG= screen -DR`). 1404 | 1405 | 1406 | 1407 | ### License 1408 | 1409 | CloudVPS Boss - Duplicity wrapper to back up to OpenStack Swift, Object 1410 | Store. Copyright (C) 2014 CloudVPS. 1411 | Author: Remy van Elst, https://raymii.org 1412 | 1413 | 1414 | This program is free software; you can redistribute it and/or modify it 1415 | under the terms of the GNU General Public License as published by the 1416 | Free Software Foundation; either version 2 of the License, or (at your 1417 | option) any later version. 1418 | 1419 | This program is distributed in the hope that it will be useful, but 1420 | WITHOUT ANY WARRANTY; without even the implied warranty of 1421 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 1422 | General Public License for more details. 1423 | 1424 | You should have received a copy of the GNU General Public License along 1425 | with this program; if not, write to the Free Software Foundation, Inc., 1426 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 1427 | 1428 | Also see LICENSE.md for full text of GPLv2. 1429 | 1430 | ### Authors 1431 | 1432 | - Code: [Remy van Elst](https://raymii.org) 1433 | - Code review: [Koert van der Veer](http://blog.ondergetekende.nl) 1434 | - Control Panel MySQL credential code adapted from work by Jeffrey Langerak 1435 | 1436 | --------------------------------------------------------------------------------