├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── backup ├── config │ ├── index │ │ ├── elasticsearch.yml │ │ ├── mariadb.yml │ │ ├── mongodb.yml │ │ ├── mysql.yml │ │ └── postgresql.yml │ ├── rancher-backup.yml │ └── templates │ │ ├── elasticsearch.yml │ │ ├── mariadb.yml │ │ ├── mongodb.yml │ │ ├── mysql.yml │ │ └── postgresql.yml ├── requirements.txt ├── run_test.sh ├── src │ ├── __init__.py │ ├── backup.py │ └── fr │ │ ├── __init__.py │ │ └── webcenter │ │ ├── __init__.py │ │ └── backup │ │ ├── Backup.py │ │ ├── Command.py │ │ ├── Config.py │ │ ├── Rancher.py │ │ ├── Singleton.py │ │ └── __init__.py └── test │ └── backup │ ├── TestBackup.py │ ├── TestCommand.py │ ├── TestConfig.py │ ├── TestRancher.py │ ├── TestTemplate.py │ └── __init__.py └── root ├── etc ├── cont-init.d │ ├── 00-welcome.sh │ ├── 01-gen-confd-config.sh │ ├── 02-init-confd.sh │ └── 04-run-backup.sh ├── fix-attrs.d │ └── 01-backup-dir └── services.d │ └── cron │ └── finish └── opt └── confd └── etc └── templates ├── cron.tmpl └── rancher-backup.yml.tmpl /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | *.pyc 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.5 2 | MAINTAINER Sebastien LANGOUREAUX 3 | 4 | # Application settings 5 | ENV CONFD_PREFIX_KEY="/backup" \ 6 | CONFD_BACKEND="env" \ 7 | CONFD_INTERVAL="60" \ 8 | CONFD_NODES="" \ 9 | S6_BEHAVIOUR_IF_STAGE2_FAILS=2 \ 10 | APP_HOME="/opt/backup" \ 11 | APP_DATA="/backup" \ 12 | USER=backup \ 13 | LANG=C.UTF-8 \ 14 | CONTAINER_NAME="rancher-backup" \ 15 | CONTAINER_AUHTOR="Sebastien LANGOUREAUX " \ 16 | CONTAINER_SUPPORT="https://github.com/disaster37/rancher-backup/issues" \ 17 | APP_WEB="https://github.com/disaster37/rancher-backup" 18 | 19 | 20 | # Add libs & tools 21 | COPY backup/requirements.txt /${APP_HOME}/ 22 | RUN apk update && \ 23 | apk add python2 py-pip bash tar curl docker duplicity lftp ncftp py-paramiko py-gobject py-boto py-lockfile ca-certificates librsync py-cryptography py-cffi &&\ 24 | pip install --upgrade pip &&\ 25 | pip install -r "${APP_HOME}/requirements.txt" &&\ 26 | rm /var/cache/apk/* 27 | 28 | 29 | # Install go-cron 30 | RUN curl -sL https://github.com/michaloo/go-cron/releases/download/v0.0.2/go-cron.tar.gz \ 31 | | tar -x -C /usr/local/bin 32 | 33 | # Install confd 34 | ENV CONFD_VERSION="0.14.0" \ 35 | CONFD_HOME="/opt/confd" 36 | RUN mkdir -p "${CONFD_HOME}/etc/conf.d" "${CONFD_HOME}/etc/templates" "${CONFD_HOME}/log" "${CONFD_HOME}/bin" &&\ 37 | curl -Lo "${CONFD_HOME}/bin/confd" "https://github.com/kelseyhightower/confd/releases/download/v${CONFD_VERSION}/confd-${CONFD_VERSION}-linux-amd64" &&\ 38 | chmod +x "${CONFD_HOME}/bin/confd" 39 | 40 | # Install s6-overlay 41 | RUN curl -sL https://github.com/just-containers/s6-overlay/releases/download/v1.19.1.1/s6-overlay-amd64.tar.gz \ 42 | | tar -zx -C / 43 | 44 | 45 | # Copy files 46 | COPY root / 47 | COPY backup/src/ /${APP_HOME}/ 48 | COPY backup/config /${APP_HOME}/config 49 | RUN mkdir -p /var/log/backup ${APP_DATA} &&\ 50 | adduser -D -h ${APP_HOME} -G docker -s /bin/sh ${USER} &&\ 51 | chown -R ${USER} ${APP_HOME} &&\ 52 | chown -R ${USER} ${APP_DATA} &&\ 53 | chown -R ${USER} /var/log/backup 54 | 55 | # CLEAN Image 56 | RUN rm -rf /tmp/* /var/tmp/* 57 | 58 | VOLUME ["${APP_DATA}"] 59 | WORKDIR "${APP_HOME}" 60 | CMD ["/init"] 61 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 disaster37 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rancher-backup 2 | 3 | It's a general purpose to solve backup matter on Rancher. 4 | The goal, it's to have ability to use docker command to perform dump (when needed) before to start external backup with duplicity. 5 | 6 | To do the job in easiest way, we use the power of Rancher API to discover the service witch must be dumped before to start the backup. 7 | We use some settings files on `/opt/backup/config` to explain how discover the service witch must be dumped and how to do that. 8 | The first time we try to apply regex on label called `backup.type`, and if not match we try to apply on image name. 9 | Next, all the contains of `BACKUP_duplicity_source-path` (default is /backup) is backuped on remote backend with duplicity. So you can map your data volume on this container to backup it in the same time. 10 | 11 | You are welcome to contribute on github to extend the supported service (use develop branch). 12 | 13 | # Compatibilities 14 | 15 | ## Standard databases compatibilities 16 | 17 | No extra need, use dump tools utilities to do remote dump. 18 | 19 | - `MySQL`: the docker image must have `mysql` on name. Use `mysqldump` to perform the dump. 20 | - `MariaDB`: the docker image must have `mariadb` on name. Use `mysqldump` to perform the dump. 21 | - `PostgreSQL`: the docker image must have `postgres` on name. Use `pgdump` to perform the dump. 22 | - `MongoDB`: the docker image must have `mongo` on name. Use `mongodump` to perform the dump. 23 | - `Elasticsearch`: the docker image must have `elasticsearch` on name. Use `elasticdump` to perform the dump. 24 | 25 | ## Distributed NoSQL databases 26 | 27 | Need to have shared volume (like glusterfs, S3, Ceph, etc.) between each database nodes and the backup service. 28 | To to dump, we use tools utilities to ask each nodes perform a local dump (on shared volume) and we mount this shared volume on backup service to perform the remote backup. 29 | 30 | For example, if you have 3 Cassandra nodes on 3 hosts, you must to have sharded storage on each hosts (`/mnt/cassandra`) witch is mounted on each nodes (`/dump`). 31 | Then, you need to mount the shared storage on backup service (`/mnt/cassandra:/backup/cassandra`). 32 | When we detect Cassandra service, we send command to Cassandra to ask it to perform a dump of each nodes on `/dump`, ans next we perform a backup with duplicity of `/backup` folder. 33 | 34 | 35 | 36 | ## Disable dump on specific service 37 | 38 | If you should to not dump a particular service witch is supported, you can add label on service `backup.disable=true` 39 | 40 | ## Backup discovery 41 | 42 | First it search on label called `backup.type` and if doesn't found, it search on image name. 43 | It's mean that if you should backup MySQL database, you need to set label `backup.type=mysql` or use docker image that contains name `mysql`. 44 | 45 | 46 | ## Backup options 47 | 48 | > We use [Confd](https://github.com/kelseyhightower/confd) to configure backup options. 49 | 50 | Confd settings: 51 | - **CONFD_PREFIX_KEY**: The prefix key use by Confd. Default is `/backup` 52 | - **CONFD_BACKEND**: The backend used by Confd. Default is `env` 53 | - **CONFD_NODES**: The nodes to use to access on backend. Defaukt is empty. 54 | 55 | The following options permit to set the backup policy : 56 | - **BACKUP_CRON_schedule**: when you should start backup. For example, to start backup each day set `0 0 0 * * *`. Default is `0 0 0 * * *` 57 | - **BACKUP_MODULE_database**: Allow to auto discover service and perform dump (when Know) before start backup with Duplicity. Default is `true`. 58 | - **BACKUP_MODULE_stack**: Allow to perform export of each stack before start backup with Duplicity. Default is `true`. 59 | - **BACKUP_MODULE_rancher-db**: Allow to perform a dump of Rancher database befaire start backup with Duplicity. Default is `true`. 60 | - **BACKUP_DUPLICITY_source-path**: The path to backup with Duplicity. Default is `/backup`. 61 | - **BACKUP_DUPLICITY_target-path**: The path were store backup on remote backend. The default value is `/`. 62 | - **BACKUP_DUPLICITY_url**: this is the target URL to externalize the backup. For example, to use FTP as external backup set `ftp://login@my-ftp.com` and add environment variable `FTP_PASSWORD`. For Amazon S3, set `s3://host[:port]/bucket_name[/prefix]`. Read the ducplicity man for [all supported backend](http://duplicity.nongnu.org/duplicity.1.html#sect7). There are no default value. 63 | - **BACKUP_DUPLICITY_options**: List of options added when start backup with Duplicity. Is usefull to add SSH options. There are no default value. 64 | - **BACKUP_DUPLICITY_full-if-older-than**: The frequency when you should make a full backup. For example, if you should make a full backup each 7 days, set `7D`. The default value is `7D`. 65 | - **BACKUP_DUPLICITY_remove-all-but-n-full**: How many full backup you should to keep. For example, to keep 3 full backup set `3`. The default value is `3`. 66 | - **BACKUP_DUPLICITY_remove-all-inc-of-but-n-full**: The number of intermediate incremental backup you should keep with the full backup. For example, if you should keep only the incremental backend after the last full backup set `1`. The default value is set to `1`. 67 | - **BACKUP_DUPLICITY_volsize**: The volume size to store the backup (in MB). The default value is `200`. 68 | - **BACKUP_DUPLICITY_encrypt-key**: The GPG key ID to crypt / decrypt backup. You need to set `PASSPHRASE` environment to allow crypt/decrypt without user interaction. You need to mount your GPG keys on `/opt/backup/.gnupg`. 69 | - **BACKUP_RANCHER_db_host**: The rancher database IP/DNS (needed if you should perform Rancher database dump). No default value. 70 | - **BACKUP_RANCHER_db_port**: The rancher database port (needed if you should perform Rancher database dump). Default is `3306`. 71 | - **BACKUP_RANCHER_db_user**: The rancher database user (needed if you should perform Rancher database dump). Default is `rancher`. 72 | - **BACKUP_RANCHER_db_password**: The rancher database password (needed if you should perform Rancher database dump). No default value. 73 | - **BACKUP_RANCHER_db_name**: The rancher database name (needed if you should perform Rancher database dump). Default is `rancher`. 74 | 75 | To set the Rancher API connection prefer to add special label that generate access on the flow: 76 | - `io.rancher.container.create_agent=true` 77 | - `io.rancher.container.agent.role=environment` 78 | 79 | Or you can define them manually : 80 | - **BACKUP_RANCHER_api_url**: the API URL with your project ID 81 | - **BACKUP_RANCHER_api_key**: the API key 82 | - **BACKUP_RANCHER_api_secret**: the API secret key 83 | 84 | ## How to extend this 85 | 86 | You need to dump another service before to save it (note yet supported) ? Just clone this repository and 2 files per service: 87 | - `backup/index/service.yml`: contain the regex to identifiy the new service 88 | - `backup/template/service.yml`: caontain the instruction about how dump the new service. We use Jinja2 templating. 89 | 90 | Then, add your new entry (sample with MySQL): 91 | 92 | backup/index/mysql.yml 93 | ```yaml 94 | mysql: 95 | regex: "mysql" 96 | template: "mysql.yml" 97 | ``` 98 | 99 | Few explanation: 100 | - **regex**: It's the regex to discover service witch must be dumped. This regex is applied first on label called `backup.type` and then to image docker used in service. 101 | - **template**: It's the template to use to perform MySQL dump. 102 | 103 | backup/template/mysql.yml 104 | ```yaml 105 | image: "mysql:latest" 106 | commands: 107 | {% if env.MYSQL_USER and env.MYSQL_DATABASE %} 108 | {# When user, password and database setted #} 109 | - "sh -c 'mysqldump -h {{ ip }} -u {{ env.MYSQL_USER }} {{env.MYSQL_DATABASE}} > {{ target_dir }}/{{ env.MYSQL_DATABASE }}.dump'" 110 | 111 | {% elif env.MYSQL_USER and not env.MYSQL_DATABASE %} 112 | {# When user, password setted #} 113 | - "sh -c 'mysqldump -h {{ ip }} -u {{ env.MYSQL_USER }} --all-databases > {{ target_dir }}/all-databases.dump'" 114 | 115 | {% elif not env.MYSQL_USER and env.MYSQL_DATABASE %} 116 | {# When database setted #} 117 | - "sh -c 'mysqldump -h {{ ip }} -u root {{env.MYSQL_DATABASE}} > {{ target_dir }}/{{ env.MYSQL_DATABASE }}.dump'" 118 | 119 | {% elif not env.MYSQL_USER and not env.MYSQL_DATABASE %} 120 | {# When just root setted #} 121 | - "sh -c 'mysqldump -h {{ ip }} -u root --all-databases > {{ target_dir }}/all-databases.dump'" 122 | 123 | {% endif %} 124 | environments: 125 | {% if env.MYSQL_PASSWORD %} 126 | - MYSQL_PWD:{{ env.MYSQL_PASSWORD }} 127 | {% elif env.MYSQL_ROOT_PASSWORD %} 128 | - MYSQL_PWD:{{ env.MYSQL_ROOT_PASSWORD }} 129 | {% endif %} 130 | ``` 131 | 132 | Few explanation: 133 | - **template**: It's the template to use to perform MySQL dump. 134 | - **image**: It's the docker image to use to run the dump (generaly the latest tag). If you not add image entry, it use the service docker image. 135 | - **commands**: It's the list of commands to launch on container to perform the dump 136 | - **environments**: It's the list of environment variables you need to perform the dump 137 | 138 | We use [Jinja2 templating](http://jinja.pocoo.org/docs/2.9/templates/) and we add some variable that you can use in template: 139 | - `{{ip}}`: the IP to join the container to perform a remote dump 140 | - `{{env.SERVICE_ENV}}`: Take the value of service environment called `SERVICE_ENV` 141 | - `{{target_dir}}`: It's the path where store the dump (`BACKUP_PATH/dump/STACK_NAME/SERVICE_NAME`) 142 | 143 | 144 | # Contribute 145 | 146 | Your PR are welcome, but please use develop branch and not master. -------------------------------------------------------------------------------- /backup/config/index/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | elasticsearch: 2 | regex: "elasticsearch" 3 | template: "elasticsearch.yml" -------------------------------------------------------------------------------- /backup/config/index/mariadb.yml: -------------------------------------------------------------------------------- 1 | mariadb: 2 | regex: "mariadb" 3 | template: "mariadb.yml" -------------------------------------------------------------------------------- /backup/config/index/mongodb.yml: -------------------------------------------------------------------------------- 1 | mongodb: 2 | regex: "mongo" 3 | template: "mongodb.yml" -------------------------------------------------------------------------------- /backup/config/index/mysql.yml: -------------------------------------------------------------------------------- 1 | mysql: 2 | regex: "mysql" 3 | template: "mysql.yml" -------------------------------------------------------------------------------- /backup/config/index/postgresql.yml: -------------------------------------------------------------------------------- 1 | postgresql: 2 | regex: "postgres" 3 | template: "postgresql.yml" -------------------------------------------------------------------------------- /backup/config/rancher-backup.yml: -------------------------------------------------------------------------------- 1 | # Rancher Api 2 | rancher: 3 | api: 4 | url: 5 | key: 6 | secret: 7 | 8 | db: 9 | host: "localhost" 10 | port: 3306 11 | user: "rancher" 12 | db.password: 13 | db.name: "rancher" 14 | 15 | # Backup module 16 | module: 17 | databases: true 18 | stack: true 19 | rancher-db: true 20 | 21 | # Duplicity Policy 22 | duplicity: 23 | source-path: "/backup" 24 | target-path: "/backup" 25 | url: 26 | full-if-older-than: "7D" 27 | remove-all-but-n-full: 3 28 | remove-all-inc-of-but-n-full: 1 29 | volsize: 200 30 | options: 31 | encrypt-key: 32 | 33 | # cron 34 | cron: 35 | schedule: '0 0 0 * * *' 36 | -------------------------------------------------------------------------------- /backup/config/templates/elasticsearch.yml: -------------------------------------------------------------------------------- 1 | image: "taskrabbit/elasticsearch-dump:latest" 2 | commands: 3 | - "-c 'rm -rf {{ target_dir }}/*.json'" 4 | - "-c 'elasticdump --input=http://{{ ip }}:9200/ --output={{ target_dir }}/dump_mapping.json --type=mapping'" 5 | - "-c 'elasticdump --input=http://{{ ip }}:9200/ --output={{ target_dir }}/dump_data.json --type=data'" 6 | entrypoint: "/bin/sh" 7 | -------------------------------------------------------------------------------- /backup/config/templates/mariadb.yml: -------------------------------------------------------------------------------- 1 | image: "mariadb:latest" 2 | commands: 3 | {% if env.MYSQL_USER and env.MYSQL_DATABASE %} 4 | {# When user, password and database setted #} 5 | - "sh -c 'mysqldump -h {{ ip }} -u {{ env.MYSQL_USER }} {{env.MYSQL_DATABASE}} > {{ target_dir }}/{{ env.MYSQL_DATABASE }}.dump'" 6 | 7 | {% elif env.MYSQL_USER and not env.MYSQL_DATABASE %} 8 | {# When user, password setted #} 9 | - "sh -c 'mysqldump -h {{ ip }} -u {{ env.MYSQL_USER }} --all-databases > {{ target_dir }}/all-databases.dump'" 10 | 11 | {% elif not env.MYSQL_USER and env.MYSQL_DATABASE %} 12 | {# When database setted #} 13 | - "sh -c 'mysqldump -h {{ ip }} -u root {{env.MYSQL_DATABASE}} > {{ target_dir }}/{{ env.MYSQL_DATABASE }}.dump'" 14 | 15 | {% elif not env.MYSQL_USER and not env.MYSQL_DATABASE %} 16 | {# When just root setted #} 17 | - "sh -c 'mysqldump -h {{ ip }} -u root --all-databases > {{ target_dir }}/all-databases.dump'" 18 | 19 | {% endif %} 20 | environments: 21 | {% if env.MYSQL_PASSWORD %} 22 | - MYSQL_PWD:{{ env.MYSQL_PASSWORD }} 23 | {% elif env.MYSQL_ROOT_PASSWORD %} 24 | - MYSQL_PWD:{{ env.MYSQL_ROOT_PASSWORD }} 25 | {% endif %} 26 | -------------------------------------------------------------------------------- /backup/config/templates/mongodb.yml: -------------------------------------------------------------------------------- 1 | image: "mongo:latest" 2 | commands: 3 | {% if env.MONGO_USER and env.MONGO_PASS %} 4 | - "mongodump --host {{ ip }} -u {{ env.MONGO_USER }} -p {{ env.MONGO_PASS }} --gzip --archive > {{ target_dir }}/databases.archive.gz" 5 | {% elif env.MONGO_PASS %} 6 | - "mongodump --host {{ ip }} -u admin -p {{ env.MONGO_PASS }} --gzip --archive > {{ target_dir }}/databases.archive.gz" 7 | {% else %} 8 | - "mongodump --host {{ ip }} --gzip --archive > {{ target_dir }}/databases.archive.gz" 9 | {% endif %} 10 | -------------------------------------------------------------------------------- /backup/config/templates/mysql.yml: -------------------------------------------------------------------------------- 1 | image: "mysql:latest" 2 | commands: 3 | {% if env.MYSQL_USER and env.MYSQL_DATABASE %} 4 | {# When user, password and database setted #} 5 | - "sh -c 'mysqldump -h {{ ip }} -u {{ env.MYSQL_USER }} {{env.MYSQL_DATABASE}} > {{ target_dir }}/{{ env.MYSQL_DATABASE }}.dump'" 6 | 7 | {% elif env.MYSQL_USER and not env.MYSQL_DATABASE %} 8 | {# When user, password setted #} 9 | - "sh -c 'mysqldump -h {{ ip }} -u {{ env.MYSQL_USER }} --all-databases > {{ target_dir }}/all-databases.dump'" 10 | 11 | {% elif not env.MYSQL_USER and env.MYSQL_DATABASE %} 12 | {# When database setted #} 13 | - "sh -c 'mysqldump -h {{ ip }} -u root {{env.MYSQL_DATABASE}} > {{ target_dir }}/{{ env.MYSQL_DATABASE }}.dump'" 14 | 15 | {% elif not env.MYSQL_USER and not env.MYSQL_DATABASE %} 16 | {# When just root setted #} 17 | - "sh -c 'mysqldump -h {{ ip }} -u root --all-databases > {{ target_dir }}/all-databases.dump'" 18 | 19 | {% endif %} 20 | environments: 21 | {% if env.MYSQL_PASSWORD %} 22 | - MYSQL_PWD:{{ env.MYSQL_PASSWORD }} 23 | {% elif env.MYSQL_ROOT_PASSWORD %} 24 | - MYSQL_PWD:{{ env.MYSQL_ROOT_PASSWORD }} 25 | {% endif %} 26 | -------------------------------------------------------------------------------- /backup/config/templates/postgresql.yml: -------------------------------------------------------------------------------- 1 | image: "postgres:latest" 2 | commands: 3 | {% if env.POSTGRES_USER and env.POSTGRES_DB %} 4 | {# When user, password and database setted #} 5 | - "pg_dump -h {{ ip }} -U {{ env.POSTGRES_USER }} -d {{ env.POSTGRES_DB }} -f {{ target_dir }}/{{ env.POSTGRES_DB }}.dump" 6 | 7 | {% elif env.POSTGRES_USER and not env.POSTGRES_DB %} 8 | {# When user, password setted #} 9 | - "pg_dump -h {{ ip }} -U {{ env.POSTGRES_USER }} -d {{ env.POSTGRES_USER }} -f {{ target_dir }}/{{ env.POSTGRES_USER }}.dump" 10 | 11 | {% elif not env.POSTGRES_USER and env.POSTGRES_DB %} 12 | {# When database setted, password #} 13 | - "pg_dump -h {{ ip }} -U postgres -d {{ env.POSTGRES_DB }} -f {{ target_dir }}/{{ env.POSTGRES_DB }}.dump" 14 | 15 | {% elif not env.POSTGRES_USER and not env.POSTGRES_DB %} 16 | {# When just root setted #} 17 | - "pg_dumpall -h {{ ip }} -U postgres --clean -f {{ target_dir }}/all-databases.dump" 18 | 19 | {% endif %} 20 | 21 | environments: 22 | {% if env.POSTGRES_PASSWORD %} 23 | - PGPASSWORD:{{ env.POSTGRES_PASSWORD}} 24 | {% endif %} 25 | -------------------------------------------------------------------------------- /backup/requirements.txt: -------------------------------------------------------------------------------- 1 | pyyaml 2 | Jinja2 3 | mock 4 | cattle 5 | azure.storage==0.20.0 6 | -------------------------------------------------------------------------------- /backup/run_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | docker run --rm -ti -e 'PYTHONPATH=/code/src' -v $PWD:/code webcenter/nosetests-python2.7:latest bash -c "pip install --upgrade -r /code/requirements.txt && nosetests -w test/backup" -------------------------------------------------------------------------------- /backup/src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disaster37/rancher-backup/3e589e6e8948309dbb9fce8423707a68db8e98cd/backup/src/__init__.py -------------------------------------------------------------------------------- /backup/src/backup.py: -------------------------------------------------------------------------------- 1 | __author__ = 'disaster' 2 | 3 | import os 4 | import sys 5 | from fr.webcenter.backup.Backup import Backup 6 | from fr.webcenter.backup.Rancher import Rancher 7 | from fr.webcenter.backup.Config import Config 8 | import logging 9 | from logging.handlers import TimedRotatingFileHandler 10 | import traceback 11 | 12 | 13 | 14 | def checkParameters(settings): 15 | """ 16 | Permit to check all parameters 17 | :param settings: The list of settings 18 | :type settings: dict 19 | """ 20 | 21 | if isinstance(settings, dict) is False: 22 | raise KeyError("Settings must be provided") 23 | 24 | if 'module' not in settings: 25 | raise KeyError("You must provide module section on your config file") 26 | if 'databases' not in settings['module'] or isinstance(settings['module']['databases'], bool) is False: 27 | raise KeyError("module.databases must be provided") 28 | if 'stack' not in settings['module'] or isinstance(settings['module']['stack'], bool) is False: 29 | raise KeyError("module.stack must be provided") 30 | if 'rancher-db' not in settings['module'] or isinstance(settings['module']['rancher-db'], bool) is False: 31 | raise KeyError("module.rancher-db must be provided") 32 | if settings['module']['databases'] is True or settings['module']['stack'] is True or settings['module']['rancher-db'] is True: 33 | if 'rancher' not in settings: 34 | raise KeyError("You must provide rancher section on your config file") 35 | if 'api' not in settings['rancher']: 36 | raise KeyError("You must provide rancher.api section on your config file") 37 | if 'url' not in settings['rancher']['api'] or settings['rancher']['api']['url'] is None or settings['rancher']['api']['url'] == "": 38 | raise KeyError("rancher.api.url must be provided") 39 | if 'key' not in settings['rancher']['api'] or settings['rancher']['api']['key'] is None or settings['rancher']['api']['key'] == "": 40 | raise KeyError("rancher.api.key must be provided") 41 | if 'secret' not in settings['rancher']['api'] or settings['rancher']['api']['secret'] is None or settings['rancher']['api']['secret'] == "": 42 | raise KeyError("rancher.api.secret must be provided") 43 | 44 | if 'duplicity' not in settings: 45 | raise KeyError("You must provide duplicity section on your config file") 46 | if 'source-path' not in settings['duplicity'] or settings['duplicity']['source-path'] is None or settings['duplicity']['source-path'] == "": 47 | raise KeyError("duplicity.source-path must be provided") 48 | if 'target-path' not in settings['duplicity'] or settings['duplicity']['target-path'] is None or settings['duplicity']['target-path'] == "": 49 | raise KeyError("duplicity.target-path must be provided") 50 | if 'url' not in settings['duplicity'] or settings['duplicity']['url'] is None or settings['duplicity']['url'] == "": 51 | raise KeyError("duplicity.url must be provided") 52 | if 'full-if-older-than' not in settings['duplicity'] or settings['duplicity']['full-if-older-than'] is None or settings['duplicity']['full-if-older-than'] == "": 53 | raise KeyError("duplicity.full-if-older-than must be provided") 54 | if 'remove-all-but-n-full' not in settings['duplicity'] or isinstance(settings['duplicity']['remove-all-but-n-full'], int) is False: 55 | raise KeyError("duplicity.remove-all-but-n-full must be provided") 56 | if 'remove-all-inc-of-but-n-full' not in settings['duplicity'] or isinstance(settings['duplicity']['remove-all-inc-of-but-n-full'], int) is False: 57 | raise KeyError("duplicity.remove-all-inc-of-but-n-full must be provided") 58 | if 'volsize' not in settings['duplicity'] or isinstance(settings['duplicity']['volsize'], int) is False : 59 | raise KeyError("duplicity.volsize must be provided") 60 | 61 | def checkAndGetDatabaseSettings(settings, rancherDatabaseSettings): 62 | """ 63 | Permit to check that all parameter is setted to dump Rancher database is needed 64 | :param settings: the settings provided by config file 65 | :param rancherDatabaseSettings: the settings provided by Rancher API 66 | :return dict: the database settings 67 | :type settings: dict 68 | :type rancherDatabaseSettings: dict 69 | """ 70 | 71 | if isinstance(settings, dict) is False: 72 | raise KeyError("You must provide settings") 73 | if isinstance(rancherDatabaseSettings, dict) is False: 74 | raise KeyError("You must provide databaseSettings") 75 | 76 | 77 | if settings['module']['rancher-db'] is True: 78 | if 'host' not in rancherDatabaseSettings or 'db' not in rancherDatabaseSettings or 'port' not in rancherDatabaseSettings or 'password' not in rancherDatabaseSettings or 'user' not in rancherDatabaseSettings: 79 | logger.info("Can't grab Rancher database settings from API, try to grab it from config file") 80 | rancherDatabaseSettings = { 81 | 'type': 'mysql' 82 | } 83 | if 'db' not in settings['rancher']: 84 | raise KeyError("You must provide rancher.db section on your config file") 85 | if 'host' not in settings['rancher']['db'] or settings['rancher']['db']['host'] is None or settings['rancher']['db']['host'] == "": 86 | raise KeyError("rancher.db.host must be provided") 87 | else: 88 | rancherDatabaseSettings['host'] = settings['rancher']['db']['host'] 89 | if 'user' not in settings['rancher']['db'] or settings['rancher']['db']['user'] is None or settings['rancher']['db']['user'] == "": 90 | raise KeyError("rancher.db.user must be provided") 91 | else: 92 | rancherDatabaseSettings['user'] = settings['rancher']['db']['user'] 93 | if 'password' not in settings['rancher']['db'] or settings['rancher']['db']['password'] is None or settings['rancher']['db']['password'] == "": 94 | raise KeyError("rancher.db.password must be provided") 95 | else: 96 | rancherDatabaseSettings['password'] = settings['rancher']['db']['password'] 97 | if 'name' not in settings['rancher']['db'] or settings['rancher']['db']['name'] is None or settings['rancher']['db']['name'] == "": 98 | raise KeyError("rancher.db.name must be provided") 99 | else: 100 | rancherDatabaseSettings['name'] = settings['rancher']['db']['name'] 101 | if 'port' not in settings['rancher']['db'] or isinstance(settings['rancher']['db']['port'], int) is False: 102 | raise KeyError("rancher.db.port must be provided") 103 | else: 104 | rancherDatabaseSettings['port'] = settings['rancher']['db']['port'] 105 | 106 | return rancherDatabaseSettings 107 | else: 108 | return None 109 | 110 | 111 | 112 | def getAndcheckAllParameters(): 113 | configService = Config() 114 | settings = configService.getSettings() 115 | 116 | try: 117 | checkParameters(settings) 118 | except KeyError as e: 119 | raise Exception("Somthing wrong on your config file: %s" % e.message) 120 | 121 | 122 | logger.info("Rancher URL: %s", settings['rancher']['api']['url'][:-2] + "v2-beta") 123 | logger.info("Rancher key: %s", settings['rancher']['api']['key']) 124 | logger.info("Rancher secret: XXXX") 125 | logger.info("Backup path: %s", settings['duplicity']['source-path']) 126 | logger.info("Backup target path: %s", settings['duplicity']['target-path']) 127 | logger.info("Backend to receive remote backup: %s", settings['duplicity']['url']) 128 | logger.info("Backup full frequency: %s", settings['duplicity']['full-if-older-than']) 129 | logger.info("Backup full to keep: %s", settings['duplicity']['remove-all-but-n-full']) 130 | logger.info("Backup incremental chain to keep: %s", settings['duplicity']['remove-all-inc-of-but-n-full']) 131 | logger.info("Volume size: %s", settings['duplicity']['volsize']) 132 | logger.info("Backup options: %s", settings['duplicity']['options']) 133 | 134 | # Init services 135 | try: 136 | rancherService = Rancher(settings['rancher']['api']['url'][:-2] + "v2-beta", settings['rancher']['api']['key'], settings['rancher']['api']['secret']) 137 | except Exception as e: 138 | raise Exception("Can't connect to rancher API : %s \n%s" % (e.message, traceback.format_exc())) 139 | 140 | try: 141 | rancherDatabaseSettings = rancherService.getDatabaseSettings() 142 | except Exception as e: 143 | rancherDatabaseSettings = {} 144 | pass 145 | 146 | # Check database settings 147 | try: 148 | rancherDatabaseSettings = checkAndGetDatabaseSettings(settings, rancherDatabaseSettings) 149 | except KeyError as e: 150 | raise Exception("You must set the Rancher database settings on config file to dump it: %s" % e.message) 151 | 152 | 153 | 154 | return (settings, rancherDatabaseSettings) 155 | 156 | 157 | 158 | if __name__ == '__main__': 159 | # Init logger 160 | # We init logger 161 | if os.getenv('DEBUG') is not None and os.getenv('DEBUG') == "true": 162 | loglevel = logging.getLevelName("DEBUG") 163 | else: 164 | loglevel = logging.getLevelName("INFO") 165 | logger = logging.getLogger() 166 | logger.setLevel(loglevel) 167 | 168 | formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') 169 | console_handler = logging.StreamHandler() 170 | console_handler.setLevel(loglevel) 171 | console_handler.setFormatter(formatter) 172 | logger.addHandler(console_handler) 173 | file_handler = TimedRotatingFileHandler( 174 | '/var/log/backup/backup.log', 175 | when='d', 176 | interval=1, 177 | backupCount=5 178 | ) 179 | file_handler.setLevel(loglevel) 180 | file_handler.setFormatter(formatter) 181 | logger.addHandler(file_handler) 182 | 183 | # Load and check settings 184 | configService = Config("config") 185 | 186 | # Just check parameters 187 | if len(sys.argv) > 1 and sys.argv[1] == "--checkParameters": 188 | 189 | try: 190 | getAndcheckAllParameters() 191 | except Exception as e: 192 | logger.error("Error - %s", e.message) 193 | sys.exit(1) 194 | 195 | # Run backup 196 | else: 197 | 198 | try: 199 | (settings, rancherDatabaseSettings) = getAndcheckAllParameters() 200 | except Exception as e: 201 | logger.error("Error - %s", e.message) 202 | sys.exit(1) 203 | 204 | backupService = Backup() 205 | rancherService = Rancher() 206 | backend = "%s%s" % (settings['duplicity']['url'], settings['duplicity']['target-path']) 207 | 208 | try: 209 | # We init duplicity 210 | try: 211 | logger.info("Start to initialize Duplicity...") 212 | backupService.initDuplicity(settings['duplicity']['source-path'], backend) 213 | logger.info("Duplicity initialization is finished.") 214 | except Exception as e: 215 | logger.info("No backup found (probably the first) or already initialized") 216 | pass 217 | 218 | 219 | # We dump the databases services if needed 220 | if settings['module']['databases'] is True: 221 | logger.info("Start to dump databases...") 222 | listServices = rancherService.getServices() 223 | listDump = backupService.searchDump(settings['duplicity']['source-path'] + '/dump', listServices) 224 | backupService.runDump(listDump) 225 | logger.info("The dumping databases is finished.") 226 | 227 | # We dump the rancher stack settings if needed 228 | if settings['module']['stack'] is True: 229 | logger.info("Start to export stack as json...") 230 | listStacks = rancherService.getStacks() 231 | backupService.dumpStacksSettings(settings['duplicity']['source-path'] + '/rancher', listStacks) 232 | logger.info("The exporting of stack if finished") 233 | 234 | 235 | # We dump the rancher database if needed 236 | if settings['module']['rancher-db'] is True: 237 | logger.info("Start to dump Rancher database...") 238 | backupService.dumpRancherDatabase(settings['duplicity']['source-path'] + '/rancher', rancherDatabaseSettings) 239 | logger.info("The Rancher database dumping is finished.") 240 | 241 | # We run the backup 242 | logger.info("Start to externalize the backup with Duplicity...") 243 | backupService.runDuplicity(settings['duplicity']['source-path'], backend, settings['duplicity']['full-if-older-than'], settings['duplicity']['remove-all-but-n-full'], settings['duplicity']['remove-all-inc-of-but-n-full'], settings['duplicity']['volsize'], settings['duplicity']['options'], settings['duplicity']['encrypt-key']) 244 | logger.info("The backup exporing is finished.") 245 | 246 | 247 | except Exception as e: 248 | logger.error("Unattented error occur : %s", e.message) 249 | logger.error(traceback.format_exc()) 250 | sys.exit(1) 251 | -------------------------------------------------------------------------------- /backup/src/fr/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disaster37/rancher-backup/3e589e6e8948309dbb9fce8423707a68db8e98cd/backup/src/fr/__init__.py -------------------------------------------------------------------------------- /backup/src/fr/webcenter/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disaster37/rancher-backup/3e589e6e8948309dbb9fce8423707a68db8e98cd/backup/src/fr/webcenter/__init__.py -------------------------------------------------------------------------------- /backup/src/fr/webcenter/backup/Backup.py: -------------------------------------------------------------------------------- 1 | __author__ = 'disaster' 2 | 3 | import re 4 | import logging 5 | import os 6 | from fr.webcenter.backup.Command import Command 7 | from fr.webcenter.backup.Singleton import Singleton 8 | from fr.webcenter.backup.Config import Config 9 | from jinja2 import Environment 10 | import yaml 11 | 12 | 13 | logger = logging 14 | class Backup(object): 15 | """ 16 | This class permit to drive dump and backup on remote target 17 | """ 18 | 19 | __metaclass__ = Singleton 20 | 21 | def searchDump(self, backupPath, listServices): 22 | """ 23 | This class search service where to perform a dump before to backup them and grab all setting to perform backup 24 | :param backupPath: The path where to store the dump 25 | :param listServices: The list of all services provider by Rancher API 26 | :type backupPath: str 27 | :type listServices: list 28 | :return dict The list of service where to perform dump and docker command line associated to them. 29 | """ 30 | 31 | if backupPath is None or backupPath == "": 32 | raise KeyError("backupPath must be provided") 33 | if isinstance(listServices, list) is False: 34 | raise KeyError("listServices must be a list") 35 | 36 | 37 | logger.debug("backupPath: %s", backupPath) 38 | logger.debug("listServices: %s", listServices) 39 | 40 | configService = Config() 41 | index = configService.getIndex() 42 | 43 | listDump = [] 44 | 45 | for service in listServices: 46 | for name, setting in index.iteritems(): 47 | # First search on label and then search on image name 48 | if ('labels' in service['launchConfig'] and 'backup.type' in service['launchConfig']['labels'] and re.search(setting['regex'], service['launchConfig']['labels']['backup.type'])) or re.search(setting['regex'], service['launchConfig']['imageUuid']): 49 | 50 | try: 51 | 52 | logger.info("Found '%s/%s' to do dumping" % (service['stack']['name'], service['name'])) 53 | template = configService.getTemplate(setting['template']) 54 | 55 | env = Environment() 56 | template = env.from_string(template) 57 | context = {} 58 | 59 | # Get environment variables 60 | if 'environment' in service['launchConfig']: 61 | context["env"] = service['launchConfig']['environment'] 62 | else: 63 | context["env"] = {} 64 | 65 | # Get IP 66 | for instance in service['instances']: 67 | if instance['state'] == "running": 68 | context["ip"] = instance['primaryIpAddress'] 69 | logger.debug("Found IP %s", context["ip"]) 70 | break 71 | 72 | # Get Taget backup 73 | context["target_dir"] = backupPath + "/" + service['stack']['name'] + "/" + service['name'] 74 | 75 | setting = yaml.load(template.render(context)) 76 | 77 | setting["service"] = service 78 | setting["target_dir"] = context["target_dir"] 79 | if "environments" not in setting or not setting["environments"]: 80 | setting["environments"] = [] 81 | if "image" not in setting: 82 | setting["image"] = service['launchConfig']['imageUuid'] 83 | 84 | listDump.append(setting) 85 | break 86 | 87 | except Excexption as e: 88 | logger.error("Error appear when extract infos from Rancher API about '%s/%s', skip : %s" % (service['stack']['name'], service['name'], e.message)) 89 | # Don't beack backup if somethink wrong 90 | pass 91 | 92 | logger.debug(listDump) 93 | 94 | return listDump 95 | 96 | 97 | def runDump(self, listDump): 98 | """ 99 | Permit to perform dump on each services with Docker command 100 | :param listDump: The list of service where to perform the dump 101 | :type listDump: list 102 | """ 103 | 104 | if isinstance(listDump, list) is False: 105 | raise KeyError("listDump must be a list") 106 | 107 | logger.debug("listDump: %s", listDump) 108 | 109 | commandService = Command() 110 | 111 | 112 | for dump in listDump: 113 | 114 | try: 115 | logger.info("Dumping %s/%s in %s" % (dump['service']['stack']['name'], dump['service']['name'], dump['target_dir'])) 116 | environments = "" 117 | for env in dump['environments']: 118 | environments += " -e '%s'" % env.replace(':', '=') 119 | 120 | 121 | if 'entrypoint' in dump: 122 | entrypoint = "--entrypoint='%s'" % dump['entrypoint'] 123 | else: 124 | entrypoint = '' 125 | 126 | # Check if folder to receive dump exist, else create it 127 | if os.path.isdir(dump['target_dir']) is False: 128 | os.makedirs(dump['target_dir']) 129 | logger.debug("Create directory '%s'", dump['target_dir']) 130 | else: 131 | logger.debug("Directory '%s' already exist", dump['target_dir']) 132 | 133 | commandService.runCmd("docker pull %s" % dump['image']) 134 | 135 | for command in dump['commands']: 136 | dockerCmd = "docker run --rm %s -v %s:%s %s %s %s" % (entrypoint, dump['target_dir'], dump['target_dir'], environments, dump['image'], command) 137 | commandService.runCmd(dockerCmd) 138 | logger.info("Dump %s/%s is finished" % (dump['service']['stack']['name'], dump['service']['name'])) 139 | 140 | except Exception as e: 141 | logger.error("Error appear when dump '%s/%s', skip : %s" % (dump['service']['stack']['name'], dump['service']['name'], e.message)) 142 | # Don't beack backup if somethink wrong 143 | pass 144 | 145 | 146 | def initDuplicity(self, backupPath, backend): 147 | """ 148 | Permit to init Duplicity with the target 149 | :param backupPath: the path where dump is stored 150 | :param backend: the full URL of remote target where to store backup 151 | :type backupPath: str 152 | :type backend: str 153 | """ 154 | 155 | if backupPath is None or backupPath == "": 156 | raise KeyError("backupPath must be provided") 157 | if backend is None or backend == "": 158 | raise KeyError("backend must be provided") 159 | 160 | logger.debug("backupPath: %s", backupPath) 161 | logger.debug("backend: %s", backend) 162 | 163 | commandService = Command() 164 | 165 | result = commandService.runCmd("duplicity --no-encryption %s %s" % (backend, backupPath)) 166 | logger.info(result) 167 | 168 | 169 | 170 | def runDuplicity(self, backupPath, backend, fullBackupFrequency, fullBackupKeep, incrementalBackupChainKeep, volumeSize, options=None, encryptKey=None): 171 | """ 172 | Permit to backup the dump on remote target 173 | :param backupPath: the path where dump is stored 174 | :param backend: the full URL of remote target where to store backup 175 | :param fullBackupFrequency: when run full backup 176 | :param fullBackupKeep: how many full backup to keep 177 | :param incrementalBackupChainKeep: how many incremental backup chain to keep 178 | :param volumeSize: how many size for each volume 179 | :param options: set some duplicity options 180 | :param encryptKey: The GPG key if you should crypt backup 181 | :type backupPath: str 182 | :type backend: str 183 | :type fullBackupFrequency: str 184 | :type incrementalBackupChainKeep: str 185 | :type volumeSize: str 186 | :type options: str 187 | :type encryptKey: str 188 | """ 189 | 190 | if backupPath is None or backupPath == "": 191 | raise KeyError("backupPath must be provided") 192 | if backend is None or backend == "": 193 | raise KeyError("backend must be provided") 194 | if fullBackupFrequency is None or fullBackupFrequency == "": 195 | raise KeyError("fullBackupFrequency must be provided") 196 | if isinstance(fullBackupKeep, int) is False: 197 | raise KeyError("fullBackupKeep must be provided") 198 | if isinstance(incrementalBackupChainKeep, int) is False: 199 | raise KeyError("incrementalBackupChainKeep must be provided") 200 | if isinstance(volumeSize, int) is False: 201 | raise KeyError("volumeSize must be provided") 202 | if options is None: 203 | options = "" 204 | if options is not None and isinstance(options, basestring) is False: 205 | raise KeyError("options must be a None or string") 206 | if encryptKey is not None and isinstance(encryptKey, basestring) is False: 207 | raise KeyError("encryptKey must be a None or string") 208 | 209 | logger.debug("backupPath: %s", backupPath) 210 | logger.debug("backend: %s", backend) 211 | logger.debug("fullBackupFrequency: %s", fullBackupFrequency) 212 | logger.debug("fullBackupKeep: %s", fullBackupKeep) 213 | logger.debug("incrementalBackupChainKeep: %s", incrementalBackupChainKeep) 214 | logger.debug("volumeSize: %s", volumeSize) 215 | logger.debug("options: %s", options) 216 | logger.debug("encryptKey: %s", encryptKey) 217 | 218 | if encryptKey is None or encryptKey == "": 219 | crypt = "--no-encryption" 220 | else: 221 | crypt = "--encrypt-key %s" % encryptKey 222 | 223 | 224 | commandService = Command() 225 | 226 | logger.info("Start backup") 227 | result = commandService.runCmd("duplicity %s --volsize %s %s --allow-source-mismatch --full-if-older-than %s %s %s" % (options, volumeSize, crypt, fullBackupFrequency, backupPath, backend)) 228 | logger.info(result) 229 | 230 | logger.info("Clean old full backup is needed") 231 | result = commandService.runCmd("duplicity remove-all-but-n-full %s --force --allow-source-mismatch %s %s" % (fullBackupKeep, crypt, backend)) 232 | logger.info(result) 233 | 234 | logger.info("Clean old incremental backup if needed") 235 | result = commandService.runCmd("duplicity remove-all-inc-of-but-n-full %s --force --allow-source-mismatch %s %s" % (incrementalBackupChainKeep, crypt, backend)) 236 | logger.info(result) 237 | 238 | logger.info("Cleanup backup") 239 | result = commandService.runCmd("duplicity cleanup --force %s %s" % (crypt, backend)) 240 | logger.info(result) 241 | 242 | 243 | def dumpStacksSettings(self,backupPath, listEnvironments): 244 | """ 245 | Permit to write the stack setting in docker-compose file and rancher-compose file. 246 | :param backupPath: the backup path where store the stack setting extraction 247 | :param listEnvironments: the list of stack 248 | :type backupPath: str 249 | :type listEnvironments: list 250 | """ 251 | 252 | if backupPath is None or backupPath == "": 253 | raise KeyError("backupPath must be provided") 254 | if isinstance(listEnvironments, list) is False: 255 | raise KeyError("listEnvironments must be provided") 256 | 257 | for environment in listEnvironments: 258 | 259 | try: 260 | 261 | targetDir = "%s/%s" % (backupPath, environment['name']) 262 | logger.info("Save the Rancher setting for stack %s in %s", environment['name'], targetDir) 263 | 264 | if os.path.isdir(targetDir) is False: 265 | os.makedirs(targetDir) 266 | logger.debug("Create directory '%s'", targetDir) 267 | else: 268 | logger.debug("Directory '%s' already exist", targetDir) 269 | 270 | # Save docker-compose 271 | fp = open(targetDir + '/docker-compose.yml', 'w') 272 | fp.write(environment['settings']['dockerComposeConfig']) 273 | fp.close() 274 | 275 | # Save rancher-compose 276 | fp = open(targetDir + '/rancher-compose.yml', 'w') 277 | fp.write(environment['settings']['rancherComposeConfig']) 278 | fp.close() 279 | 280 | except Exception as e: 281 | logger.error("Error appear when save setting for stack '%s', skip : %s" % (environment['name'], e.message)) 282 | pass 283 | 284 | 285 | def dumpRancherDatabase(self, backupPath, listDatabaseSettings): 286 | """ 287 | Permit to dump Rancher database 288 | :param backupPath: the backup path where store the database dump 289 | :param listDatabaseSettings: the database parameters to connect on it 290 | :type backupPath: basestring 291 | :type listDatabaseSettings: dict 292 | """ 293 | 294 | if backupPath is None or backupPath == "": 295 | raise KeyError("backupPath must be provided") 296 | if isinstance(listDatabaseSettings, dict) is False: 297 | raise KeyError("listDatabaseSettings must be provided") 298 | 299 | if "type" not in listDatabaseSettings: 300 | raise KeyError("You must provide the database type") 301 | if "host" not in listDatabaseSettings: 302 | raise KeyError("You must provide the database host") 303 | if "port" not in listDatabaseSettings: 304 | raise KeyError("You must provide the database port") 305 | if "user" not in listDatabaseSettings: 306 | raise KeyError("You must provide the database user") 307 | if "password" not in listDatabaseSettings: 308 | raise KeyError("You must provide the database password") 309 | if "name" not in listDatabaseSettings: 310 | raise KeyError("You must provide the database name") 311 | 312 | commandService = Command() 313 | target_dir = "%s/database" % (backupPath) 314 | image = "mysql:latest" 315 | logger.info("Dumping the Rancher database '%s' in '%s'", listDatabaseSettings['name'], target_dir) 316 | 317 | if os.path.isdir(target_dir) is False: 318 | os.makedirs(target_dir) 319 | logger.debug("Create directory '%s'", target_dir) 320 | else: 321 | logger.debug("Directory '%s' already exist", target_dir) 322 | 323 | commandService.runCmd("docker pull %s" % image) 324 | command = "sh -c 'mysqldump -h %s -P %s -u %s %s > %s/%s.dump'" % (listDatabaseSettings['host'], listDatabaseSettings['port'], listDatabaseSettings['user'], listDatabaseSettings['name'], target_dir, listDatabaseSettings['name']) 325 | dockerCmd = "docker run --rm -v %s:%s -e 'MYSQL_PWD=%s' %s %s" % (target_dir, target_dir, listDatabaseSettings['password'], image, command) 326 | commandService.runCmd(dockerCmd) 327 | logger.info("Dump Rancher database is finished") 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | -------------------------------------------------------------------------------- /backup/src/fr/webcenter/backup/Command.py: -------------------------------------------------------------------------------- 1 | __author__ = 'disaster' 2 | 3 | import subprocess 4 | import logging 5 | from fr.webcenter.backup.Singleton import Singleton 6 | 7 | 8 | logger = logging 9 | class Command(object): 10 | """ 11 | Class permit to lauch shell command and control the return code. 12 | """ 13 | 14 | __metaclass__ = Singleton 15 | 16 | def runCmd(self, cmd): 17 | """ 18 | Permit to run system command and get the result 19 | :type cmd: str 20 | :param cmd: the command to excute 21 | :return str: the answere when run the command 22 | """ 23 | 24 | if cmd is None or cmd == "": 25 | raise KeyError("You must set cmd") 26 | 27 | logger.debug('Command to exec : %s', cmd) 28 | 29 | 30 | p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 31 | out, err = p.communicate() 32 | 33 | logger.debug("Results : %s", out) 34 | 35 | 36 | if p.returncode != 0: 37 | raise Exception("Error when run cmd " + cmd + " : " + out + "\n" + err) 38 | 39 | out = out.decode('utf-8') 40 | 41 | return out -------------------------------------------------------------------------------- /backup/src/fr/webcenter/backup/Config.py: -------------------------------------------------------------------------------- 1 | __author__ = 'disaster' 2 | 3 | import yaml 4 | import glob 5 | import logging 6 | from fr.webcenter.backup.Singleton import Singleton 7 | from jinja2 import Environment 8 | 9 | 10 | logger = logging 11 | class Config(object): 12 | 13 | __metaclass__ = Singleton 14 | 15 | def __init__(self, path=None): 16 | """ 17 | Permit to load all settings 18 | :param path: the base path where settings is stored 19 | :type path: basestring 20 | """ 21 | 22 | 23 | # Load main settings 24 | self._settings, self._index, self._templates = self._load(path) 25 | 26 | self._path = path 27 | 28 | def _load(self, path): 29 | 30 | if isinstance(path, basestring) is False: 31 | raise KeyError("Path must be a string") 32 | 33 | contendSetting = open(path + '/rancher-backup.yml', 'r').read() 34 | settings = yaml.load(contendSetting) 35 | 36 | # Load index settings 37 | contendIndex = "" 38 | for file in glob.glob(path + '/index/*.yml'): 39 | contendIndex += open(file, 'r').read() + "\n" 40 | 41 | logger.debug("Index settings : %s", contendIndex) 42 | indexes = yaml.load(contendIndex) 43 | 44 | # Load templates 45 | contendTemplates = {} 46 | for file in glob.glob(path + '/templates/*'): 47 | contendTemplates[file] = open(file, 'r').read() 48 | templates = contendTemplates 49 | 50 | return (settings, indexes, templates) 51 | 52 | 53 | def getSettings(self): 54 | """ 55 | Permit to get the settings 56 | :return dict: the settings 57 | """ 58 | if isinstance(self._settings, dict) is False: 59 | raise Exception("You must init Config first") 60 | 61 | logger.debug("return: %s", self._settings) 62 | return self._settings 63 | 64 | def getIndex(self): 65 | """ 66 | Permit to get the index 67 | :return dict: the index 68 | """ 69 | if isinstance(self._index, dict) is False: 70 | raise Exception("You must init Config first") 71 | 72 | logger.debug("return : %s", self._index) 73 | return self._index 74 | 75 | def getTemplate(self, templateFile): 76 | """ 77 | Permit to get the template by name 78 | :param templateFile: The template filename that needed 79 | :type templateFile: basestring 80 | :return basestring: the template 81 | """ 82 | if self._templates is None: 83 | raise Exception("You must init Config first") 84 | 85 | if templateFile is None or templateFile == "": 86 | raise KeyError("You must provide templateFile") 87 | 88 | templateFile = "%s/templates/%s" % (self._path, templateFile) 89 | 90 | if templateFile in self._templates: 91 | logger.debug("return: %s", self._templates[templateFile]) 92 | return self._templates[templateFile] 93 | else: 94 | raise Exception("Template %s not found", templateFile) 95 | 96 | 97 | -------------------------------------------------------------------------------- /backup/src/fr/webcenter/backup/Rancher.py: -------------------------------------------------------------------------------- 1 | __author__ = 'disaster' 2 | 3 | import cattle 4 | import logging 5 | from fr.webcenter.backup.Singleton import Singleton 6 | 7 | logger = logging 8 | class Rancher(object): 9 | """ 10 | This class provide helper for Rancher API. 11 | """ 12 | 13 | __metaclass__ = Singleton 14 | 15 | def __init__(self, url=None, key=None, secret=None): 16 | """ 17 | Init the connexion on Rancher API 18 | :param url: the Rancher URL to access on API 19 | :param key: The rancher key to access on API 20 | :param secret: The Rancher secret to access on API 21 | :type url: str 22 | :type key: str 23 | :type secret: str 24 | """ 25 | 26 | logger.debug("url: %s", url) 27 | logger.debug("key: %s", key) 28 | logger.debug("secret: xxxx") 29 | 30 | self._client = cattle.Client(url=url, access_key=key, secret_key=secret) 31 | 32 | 33 | def getServices(self): 34 | """ 35 | Get all services and many information associatied to them (stacks, instances and hosts) 36 | :return list The list of services 37 | """ 38 | 39 | listServices = self._client.list('service') 40 | 41 | # We keep only enable services and services that have not 'backup.disable' label set to true 42 | targetListServices = [] 43 | for service in listServices: 44 | if service["type"] == "service" and "imageUuid" in service['launchConfig'] and service["state"] == "active" and ("labels" not in service['launchConfig'] or ("backup.disable" not in service['launchConfig']['labels'] or service['launchConfig']['labels']['backup.disable'] == "false")): 45 | 46 | logger.debug("Found service %s", service["name"]) 47 | 48 | # We add the stack associated to it 49 | if 'stack' in service['links']: 50 | service['stack'] = self._client._get(service['links']['stack']) 51 | logger.debug("Service %s is on stack %s", service["name"], service['stack']['name']) 52 | else: 53 | logger.debug("No stack for service %s", service["name"]) 54 | 55 | 56 | # We add the instances 57 | if 'instances' in service['links']: 58 | service['instances'] = self._client._get(service['links']['instances']) 59 | for instance in service['instances']: 60 | instance['host'] = self._client._get(instance['links']['hosts'])[0] 61 | 62 | logger.debug("Service %s have %d intances", service["name"], len(service['instances'])) 63 | else: 64 | logger.debug("No instance for service %s", service["name"]) 65 | 66 | 67 | targetListServices.append(service) 68 | 69 | 70 | return targetListServices 71 | 72 | 73 | def getStacks(self): 74 | """ 75 | Permit to get all stack on Rancher environment 76 | :return list: the list of Rancher stack 77 | """ 78 | 79 | listStacks = self._client.list('stack') 80 | targetListStacks = [] 81 | for stack in listStacks: 82 | if stack['type'] == 'stack': 83 | logger.debug("Grab setting for stack %s", stack['name']) 84 | stack['settings'] = self._client.action(stack, 'exportconfig') 85 | targetListStacks.append(stack) 86 | else: 87 | logger.info("Skip Grab setting for stack %s because it's %s type", stack['name'], stack['type']) 88 | 89 | 90 | logger.debug("Return: %s", targetListStacks) 91 | 92 | return targetListStacks 93 | 94 | 95 | def getDatabaseSettings(self): 96 | """ 97 | Permit to get the Rancher database settings to perform backup on it 98 | :return dict: the database settings to connect on it 99 | """ 100 | 101 | listSettings = self._client.list('setting') 102 | listTargetSettings = {} 103 | 104 | for setting in listSettings: 105 | if setting["name"] == "cattle.db.cattle.database": 106 | listTargetSettings["type"] = setting["activeValue"] 107 | elif setting["name"] == "cattle.db.cattle.mysql.host": 108 | listTargetSettings["host"] = setting["activeValue"] 109 | elif setting["name"] == "cattle.db.cattle.mysql.name": 110 | listTargetSettings["db"] = setting["activeValue"] 111 | elif setting["name"] == "cattle.db.cattle.mysql.port": 112 | listTargetSettings["port"] = setting["activeValue"] 113 | elif setting["name"] == "cattle.db.cattle.password": 114 | listTargetSettings["password"] = setting["activeValue"] 115 | elif setting["name"] == "cattle.db.cattle.username": 116 | listTargetSettings["user"] = setting["activeValue"] 117 | 118 | return listTargetSettings 119 | 120 | 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /backup/src/fr/webcenter/backup/Singleton.py: -------------------------------------------------------------------------------- 1 | __author__ = 'disaster' 2 | 3 | class Singleton(type): 4 | def __init__(cls,name,bases,dic): 5 | super(Singleton,cls).__init__(name,bases,dic) 6 | cls.instance=None 7 | def __call__(cls,*args,**kw): 8 | if cls.instance is None: 9 | cls.instance=super(Singleton,cls).__call__(*args,**kw) 10 | return cls.instance 11 | 12 | def _drop(cls): 13 | cls.instance=None 14 | 15 | -------------------------------------------------------------------------------- /backup/src/fr/webcenter/backup/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disaster37/rancher-backup/3e589e6e8948309dbb9fce8423707a68db8e98cd/backup/src/fr/webcenter/backup/__init__.py -------------------------------------------------------------------------------- /backup/test/backup/TestBackup.py: -------------------------------------------------------------------------------- 1 | __author__ = 'disaster' 2 | 3 | import unittest 4 | import mock 5 | import os 6 | import logging 7 | import sys 8 | import yaml 9 | import io 10 | from mock import mock_open 11 | import __builtin__ 12 | from fr.webcenter.backup.Backup import Backup 13 | from fr.webcenter.backup.Command import Command 14 | from fr.webcenter.backup.Config import Config 15 | 16 | logger = logging.getLogger() 17 | logger.level = logging.DEBUG 18 | stream_handler = logging.StreamHandler(sys.stdout) 19 | logger.addHandler(stream_handler) 20 | 21 | def fakeInitDuplicity(cmd): 22 | return cmd 23 | 24 | 25 | def fakeSetting(path=None): 26 | 27 | Config._index = { 28 | 'postgresql': { 29 | 'regex': 'postgres', 30 | 'template': 'postgres.yml' 31 | } 32 | } 33 | 34 | Config._templates = { 35 | path + '/templates/postgres.yml': """ 36 | image: "postgres:latest" 37 | commands: 38 | {% if env.POSTGRES_USER and env.POSTGRES_DB %} 39 | {# When user, password and database setted #} 40 | - "pg_dump -h {{ ip }} -U {{ env.POSTGRES_USER }} -d {{ env.POSTGRES_DB }} -f {{ target_dir }}/{{ env.POSTGRES_DB }}.dump" 41 | 42 | {% elif env.POSTGRES_USER and not env.POSTGRES_DB %} 43 | {# When user, password setted #} 44 | - "pg_dump -h {{ ip }} -U {{ env.POSTGRES_USER }} -d {{ env.POSTGRES_USER }} -f {{ target_dir }}/{{ env.POSTGRES_USER }}.dump" 45 | 46 | {% elif not env.POSTGRES_USER and env.POSTGRES_DB %} 47 | {# When database setted, password #} 48 | - "pg_dump -h {{ ip }} -U postgres -d {{ env.POSTGRES_DB }} -f {{ target_dir }}/{{ env.POSTGRES_DB }}.dump" 49 | 50 | {% elif not env.POSTGRES_USER and not env.POSTGRES_DB %} 51 | {# When just root setted #} 52 | - "pg_dumpall -h {{ ip }} -U postgres --clean -f {{ target_dir }}/all-databases.dump" 53 | 54 | {% endif %} 55 | 56 | environments: 57 | {% if env.POSTGRES_PASSWORD %} 58 | - PGPASSWORD:{{ env.POSTGRES_PASSWORD}} 59 | {% endif %} 60 | """ 61 | } 62 | 63 | Config._path = path 64 | 65 | class TestBackup(unittest.TestCase): 66 | 67 | 68 | @mock.patch.object(Command, 'runCmd', autospec=True) 69 | def testInitDuplicity(self, mock_runCmd): 70 | backupService = Backup() 71 | backupService.initDuplicity('/backup', 'ftp://user:pass@my-server.com/backup/dump') 72 | mock_runCmd.assert_any_call(mock.ANY, 'duplicity --no-encryption ftp://user:pass@my-server.com/backup/dump /backup') 73 | 74 | @mock.patch.object(Command, 'runCmd', autospec=True) 75 | def testRunDuplicity(self, mock_runCmd): 76 | backupService = Backup() 77 | backupService.runDuplicity('/backup', 'ftp://user:pass@my-server.com/backup/dump', '1D', 7, 1, 1000) 78 | mock_runCmd.assert_any_call(mock.ANY, 'duplicity --volsize 1000 --no-encryption --allow-source-mismatch --full-if-older-than 1D /backup ftp://user:pass@my-server.com/backup/dump') 79 | mock_runCmd.assert_any_call(mock.ANY, 'duplicity remove-all-but-n-full 7 --force --allow-source-mismatch --no-encryption ftp://user:pass@my-server.com/backup/dump') 80 | mock_runCmd.assert_any_call(mock.ANY, 'duplicity remove-all-inc-of-but-n-full 1 --force --allow-source-mismatch --no-encryption ftp://user:pass@my-server.com/backup/dump') 81 | mock_runCmd.assert_any_call(mock.ANY, 'duplicity cleanup --force --no-encryption ftp://user:pass@my-server.com/backup/dump') 82 | 83 | @mock.patch.object(Config, '__init__', side_effect=fakeSetting) 84 | def testSearchDump(self, mock_config): 85 | backupService = Backup() 86 | Config('/fake/path') 87 | 88 | # Search template backup with image name 89 | listServices = [ 90 | { 91 | 'type': 'service', 92 | 'name': 'test', 93 | 'state': 'active', 94 | 'launchConfig': { 95 | 'imageUuid': 'test/postgres:latest', 96 | 'environment': { 97 | 'POSTGRES_USER': 'user', 98 | 'POSTGRES_DB':'test', 99 | 'POSTGRES_PASSWORD':'pass' 100 | } 101 | }, 102 | 'links': { 103 | 'environment': 'https://fake/environment', 104 | 'instances': 'https://fake/instances', 105 | }, 106 | 'stack': { 107 | 'name': 'stack-test' 108 | }, 109 | 'instances': [ 110 | { 111 | 'state': 'disabled', 112 | 'primaryIpAddress': '10.0.0.1', 113 | 'host': { 114 | 'name': 'host-1' 115 | }, 116 | 'links': { 117 | 'hosts': 'https://fake/hosts' 118 | } 119 | }, 120 | { 121 | 'state': 'running', 122 | 'primaryIpAddress': '10.0.0.2', 123 | 'host': { 124 | 'name': 'host-1' 125 | }, 126 | 'links': { 127 | 'hosts': 'https://fake/hosts' 128 | } 129 | }, 130 | { 131 | 'state': 'running', 132 | 'primaryIpAddress': '10.0.0.3', 133 | 'host': { 134 | 'name': 'host-1' 135 | }, 136 | 'links': { 137 | 'hosts': 'https://fake/hosts' 138 | } 139 | } 140 | 141 | ], 142 | }, 143 | { 144 | 'type': 'service', 145 | 'name': 'test2', 146 | 'state': 'active', 147 | 'launchConfig': { 148 | 'imageUuid': 'test/mysql:latest', 149 | 'environment': { 150 | 'MYSQL_USER': 'user', 151 | 'MYSQL_DATABASE': 'test', 152 | 'MYSQL_PASSWORD': 'pass' 153 | } 154 | }, 155 | 'links': { 156 | 'environment': 'https://fake/environment', 157 | 'instances': 'https://fake/instances', 158 | }, 159 | 'stack': { 160 | 'name': 'stack-test' 161 | }, 162 | 'instances': [ 163 | { 164 | 'state': 'disabled', 165 | 'primaryIpAddress': '10.1.0.1', 166 | 'host': { 167 | 'name': 'host-1' 168 | }, 169 | 'links': { 170 | 'hosts': 'https://fake/hosts' 171 | } 172 | }, 173 | { 174 | 'state': 'running', 175 | 'primaryIpAddress': '10.1.0.2', 176 | 'host': { 177 | 'name': 'host-1' 178 | }, 179 | 'links': { 180 | 'hosts': 'https://fake/hosts' 181 | } 182 | }, 183 | { 184 | 'state': 'running', 185 | 'primaryIpAddress': '10.1.0.3', 186 | 'host': { 187 | 'name': 'host-1' 188 | }, 189 | 'links': { 190 | 'hosts': 'https://fake/hosts' 191 | } 192 | } 193 | 194 | ], 195 | } 196 | ] 197 | 198 | 199 | result = backupService.searchDump('/tmp/backup',listServices) 200 | 201 | targetResult = [ 202 | { 203 | 'service': listServices[0], 204 | 'target_dir': '/tmp/backup/stack-test/test', 205 | 'commands': [ 206 | 'pg_dump -h 10.0.0.2 -U user -d test -f /tmp/backup/stack-test/test/test.dump' 207 | ], 208 | 'environments': ['PGPASSWORD:pass'], 209 | 'image': 'postgres:latest' 210 | } 211 | ] 212 | 213 | self.assertEqual(targetResult, result) 214 | 215 | 216 | # Search backup with labels 217 | # Search template backup with image name 218 | listServices = [ 219 | { 220 | 'type': 'service', 221 | 'name': 'test', 222 | 'state': 'active', 223 | 'launchConfig': { 224 | 'imageUuid': 'test/my-db:latest', 225 | 'environment': { 226 | 'POSTGRES_USER': 'user', 227 | 'POSTGRES_DB':'test', 228 | 'POSTGRES_PASSWORD':'pass' 229 | }, 230 | 'labels': { 231 | 'backup.type': 'postgres' 232 | } 233 | }, 234 | 'links': { 235 | 'environment': 'https://fake/environment', 236 | 'instances': 'https://fake/instances', 237 | }, 238 | 'stack': { 239 | 'name': 'stack-test' 240 | }, 241 | 'instances': [ 242 | { 243 | 'state': 'disabled', 244 | 'primaryIpAddress': '10.0.0.1', 245 | 'host': { 246 | 'name': 'host-1' 247 | }, 248 | 'links': { 249 | 'hosts': 'https://fake/hosts' 250 | } 251 | }, 252 | { 253 | 'state': 'running', 254 | 'primaryIpAddress': '10.0.0.2', 255 | 'host': { 256 | 'name': 'host-1' 257 | }, 258 | 'links': { 259 | 'hosts': 'https://fake/hosts' 260 | } 261 | }, 262 | { 263 | 'state': 'running', 264 | 'primaryIpAddress': '10.0.0.3', 265 | 'host': { 266 | 'name': 'host-1' 267 | }, 268 | 'links': { 269 | 'hosts': 'https://fake/hosts' 270 | } 271 | } 272 | 273 | ], 274 | }, 275 | { 276 | 'type': 'service', 277 | 'name': 'test2', 278 | 'state': 'active', 279 | 'launchConfig': { 280 | 'imageUuid': 'test/my-db:latest', 281 | 'environment': { 282 | 'MYSQL_USER': 'user', 283 | 'MYSQL_DATABASE': 'test', 284 | 'MYSQL_PASSWORD': 'pass' 285 | }, 286 | 'labels': { 287 | 'backup.type': 'mysql' 288 | } 289 | }, 290 | 'links': { 291 | 'environment': 'https://fake/environment', 292 | 'instances': 'https://fake/instances', 293 | }, 294 | 'stack': { 295 | 'name': 'stack-test' 296 | }, 297 | 'instances': [ 298 | { 299 | 'state': 'disabled', 300 | 'primaryIpAddress': '10.1.0.1', 301 | 'host': { 302 | 'name': 'host-1' 303 | }, 304 | 'links': { 305 | 'hosts': 'https://fake/hosts' 306 | } 307 | }, 308 | { 309 | 'state': 'running', 310 | 'primaryIpAddress': '10.1.0.2', 311 | 'host': { 312 | 'name': 'host-1' 313 | }, 314 | 'links': { 315 | 'hosts': 'https://fake/hosts' 316 | } 317 | }, 318 | { 319 | 'state': 'running', 320 | 'primaryIpAddress': '10.1.0.3', 321 | 'host': { 322 | 'name': 'host-1' 323 | }, 324 | 'links': { 325 | 'hosts': 'https://fake/hosts' 326 | } 327 | } 328 | 329 | ], 330 | } 331 | ] 332 | 333 | 334 | result = backupService.searchDump('/tmp/backup',listServices) 335 | 336 | targetResult = [ 337 | { 338 | 'service': listServices[0], 339 | 'target_dir': '/tmp/backup/stack-test/test', 340 | 'commands': [ 341 | 'pg_dump -h 10.0.0.2 -U user -d test -f /tmp/backup/stack-test/test/test.dump' 342 | ], 343 | 'environments': ['PGPASSWORD:pass'], 344 | 'image': 'postgres:latest' 345 | } 346 | ] 347 | 348 | self.assertEqual(targetResult, result) 349 | 350 | 351 | 352 | 353 | 354 | @mock.patch.object(Command, 'runCmd', autospec=True) 355 | def testRunDump(self, mock_runCmd): 356 | 357 | backupService = Backup() 358 | 359 | listServices = [ 360 | { 361 | 'type': 'service', 362 | 'name': 'test', 363 | 'state': 'active', 364 | 'launchConfig': { 365 | 'imageUuid': 'test/postgres:latest', 366 | 'environment': { 367 | 'POSTGRES_USER': 'user', 368 | 'POSTGRES_DB': 'test', 369 | 'POSTGRES_PASSWORD': 'pass' 370 | } 371 | }, 372 | 'links': { 373 | 'environment': 'https://fake/environment', 374 | 'instances': 'https://fake/instances', 375 | }, 376 | 'stack': { 377 | 'name': 'stack-test' 378 | }, 379 | 'instances': [ 380 | { 381 | 'state': 'disabled', 382 | 'primaryIpAddress': '10.0.0.1', 383 | 'host': { 384 | 'name': 'host-1' 385 | }, 386 | 'links': { 387 | 'hosts': 'https://fake/hosts' 388 | } 389 | }, 390 | { 391 | 'state': 'running', 392 | 'primaryIpAddress': '10.0.0.2', 393 | 'host': { 394 | 'name': 'host-1' 395 | }, 396 | 'links': { 397 | 'hosts': 'https://fake/hosts' 398 | } 399 | }, 400 | { 401 | 'state': 'running', 402 | 'primaryIpAddress': '10.0.0.3', 403 | 'host': { 404 | 'name': 'host-1' 405 | }, 406 | 'links': { 407 | 'hosts': 'https://fake/hosts' 408 | } 409 | } 410 | 411 | ], 412 | }, 413 | { 414 | 'type': 'service', 415 | 'name': 'test2', 416 | 'state': 'active', 417 | 'launchConfig': { 418 | 'imageUuid': 'test/mysql:latest', 419 | 'environment': { 420 | 'MYSQL_USER': 'user', 421 | 'MYSQL_DB': 'test', 422 | 'MYSQL_PASSWORD': 'pass' 423 | } 424 | }, 425 | 'links': { 426 | 'environment': 'https://fake/environment', 427 | 'instances': 'https://fake/instances', 428 | }, 429 | 'stack': { 430 | 'name': 'stack-test' 431 | }, 432 | 'instances': [ 433 | { 434 | 'state': 'disabled', 435 | 'primaryIpAddress': '10.1.0.1', 436 | 'host': { 437 | 'name': 'host-1' 438 | }, 439 | 'links': { 440 | 'hosts': 'https://fake/hosts' 441 | } 442 | }, 443 | { 444 | 'state': 'running', 445 | 'primaryIpAddress': '10.1.0.2', 446 | 'host': { 447 | 'name': 'host-1' 448 | }, 449 | 'links': { 450 | 'hosts': 'https://fake/hosts' 451 | } 452 | }, 453 | { 454 | 'state': 'running', 455 | 'primaryIpAddress': '10.1.0.3', 456 | 'host': { 457 | 'name': 'host-1' 458 | }, 459 | 'links': { 460 | 'hosts': 'https://fake/hosts' 461 | } 462 | } 463 | 464 | ], 465 | } 466 | ] 467 | 468 | listDump = [ 469 | { 470 | 'service': listServices[0], 471 | 'target_dir': '/tmp/backup/stack-test/test', 472 | 'commands': ['pg_dump -h 10.0.0.2 -U user -d test -f /tmp/backup/stack-test/test/test.dump'], 473 | 'entrypoint': "sh -c", 474 | 'environments': ['PGPASSWORD:pass'], 475 | 'image': 'postgres:latest' 476 | }, 477 | { 478 | 'service': listServices[1], 479 | 'target_dir': '/tmp/backup/stack-test/test2', 480 | 'commands': [ 481 | 'mysqldump -h 10.1.0.2 -U user -d test -f /tmp/backup/stack-test/test2/test.dump', 482 | 'fake_cmd2 -h 10.1.0.2 -U user' 483 | ], 484 | 'environments': ['MYSQLPASSWORD:pass'], 485 | 'image': 'mysql:latest' 486 | } 487 | ] 488 | 489 | backupService.runDump(listDump) 490 | #print("Nb call %s" % mock_runCmd.call_args_list) 491 | mock_runCmd.assert_any_call(mock.ANY, 'docker pull postgres:latest') 492 | mock_runCmd.assert_any_call(mock.ANY, "docker run --rm --entrypoint='sh -c' -v /tmp/backup/stack-test/test:/tmp/backup/stack-test/test -e 'PGPASSWORD=pass' postgres:latest pg_dump -h 10.0.0.2 -U user -d test -f /tmp/backup/stack-test/test/test.dump") 493 | mock_runCmd.assert_any_call(mock.ANY, 'docker pull mysql:latest') 494 | mock_runCmd.assert_any_call(mock.ANY, "docker run --rm -v /tmp/backup/stack-test/test2:/tmp/backup/stack-test/test2 -e 'MYSQLPASSWORD=pass' mysql:latest mysqldump -h 10.1.0.2 -U user -d test -f /tmp/backup/stack-test/test2/test.dump") 495 | 496 | 497 | @mock.patch('__builtin__.open', new_callable=mock.mock_open(), create=True) 498 | @mock.patch.object(os, 'makedirs', autospec=True) 499 | def testdumpStacksSettings(self, mock_makedirs, mock_file): 500 | backupService = Backup() 501 | 502 | listStack = [ 503 | { 504 | "id": "1e44", 505 | "type": "environment", 506 | "links": { 507 | "self": "/v1/projects/1a203/environments/1e44", 508 | "account": "/v1/projects/1a203/environments/1e44/account", 509 | "services": "/v1/projects/1a203/environments/1e44/services", 510 | "composeConfig": "/v1/projects/1a203/environments/1e44/composeconfig", 511 | }, 512 | "actions": { 513 | "upgrade": "/v1/projects/1a203/environments/1e44/?action=upgrade", 514 | "update": "/v1/projects/1a203/environments/1e44/?action=update", 515 | "remove": "/v1/projects/1a203/environments/1e44/?action=remove", 516 | "addoutputs": "/v1/projects/1a203/environments/1e44/?action=addoutputs", 517 | "activateservices": "/v1/projects/1a203/environments/1e44/?action=activateservices", 518 | "deactivateservices": "/v1/projects/1a203/environments/1e44/?action=deactivateservices", 519 | "exportconfig": "/v1/projects/1a203/environments/1e44/?action=exportconfig", 520 | }, 521 | "name": "Default", 522 | "state": "active", 523 | "accountId": "1a203", 524 | "created": "2016-09-14T07:41:09Z", 525 | "createdTS": 1473838869000, 526 | "description": None, 527 | "dockerCompose": None, 528 | "environment": None, 529 | "externalId": None, 530 | "healthState": "healthy", 531 | "kind": "environment", 532 | "outputs": None, 533 | "previousEnvironment": None, 534 | "previousExternalId": None, 535 | "rancherCompose": None, 536 | "removed": None, 537 | "startOnCreate": None, 538 | "transitioning": "no", 539 | "transitioningMessage": None, 540 | "transitioningProgress": None, 541 | "uuid": "e2c02a5f-5585-4ee7-8bdb-3672e874de10", 542 | 'settings': { 543 | "id": None, 544 | "type": "composeConfig", 545 | "links": {}, 546 | "actions": {}, 547 | "dockerComposeConfig": "test:\r\n environment:\r\n BACKEND: ftp://test\r\n FTP_PASSWORD: test\r\n CRON_SCHEDULE: 0 0 * * *\r\n BK_FULL_FREQ: 1D\r\n BK_KEEP_FULL: '7'\r\n BK_KEEP_FULL_CHAIN: '1'\r\n VOLUME_SIZE: '1000'\r\n RANCHER_API_URL: https://test/v1/projects/1a203\r\n RANCHER_API_KEY: test\r\n RANCHER_API_SECRET: test\r\n DEBUG: 'false'\r\n labels:\r\n io.rancher.container.pull_image: always\r\n backup.disable: 'true'\r\n tty: true\r\n image: webcenter/rancher-backup:develop\r\n privileged: true\r\n stdin_open: true\r\nmariadb:\r\n environment:\r\n MYSQL_ROOT_PASSWORD: root-pass\r\n MYSQL_DATABASE: teampass\r\n MYSQL_PASSWORD: user-pass\r\n MYSQL_USER: teampass\r\n labels:\r\n io.rancher.container.pull_image: always\r\n tty: true\r\n image: mariadb\r\n stdin_open: true\r\npostgres:\r\n environment:\r\n PGDATA: /var/lib/postgresql/data/pgdata\r\n POSTGRES_DB: alfresco\r\n POSTGRES_USER: user\r\n POSTGRES_PASSWORD: pass\r\n labels:\r\n io.rancher.container.pull_image: always\r\n tty: true\r\n image: postgres:9.4\r\n volumes:\r\n - /data/postgres:/var/lib/postgresql/data/pgdata\r\n stdin_open: true\r\nmysql:\r\n environment:\r\n MYSQL_ROOT_PASSWORD: root-pass\r\n MYSQL_DATABASE: teampass\r\n MYSQL_PASSWORD: user-pass\r\n MYSQL_USER: teampass\r\n labels:\r\n io.rancher.container.pull_image: always\r\n tty: true\r\n image: mysql/mysql-server:5.5\r\n stdin_open: true\r\n", 548 | "rancherComposeConfig": "test:\r\n scale: 1\r\nmariadb:\r\n scale: 1\r\npostgres:\r\n scale: 1\r\nmysql:\r\n scale: 1\r\n", 549 | } 550 | 551 | }, 552 | { 553 | "id": "1e45", 554 | "type": "environment", 555 | "links": { 556 | "self": "/v1/projects/1a203/environments/1e45", 557 | "account": "/v1/projects/1a203/environments/1e45/account", 558 | "services": "/v1/projects/1a203/environments/1e45/services", 559 | "composeConfig": "/v1/projects/1a203/environments/1e45/composeconfig", 560 | }, 561 | "actions": { 562 | "upgrade": "/v1/projects/1a203/environments/1e45/?action=upgrade", 563 | "update": "/v1/projects/1a203/environments/1e45/?action=update", 564 | "remove": "/v1/projects/1a203/environments/1e45/?action=remove", 565 | "addoutputs": "/v1/projects/1a203/environments/1e45/?action=addoutputs", 566 | "activateservices": "/v1/projects/1a203/environments/1e45/?action=activateservices", 567 | "deactivateservices": "/v1/projects/1a203/environments/1e45/?action=deactivateservices", 568 | "exportconfig": "/v1/projects/1a203/environments/1e45/?action=exportconfig", 569 | }, 570 | "name": "seedbox", 571 | "state": "active", 572 | "accountId": "1a203", 573 | "created": "2016-09-14T07:43:01Z", 574 | "createdTS": 1473838981000, 575 | "description": None, 576 | "dockerCompose": "plex:\n ports:\n - 32400:32400/tcp\n environment:\n PLEX_USERNAME: user\n PLEX_PASSWORD: pass\n PLEX_DISABLE_SECURITY: '0'\n SKIP_CHOWN_CONFIG: 'false'\n PLEX_ALLOWED_NETWORKS: 10.0.0.0/8\n labels:\n io.rancher.container.pull_image: always\n tty: true\n hostname: home\n image: timhaak/plex\n volumes:\n - /mnt/nas:/data\n - /data/seedbox/plex:/config\n stdin_open: true", 577 | "environment": None, 578 | "externalId": "", 579 | "healthState": "healthy", 580 | "kind": "environment", 581 | "outputs": None, 582 | "previousEnvironment": None, 583 | "previousExternalId": None, 584 | "rancherCompose": "plex:\n scale: 1", 585 | "removed": None, 586 | "startOnCreate": True, 587 | "transitioning": "no", 588 | "transitioningMessage": None, 589 | "transitioningProgress": None, 590 | "uuid": "1a5c08a4-c851-4651-b516-e950982d617b", 591 | 'settings': { 592 | "id": None, 593 | "type": "composeConfig", 594 | "links": {}, 595 | "actions": {}, 596 | "dockerComposeConfig": "plex:\r\n ports:\r\n - 32400:32400/tcp\r\n environment:\r\n PLEX_ALLOWED_NETWORKS: 10.0.0.0/8\r\n PLEX_DISABLE_SECURITY: '0'\r\n PLEX_PASSWORD: test\r\n PLEX_USERNAME: test\r\n SKIP_CHOWN_CONFIG: 'false'\r\n labels:\r\n io.rancher.container.pull_image: always\r\n tty: true\r\n hostname: home\r\n image: timhaak/plex\r\n volumes:\r\n - /mnt/nas:/data\r\n - /data/seedbox/plex:/config\r\n stdin_open: true\r\n", 597 | "rancherComposeConfig": "plex:\r\n scale: 1\r\n", 598 | } 599 | }, 600 | { 601 | "id": "1e48", 602 | "type": "environment", 603 | "links": { 604 | "self": "/v1/projects/1a203/environments/1e48", 605 | "account": "/v1/projects/1a203/environments/1e48/account", 606 | "services": "/v1/projects/1a203/environments/1e48/services", 607 | "composeConfig": "/v1/projects/1a203/environments/1e48/composeconfig", 608 | }, 609 | "actions": { 610 | "upgrade": "/v1/projects/1a203/environments/1e48/?action=upgrade", 611 | "update": "/v1/projects/1a203/environments/1e48/?action=update", 612 | "remove": "/v1/projects/1a203/environments/1e48/?action=remove", 613 | "addoutputs": "/v1/projects/1a203/environments/1e48/?action=addoutputs", 614 | "activateservices": "/v1/projects/1a203/environments/1e48/?action=activateservices", 615 | "deactivateservices": "/v1/projects/1a203/environments/1e48/?action=deactivateservices", 616 | "exportconfig": "/v1/projects/1a203/environments/1e48/?action=exportconfig", 617 | }, 618 | "name": "test", 619 | "state": "active", 620 | "accountId": "1a203", 621 | "created": "2016-10-21T09:21:46Z", 622 | "createdTS": 1477041706000, 623 | "description": None, 624 | "dockerCompose": None, 625 | "environment": None, 626 | "externalId": "", 627 | "healthState": "healthy", 628 | "kind": "environment", 629 | "outputs": None, 630 | "previousEnvironment": None, 631 | "previousExternalId": None, 632 | "rancherCompose": None, 633 | "removed": None, 634 | "startOnCreate": True, 635 | "transitioning": "no", 636 | "transitioningMessage": None, 637 | "transitioningProgress": None, 638 | "uuid": "0fec46ea-99d5-494d-b430-eac97beb419f", 639 | "settings": { 640 | "id": None, 641 | "type": "composeConfig", 642 | "links": {}, 643 | "actions": {}, 644 | "dockerComposeConfig": "elasticsearch:\r\n labels:\r\n io.rancher.container.pull_image: always\r\n tty: true\r\n image: elasticsearch\r\n stdin_open: true\r\nmongo:\r\n labels:\r\n io.rancher.container.pull_image: always\r\n tty: true\r\n command:\r\n - mongod\r\n - --smallfiles\r\n - --oplogSize\r\n - '128'\r\n image: mongo\r\n stdin_open: true\r\n", 645 | "rancherComposeConfig": "elasticsearch:\r\n scale: 1\r\nmongo:\r\n scale: 1\r\n", 646 | } 647 | 648 | }, 649 | { 650 | "id": "1e49", 651 | "type": "environment", 652 | "links": { 653 | "self": "/v1/projects/1a203/environments/1e49", 654 | "account": "/v1/projects/1a203/environments/1e49/account", 655 | "services": "/v1/projects/1a203/environments/1e49/services", 656 | "composeConfig": "/v1/projects/1a203/environments/1e49/composeconfig", 657 | }, 658 | "actions": { 659 | "upgrade": "/v1/projects/1a203/environments/1e49/?action=upgrade", 660 | "update": "/v1/projects/1a203/environments/1e49/?action=update", 661 | "remove": "/v1/projects/1a203/environments/1e49/?action=remove", 662 | "addoutputs": "/v1/projects/1a203/environments/1e49/?action=addoutputs", 663 | "activateservices": "/v1/projects/1a203/environments/1e49/?action=activateservices", 664 | "deactivateservices": "/v1/projects/1a203/environments/1e49/?action=deactivateservices", 665 | "exportconfig": "/v1/projects/1a203/environments/1e49/?action=exportconfig", 666 | }, 667 | "name": "lb", 668 | "state": "active", 669 | "accountId": "1a203", 670 | "created": "2016-10-24T09:55:44Z", 671 | "createdTS": 1477302944000, 672 | "description": None, 673 | "dockerCompose": None, 674 | "environment": None, 675 | "externalId": "", 676 | "healthState": "healthy", 677 | "kind": "environment", 678 | "outputs": None, 679 | "previousEnvironment": None, 680 | "previousExternalId": None, 681 | "rancherCompose": None, 682 | "removed": None, 683 | "startOnCreate": True, 684 | "transitioning": "no", 685 | "transitioningMessage": None, 686 | "transitioningProgress": None, 687 | "uuid": "7ac2a47f-b084-4002-a2fb-919a1e738bda", 688 | "settings": { 689 | "id": None, 690 | "type": "composeConfig", 691 | "links": {}, 692 | "actions": {}, 693 | "dockerComposeConfig": "{}\r\n", 694 | "rancherComposeConfig": "{}\r\n", 695 | } 696 | }, 697 | ] 698 | 699 | backupService.dumpStacksSettings('/backup', listStack) 700 | 701 | #print("Call makedirs: %s", mock_makedirs.call_args_list) 702 | #print("Call open: %s", mock_file.call_args_list) 703 | 704 | mock_makedirs.assert_any_call('/backup/Default') 705 | mock_makedirs.assert_any_call('/backup/seedbox') 706 | mock_makedirs.assert_any_call('/backup/test') 707 | mock_makedirs.assert_any_call('/backup/lb') 708 | 709 | mock_file.assert_any_call('/backup/Default/docker-compose.yml', 'w') 710 | mock_file.assert_any_call('/backup/Default/rancher-compose.yml', 'w') 711 | mock_file.assert_any_call('/backup/seedbox/docker-compose.yml', 'w') 712 | mock_file.assert_any_call('/backup/seedbox/rancher-compose.yml', 'w') 713 | mock_file.assert_any_call('/backup/test/docker-compose.yml', 'w') 714 | mock_file.assert_any_call('/backup/test/rancher-compose.yml', 'w') 715 | mock_file.assert_any_call('/backup/lb/docker-compose.yml', 'w') 716 | mock_file.assert_any_call('/backup/lb/rancher-compose.yml', 'w') 717 | 718 | @mock.patch.object(Command, 'runCmd', autospec=True) 719 | def testDumpRancherDatabase(self, mock_runCmd): 720 | backupService = Backup() 721 | 722 | listDatabaseSettings = { 723 | "type": "mysql", 724 | "host": "db-host", 725 | "port": "3306", 726 | "name": "rancher", 727 | "user": "user", 728 | "password": "password" 729 | } 730 | 731 | backupService.dumpRancherDatabase('/tmp/backup', listDatabaseSettings) 732 | 733 | #print("Call run: %s", mock_runCmd.call_args_list) 734 | 735 | mock_runCmd.assert_any_call(mock.ANY, 'docker pull mysql:latest') 736 | mock_runCmd.assert_any_call(mock.ANY, "docker run --rm -v /tmp/backup/database:/tmp/backup/database -e 'MYSQL_PWD=password' mysql:latest sh -c 'mysqldump -h db-host -P 3306 -u user rancher > /tmp/backup/database/rancher.dump'") 737 | 738 | 739 | 740 | if __name__ == '__main__': 741 | unittest.main() 742 | -------------------------------------------------------------------------------- /backup/test/backup/TestCommand.py: -------------------------------------------------------------------------------- 1 | __author__ = 'disaster' 2 | 3 | import unittest 4 | import mock 5 | 6 | import subprocess 7 | 8 | from fr.webcenter.backup.Command import Command 9 | 10 | 11 | class TestCommand(unittest.TestCase): 12 | 13 | @mock.patch.object(subprocess, 'Popen', autospec=True) 14 | def testRunCmd(self, mock_popen): 15 | 16 | commandService = Command() 17 | 18 | # When no error 19 | mock_popen.return_value.returncode = 0 20 | mock_popen.return_value.communicate.return_value = ("output", None) 21 | result = commandService.runCmd("fake cmd") 22 | self.assertEqual("output", result) 23 | 24 | # With error 25 | mock_popen.return_value.returncode = 0 26 | mock_popen.return_value.communicate.return_value = ("output", "Error") 27 | self.assertEqual("output", result) 28 | 29 | # With bad return code 30 | mock_popen.return_value.returncode = 1 31 | mock_popen.return_value.communicate.return_value = ("output", None) 32 | self.assertRaises(Exception, commandService.runCmd, "fake cmd") 33 | 34 | if __name__ == '__main__': 35 | unittest.main() -------------------------------------------------------------------------------- /backup/test/backup/TestConfig.py: -------------------------------------------------------------------------------- 1 | __author__ = 'disaster' 2 | 3 | import unittest 4 | import mock 5 | from fr.webcenter.backup.Config import Config 6 | 7 | 8 | 9 | 10 | def fakeSetting(path): 11 | 12 | settings = { 13 | 'rancher.host': "test", 14 | 'rancher.port': 1234 15 | } 16 | 17 | indexes = { 18 | 'mysql': { 19 | 'image': 'fake/image', 20 | 'command': 'facke command' 21 | } 22 | } 23 | 24 | templates = { 25 | path + '/templates/mysql.yml': "my mysql fake sample", 26 | 'postgresql.yml': "my postgresql fake sample" 27 | } 28 | 29 | return (settings, indexes, templates) 30 | 31 | class TestConfig(unittest.TestCase): 32 | 33 | def tearDown(self): 34 | Config._drop() 35 | 36 | @mock.patch.object(Config, '_load', side_effect=fakeSetting) 37 | def testGetSettings(self, run_mock): 38 | Config._drop() 39 | configService = Config("/fake/path") 40 | settings = configService.getSettings() 41 | 42 | targetSettings = { 43 | 'rancher.host': "test", 44 | 'rancher.port': 1234 45 | } 46 | 47 | self.assertEqual(settings, targetSettings) 48 | 49 | configService = Config() 50 | settings = configService.getSettings() 51 | self.assertEqual(settings, targetSettings) 52 | 53 | @mock.patch.object(Config, '_load', side_effect=fakeSetting) 54 | def testGetIndex(self, run_mock): 55 | Config._drop() 56 | configService = Config("/fake/path") 57 | index = configService.getIndex() 58 | 59 | targetIndex = { 60 | 'mysql': { 61 | 'image': 'fake/image', 62 | 'command': 'facke command' 63 | } 64 | } 65 | 66 | self.assertEqual(index, targetIndex) 67 | 68 | configService = Config() 69 | index = configService.getIndex() 70 | self.assertEqual(index, targetIndex) 71 | 72 | @mock.patch.object(Config, '_load', side_effect=fakeSetting) 73 | def testGetTemplate(self, run_mock): 74 | Config._drop() 75 | configService = Config("/fake/path") 76 | template = configService.getTemplate("mysql.yml") 77 | 78 | targetTemplate = "my mysql fake sample" 79 | 80 | 81 | self.assertEqual(template, targetTemplate) 82 | 83 | configService = Config() 84 | template = configService.getTemplate("mysql.yml") 85 | self.assertEqual(template, targetTemplate) 86 | 87 | 88 | 89 | if __name__ == '__main__': 90 | unittest.main() -------------------------------------------------------------------------------- /backup/test/backup/TestRancher.py: -------------------------------------------------------------------------------- 1 | __author__ = 'disaster' 2 | 3 | import unittest 4 | import mock 5 | from cattle import Client 6 | from fr.webcenter.backup.Rancher import Rancher 7 | 8 | def fakeCallApiList(section): 9 | 10 | if section == "service": 11 | 12 | service = {} 13 | service['type'] = "service" 14 | service['name'] = "test" 15 | service["state"] = "active" 16 | service['launchConfig'] = {} 17 | service['launchConfig']['imageUuid'] = "test/postgres:latest" 18 | service['launchConfig']['environment'] = { 19 | 'POSTGRES_USER': 'user', 20 | 'POSTGRES_DB': 'test', 21 | 'PGPASSWORD': 'pass' 22 | } 23 | service['links'] = {} 24 | service['links']['stack'] = 'https://fake/stack' 25 | service['links']['instances'] = 'https://fake/instances' 26 | 27 | return [service] 28 | 29 | elif section == "setting": 30 | listSettings = [] 31 | setting = {} 32 | setting['id'] = "1as!account.by.key.credential.types" 33 | setting['type'] = "activeSetting" 34 | setting['name'] = "account.by.key.credential.types" 35 | setting['activeValue'] = "agentApiKey, apiKey, usernamePassword" 36 | listSettings.append(setting) 37 | 38 | setting = {} 39 | setting['id'] = "1as!cattle.db.cattle.database" 40 | setting['type'] = "activeSetting" 41 | setting['name'] = "cattle.db.cattle.database" 42 | setting['activeValue'] = "mysql" 43 | listSettings.append(setting) 44 | 45 | setting = {} 46 | setting['id'] = "1as!cattle.db.cattle.mysql.host" 47 | setting['type'] = "activeSetting" 48 | setting['name'] = "cattle.db.cattle.mysql.host" 49 | setting['activeValue'] = "db" 50 | listSettings.append(setting) 51 | 52 | setting = {} 53 | setting['id'] = "1as!cattle.db.cattle.mysql.name" 54 | setting['type'] = "activeSetting" 55 | setting['name'] = "cattle.db.cattle.mysql.name" 56 | setting['activeValue'] = "rancher" 57 | listSettings.append(setting) 58 | 59 | setting = {} 60 | setting['id'] = "1as!cattle.db.cattle.mysql.port" 61 | setting['type'] = "activeSetting" 62 | setting['name'] = "cattle.db.cattle.mysql.port" 63 | setting['activeValue'] = "3306" 64 | listSettings.append(setting) 65 | 66 | setting = {} 67 | setting['id'] = "1as!cattle.db.cattle.password" 68 | setting['type'] = "activeSetting" 69 | setting['name'] = "cattle.db.cattle.password" 70 | setting['activeValue'] = "password" 71 | listSettings.append(setting) 72 | 73 | setting = {} 74 | setting['id'] = "1as!cattle.db.cattle.username" 75 | setting['type'] = "activeSetting" 76 | setting['name'] = "cattle.db.cattle.username" 77 | setting['activeValue'] = "rancher" 78 | listSettings.append(setting) 79 | 80 | return listSettings 81 | 82 | else: 83 | return None 84 | 85 | def fakeCallApiGet(url): 86 | if url == "https://fake/stack": 87 | stack = {} 88 | stack['name'] = 'stack-test' 89 | return stack 90 | 91 | if url == "https://fake/hosts": 92 | host = {} 93 | host['name'] = "host-1" 94 | host2 = {} 95 | host2['name'] = "host-2" 96 | 97 | return [host, host2] 98 | 99 | if url == 'https://fake/instances': 100 | instance = {} 101 | instance['state'] = "disabled" 102 | instance['primaryIpAddress'] = '10.0.0.1' 103 | instance['links'] = { 104 | 'hosts': 'https://fake/hosts' 105 | } 106 | 107 | instance2 = {} 108 | instance2['state'] = "running" 109 | instance2['primaryIpAddress'] = '10.0.0.2' 110 | instance2['links'] = { 111 | 'hosts': 'https://fake/hosts' 112 | } 113 | 114 | instance3 = {} 115 | instance3['state'] = "running" 116 | instance3['primaryIpAddress'] = '10.0.0.3' 117 | instance3['links'] = { 118 | 'hosts': 'https://fake/hosts' 119 | } 120 | 121 | return [instance, instance2, instance3] 122 | 123 | def fakeClient(url, access_key, secret_key): 124 | return None 125 | 126 | 127 | class TestRancher(unittest.TestCase): 128 | 129 | 130 | @mock.patch.object(Client, 'list', side_effect=fakeCallApiList) 131 | @mock.patch.object(Client, '_get', side_effect=fakeCallApiGet) 132 | @mock.patch.object(Client, '__init__', side_effect=fakeClient) 133 | def testGetServices(self, mock_run, mock_run2, mock_run4): 134 | 135 | rancherService = Rancher("https://url", "key", "secret") 136 | listServices = rancherService.getServices() 137 | 138 | targetListServices = [ 139 | { 140 | 'type': 'service', 141 | 'name': 'test', 142 | 'state': 'active', 143 | 'launchConfig': { 144 | 'imageUuid': 'test/postgres:latest', 145 | 'environment': { 146 | 'POSTGRES_USER': 'user', 147 | 'POSTGRES_DB': 'test', 148 | 'PGPASSWORD': 'pass' 149 | } 150 | }, 151 | 'links': { 152 | 'stack': 'https://fake/stack', 153 | 'instances': 'https://fake/instances', 154 | }, 155 | 'stack': { 156 | 'name': 'stack-test' 157 | }, 158 | 'instances': [ 159 | { 160 | 'state': 'disabled', 161 | 'primaryIpAddress': '10.0.0.1', 162 | 'host': { 163 | 'name': 'host-1' 164 | }, 165 | 'links': { 166 | 'hosts': 'https://fake/hosts' 167 | } 168 | }, 169 | { 170 | 'state': 'running', 171 | 'primaryIpAddress': '10.0.0.2', 172 | 'host': { 173 | 'name': 'host-1' 174 | }, 175 | 'links': { 176 | 'hosts': 'https://fake/hosts' 177 | } 178 | }, 179 | { 180 | 'state': 'running', 181 | 'primaryIpAddress': '10.0.0.3', 182 | 'host': { 183 | 'name': 'host-1' 184 | }, 185 | 'links': { 186 | 'hosts': 'https://fake/hosts' 187 | } 188 | } 189 | 190 | ], 191 | } 192 | ] 193 | 194 | self.assertEquals(listServices, targetListServices) 195 | 196 | rancherService = Rancher() 197 | listServices = rancherService.getServices() 198 | self.assertEquals(listServices, targetListServices) 199 | 200 | @mock.patch.object(Client, '_get', autospec=True) 201 | @mock.patch.object(Client, 'list', autospec=True) 202 | @mock.patch.object(Client, '__init__', side_effect=fakeClient) 203 | def testGetStacks(self, mock_init, mock_list, mock_get): 204 | 205 | rancherService = Rancher("https://url", "key", "secret") 206 | rancherService.getStacks() 207 | mock_list.assert_any_call(mock.ANY,'stack') 208 | 209 | @mock.patch.object(Client, 'list', side_effect=fakeCallApiList) 210 | @mock.patch.object(Client, '__init__', side_effect=fakeClient) 211 | def testGetDatabaseSettings(self, mock_init, mock_list): 212 | rancherService = Rancher("https://url", "key", "secret") 213 | 214 | targetListSettings = { 215 | "type": "mysql", 216 | "host": "db", 217 | "db": "rancher", 218 | "user": "rancher", 219 | "password": "password", 220 | "port": "3306" 221 | } 222 | 223 | listSettings = rancherService.getDatabaseSettings() 224 | 225 | self.assertEquals(listSettings, targetListSettings) 226 | 227 | if __name__ == '__main__': 228 | unittest.main() -------------------------------------------------------------------------------- /backup/test/backup/TestTemplate.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from fr.webcenter.backup.Config import Config 3 | from fr.webcenter.backup.Backup import Backup 4 | 5 | __author__ = 'disaster' 6 | 7 | 8 | class TestTemplate(unittest.TestCase): 9 | 10 | def setUp(self): 11 | 12 | self.maxDiff = None 13 | configService = Config("../../config") 14 | 15 | 16 | 17 | def testTemplateMysql(self): 18 | 19 | backupService = Backup() 20 | 21 | # When user, password and database are setted 22 | listServices = [ 23 | { 24 | 'type': 'service', 25 | 'name': 'test2', 26 | 'state': 'active', 27 | 'launchConfig': { 28 | 'imageUuid': 'test/mysql:latest', 29 | 'environment': { 30 | 'MYSQL_USER': 'user', 31 | 'MYSQL_DATABASE': 'test', 32 | 'MYSQL_PASSWORD': 'pass' 33 | } 34 | }, 35 | 'links': { 36 | 'stack': 'https://fake/stack', 37 | 'instances': 'https://fake/instances', 38 | }, 39 | 'stack': { 40 | 'name': 'stack-test' 41 | }, 42 | 'instances': [ 43 | { 44 | 'state': 'disabled', 45 | 'primaryIpAddress': '10.1.0.1', 46 | 'host': { 47 | 'name': 'host-1' 48 | }, 49 | 'links': { 50 | 'hosts': 'https://fake/hosts' 51 | } 52 | }, 53 | { 54 | 'state': 'running', 55 | 'primaryIpAddress': '10.1.0.2', 56 | 'host': { 57 | 'name': 'host-1' 58 | }, 59 | 'links': { 60 | 'hosts': 'https://fake/hosts' 61 | } 62 | }, 63 | { 64 | 'state': 'running', 65 | 'primaryIpAddress': '10.1.0.3', 66 | 'host': { 67 | 'name': 'host-1' 68 | }, 69 | 'links': { 70 | 'hosts': 'https://fake/hosts' 71 | } 72 | } 73 | 74 | ], 75 | } 76 | ] 77 | 78 | targetResult = [ 79 | { 80 | 'service': listServices[0], 81 | 'target_dir': '/backup/stack-test/test2', 82 | 'commands': [ 83 | "sh -c 'mysqldump -h 10.1.0.2 -u user test > /backup/stack-test/test2/test.dump'", 84 | ], 85 | 'environments': ['MYSQL_PWD:pass'], 86 | 'image': 'mysql:latest' 87 | } 88 | ] 89 | 90 | result = backupService.searchDump('/backup', listServices) 91 | self.assertEqual(targetResult, result) 92 | 93 | # When user, password are setted 94 | listServices = [ 95 | { 96 | 'type': 'service', 97 | 'name': 'test2', 98 | 'state': 'active', 99 | 'launchConfig': { 100 | 'imageUuid': 'test/mysql:latest', 101 | 'environment': { 102 | 'MYSQL_USER': 'user', 103 | 'MYSQL_PASSWORD': 'pass' 104 | } 105 | }, 106 | 'links': { 107 | 'stack': 'https://fake/stack', 108 | 'instances': 'https://fake/instances', 109 | }, 110 | 'stack': { 111 | 'name': 'stack-test' 112 | }, 113 | 'instances': [ 114 | { 115 | 'state': 'disabled', 116 | 'primaryIpAddress': '10.1.0.1', 117 | 'host': { 118 | 'name': 'host-1' 119 | }, 120 | 'links': { 121 | 'hosts': 'https://fake/hosts' 122 | } 123 | }, 124 | { 125 | 'state': 'running', 126 | 'primaryIpAddress': '10.1.0.2', 127 | 'host': { 128 | 'name': 'host-1' 129 | }, 130 | 'links': { 131 | 'hosts': 'https://fake/hosts' 132 | } 133 | }, 134 | { 135 | 'state': 'running', 136 | 'primaryIpAddress': '10.1.0.3', 137 | 'host': { 138 | 'name': 'host-1' 139 | }, 140 | 'links': { 141 | 'hosts': 'https://fake/hosts' 142 | } 143 | } 144 | 145 | ], 146 | } 147 | ] 148 | 149 | targetResult = [ 150 | { 151 | 'service': listServices[0], 152 | 'target_dir': '/backup/stack-test/test2', 153 | 'commands': [ 154 | "sh -c 'mysqldump -h 10.1.0.2 -u user --all-databases > /backup/stack-test/test2/all-databases.dump'", 155 | ], 156 | 'environments': ['MYSQL_PWD:pass'], 157 | 'image': 'mysql:latest' 158 | } 159 | ] 160 | 161 | result = backupService.searchDump('/backup', listServices) 162 | self.assertEqual(targetResult, result) 163 | 164 | # When root password and database are setted 165 | listServices = [ 166 | { 167 | 'type': 'service', 168 | 'name': 'test2', 169 | 'state': 'active', 170 | 'launchConfig': { 171 | 'imageUuid': 'test/mysql:latest', 172 | 'environment': { 173 | 'MYSQL_DATABASE': 'test', 174 | 'MYSQL_ROOT_PASSWORD': 'pass' 175 | } 176 | }, 177 | 'links': { 178 | 'stack': 'https://fake/stack', 179 | 'instances': 'https://fake/instances', 180 | }, 181 | 'stack': { 182 | 'name': 'stack-test' 183 | }, 184 | 'instances': [ 185 | { 186 | 'state': 'disabled', 187 | 'primaryIpAddress': '10.1.0.1', 188 | 'host': { 189 | 'name': 'host-1' 190 | }, 191 | 'links': { 192 | 'hosts': 'https://fake/hosts' 193 | } 194 | }, 195 | { 196 | 'state': 'running', 197 | 'primaryIpAddress': '10.1.0.2', 198 | 'host': { 199 | 'name': 'host-1' 200 | }, 201 | 'links': { 202 | 'hosts': 'https://fake/hosts' 203 | } 204 | }, 205 | { 206 | 'state': 'running', 207 | 'primaryIpAddress': '10.1.0.3', 208 | 'host': { 209 | 'name': 'host-1' 210 | }, 211 | 'links': { 212 | 'hosts': 'https://fake/hosts' 213 | } 214 | } 215 | 216 | ], 217 | } 218 | ] 219 | 220 | targetResult = [ 221 | { 222 | 'service': listServices[0], 223 | 'target_dir': '/backup/stack-test/test2', 224 | 'commands': [ 225 | "sh -c 'mysqldump -h 10.1.0.2 -u root test > /backup/stack-test/test2/test.dump'", 226 | ], 227 | 'environments': ['MYSQL_PWD:pass'], 228 | 'image': 'mysql:latest' 229 | } 230 | ] 231 | 232 | result = backupService.searchDump('/backup', listServices) 233 | self.assertEqual(targetResult, result) 234 | 235 | # When root password is setted 236 | listServices = [ 237 | { 238 | 'type': 'service', 239 | 'name': 'test2', 240 | 'state': 'active', 241 | 'launchConfig': { 242 | 'imageUuid': 'test/mysql:latest', 243 | 'environment': { 244 | 'MYSQL_ROOT_PASSWORD': 'pass' 245 | } 246 | }, 247 | 'links': { 248 | 'stack': 'https://fake/stack', 249 | 'instances': 'https://fake/instances', 250 | }, 251 | 'stack': { 252 | 'name': 'stack-test' 253 | }, 254 | 'instances': [ 255 | { 256 | 'state': 'disabled', 257 | 'primaryIpAddress': '10.1.0.1', 258 | 'host': { 259 | 'name': 'host-1' 260 | }, 261 | 'links': { 262 | 'hosts': 'https://fake/hosts' 263 | } 264 | }, 265 | { 266 | 'state': 'running', 267 | 'primaryIpAddress': '10.1.0.2', 268 | 'host': { 269 | 'name': 'host-1' 270 | }, 271 | 'links': { 272 | 'hosts': 'https://fake/hosts' 273 | } 274 | }, 275 | { 276 | 'state': 'running', 277 | 'primaryIpAddress': '10.1.0.3', 278 | 'host': { 279 | 'name': 'host-1' 280 | }, 281 | 'links': { 282 | 'hosts': 'https://fake/hosts' 283 | } 284 | } 285 | 286 | ], 287 | } 288 | ] 289 | 290 | targetResult = [ 291 | { 292 | 'service': listServices[0], 293 | 'target_dir': '/backup/stack-test/test2', 294 | 'commands': [ 295 | "sh -c 'mysqldump -h 10.1.0.2 -u root --all-databases > /backup/stack-test/test2/all-databases.dump'", 296 | ], 297 | 'environments': ['MYSQL_PWD:pass'], 298 | 'image': 'mysql:latest' 299 | } 300 | ] 301 | 302 | result = backupService.searchDump('/backup', listServices) 303 | self.assertEqual(targetResult, result) 304 | 305 | # When user, password, root password and database are setted 306 | listServices = [ 307 | { 308 | 'type': 'service', 309 | 'name': 'test2', 310 | 'state': 'active', 311 | 'launchConfig': { 312 | 'imageUuid': 'test/mysql:latest', 313 | 'environment': { 314 | 'MYSQL_USER': 'user', 315 | 'MYSQL_DATABASE': 'test', 316 | 'MYSQL_PASSWORD': 'pass', 317 | 'MYSQL_ROOT_PASSWORD': 'root_pass' 318 | } 319 | }, 320 | 'links': { 321 | 'stack': 'https://fake/stack', 322 | 'instances': 'https://fake/instances', 323 | }, 324 | 'stack': { 325 | 'name': 'stack-test' 326 | }, 327 | 'instances': [ 328 | { 329 | 'state': 'disabled', 330 | 'primaryIpAddress': '10.1.0.1', 331 | 'host': { 332 | 'name': 'host-1' 333 | }, 334 | 'links': { 335 | 'hosts': 'https://fake/hosts' 336 | } 337 | }, 338 | { 339 | 'state': 'running', 340 | 'primaryIpAddress': '10.1.0.2', 341 | 'host': { 342 | 'name': 'host-1' 343 | }, 344 | 'links': { 345 | 'hosts': 'https://fake/hosts' 346 | } 347 | }, 348 | { 349 | 'state': 'running', 350 | 'primaryIpAddress': '10.1.0.3', 351 | 'host': { 352 | 'name': 'host-1' 353 | }, 354 | 'links': { 355 | 'hosts': 'https://fake/hosts' 356 | } 357 | } 358 | 359 | ], 360 | } 361 | ] 362 | 363 | targetResult = [ 364 | { 365 | 'service': listServices[0], 366 | 'target_dir': '/backup/stack-test/test2', 367 | 'commands': [ 368 | "sh -c 'mysqldump -h 10.1.0.2 -u user test > /backup/stack-test/test2/test.dump'", 369 | ], 370 | 'environments': ['MYSQL_PWD:pass'], 371 | 'image': 'mysql:latest' 372 | } 373 | ] 374 | 375 | result = backupService.searchDump('/backup', listServices) 376 | self.assertEqual(targetResult, result) 377 | 378 | def testTemplateMariadb(self): 379 | backupService = Backup() 380 | 381 | # When user, password and database are setted 382 | listServices = [ 383 | { 384 | 'type': 'service', 385 | 'name': 'test2', 386 | 'state': 'active', 387 | 'launchConfig': { 388 | 'imageUuid': 'test/mariadb:latest', 389 | 'environment': { 390 | 'MYSQL_USER': 'user', 391 | 'MYSQL_DATABASE': 'test', 392 | 'MYSQL_PASSWORD': 'pass' 393 | } 394 | }, 395 | 'links': { 396 | 'stack': 'https://fake/stack', 397 | 'instances': 'https://fake/instances', 398 | }, 399 | 'stack': { 400 | 'name': 'stack-test' 401 | }, 402 | 'instances': [ 403 | { 404 | 'state': 'disabled', 405 | 'primaryIpAddress': '10.1.0.1', 406 | 'host': { 407 | 'name': 'host-1' 408 | }, 409 | 'links': { 410 | 'hosts': 'https://fake/hosts' 411 | } 412 | }, 413 | { 414 | 'state': 'running', 415 | 'primaryIpAddress': '10.1.0.2', 416 | 'host': { 417 | 'name': 'host-1' 418 | }, 419 | 'links': { 420 | 'hosts': 'https://fake/hosts' 421 | } 422 | }, 423 | { 424 | 'state': 'running', 425 | 'primaryIpAddress': '10.1.0.3', 426 | 'host': { 427 | 'name': 'host-1' 428 | }, 429 | 'links': { 430 | 'hosts': 'https://fake/hosts' 431 | } 432 | } 433 | 434 | ], 435 | } 436 | ] 437 | 438 | targetResult = [ 439 | { 440 | 'service': listServices[0], 441 | 'target_dir': '/backup/stack-test/test2', 442 | 'commands': [ 443 | "sh -c 'mysqldump -h 10.1.0.2 -u user test > /backup/stack-test/test2/test.dump'", 444 | ], 445 | 'environments': ['MYSQL_PWD:pass'], 446 | 'image': 'mariadb:latest' 447 | } 448 | ] 449 | 450 | result = backupService.searchDump('/backup', listServices) 451 | self.assertEqual(targetResult, result) 452 | 453 | # When user, password are setted 454 | listServices = [ 455 | { 456 | 'type': 'service', 457 | 'name': 'test2', 458 | 'state': 'active', 459 | 'launchConfig': { 460 | 'imageUuid': 'test/mariadb:latest', 461 | 'environment': { 462 | 'MYSQL_USER': 'user', 463 | 'MYSQL_PASSWORD': 'pass' 464 | } 465 | }, 466 | 'links': { 467 | 'stack': 'https://fake/stack', 468 | 'instances': 'https://fake/instances', 469 | }, 470 | 'stack': { 471 | 'name': 'stack-test' 472 | }, 473 | 'instances': [ 474 | { 475 | 'state': 'disabled', 476 | 'primaryIpAddress': '10.1.0.1', 477 | 'host': { 478 | 'name': 'host-1' 479 | }, 480 | 'links': { 481 | 'hosts': 'https://fake/hosts' 482 | } 483 | }, 484 | { 485 | 'state': 'running', 486 | 'primaryIpAddress': '10.1.0.2', 487 | 'host': { 488 | 'name': 'host-1' 489 | }, 490 | 'links': { 491 | 'hosts': 'https://fake/hosts' 492 | } 493 | }, 494 | { 495 | 'state': 'running', 496 | 'primaryIpAddress': '10.1.0.3', 497 | 'host': { 498 | 'name': 'host-1' 499 | }, 500 | 'links': { 501 | 'hosts': 'https://fake/hosts' 502 | } 503 | } 504 | 505 | ], 506 | } 507 | ] 508 | 509 | targetResult = [ 510 | { 511 | 'service': listServices[0], 512 | 'target_dir': '/backup/stack-test/test2', 513 | 'commands': [ 514 | "sh -c 'mysqldump -h 10.1.0.2 -u user --all-databases > /backup/stack-test/test2/all-databases.dump'", 515 | ], 516 | 'environments': ['MYSQL_PWD:pass'], 517 | 'image': 'mariadb:latest' 518 | } 519 | ] 520 | 521 | result = backupService.searchDump('/backup', listServices) 522 | self.assertEqual(targetResult, result) 523 | 524 | # When root password and database are setted 525 | listServices = [ 526 | { 527 | 'type': 'service', 528 | 'name': 'test2', 529 | 'state': 'active', 530 | 'launchConfig': { 531 | 'imageUuid': 'test/mariadb:latest', 532 | 'environment': { 533 | 'MYSQL_DATABASE': 'test', 534 | 'MYSQL_ROOT_PASSWORD': 'pass' 535 | } 536 | }, 537 | 'links': { 538 | 'stack': 'https://fake/stack', 539 | 'instances': 'https://fake/instances', 540 | }, 541 | 'stack': { 542 | 'name': 'stack-test' 543 | }, 544 | 'instances': [ 545 | { 546 | 'state': 'disabled', 547 | 'primaryIpAddress': '10.1.0.1', 548 | 'host': { 549 | 'name': 'host-1' 550 | }, 551 | 'links': { 552 | 'hosts': 'https://fake/hosts' 553 | } 554 | }, 555 | { 556 | 'state': 'running', 557 | 'primaryIpAddress': '10.1.0.2', 558 | 'host': { 559 | 'name': 'host-1' 560 | }, 561 | 'links': { 562 | 'hosts': 'https://fake/hosts' 563 | } 564 | }, 565 | { 566 | 'state': 'running', 567 | 'primaryIpAddress': '10.1.0.3', 568 | 'host': { 569 | 'name': 'host-1' 570 | }, 571 | 'links': { 572 | 'hosts': 'https://fake/hosts' 573 | } 574 | } 575 | 576 | ], 577 | } 578 | ] 579 | 580 | targetResult = [ 581 | { 582 | 'service': listServices[0], 583 | 'target_dir': '/backup/stack-test/test2', 584 | 'commands': [ 585 | "sh -c 'mysqldump -h 10.1.0.2 -u root test > /backup/stack-test/test2/test.dump'", 586 | ], 587 | 'environments': ['MYSQL_PWD:pass'], 588 | 'image': 'mariadb:latest' 589 | } 590 | ] 591 | 592 | result = backupService.searchDump('/backup', listServices) 593 | self.assertEqual(targetResult, result) 594 | 595 | # When root password is setted 596 | listServices = [ 597 | { 598 | 'type': 'service', 599 | 'name': 'test2', 600 | 'state': 'active', 601 | 'launchConfig': { 602 | 'imageUuid': 'test/mariadb:latest', 603 | 'environment': { 604 | 'MYSQL_ROOT_PASSWORD': 'pass' 605 | } 606 | }, 607 | 'links': { 608 | 'stack': 'https://fake/stack', 609 | 'instances': 'https://fake/instances', 610 | }, 611 | 'stack': { 612 | 'name': 'stack-test' 613 | }, 614 | 'instances': [ 615 | { 616 | 'state': 'disabled', 617 | 'primaryIpAddress': '10.1.0.1', 618 | 'host': { 619 | 'name': 'host-1' 620 | }, 621 | 'links': { 622 | 'hosts': 'https://fake/hosts' 623 | } 624 | }, 625 | { 626 | 'state': 'running', 627 | 'primaryIpAddress': '10.1.0.2', 628 | 'host': { 629 | 'name': 'host-1' 630 | }, 631 | 'links': { 632 | 'hosts': 'https://fake/hosts' 633 | } 634 | }, 635 | { 636 | 'state': 'running', 637 | 'primaryIpAddress': '10.1.0.3', 638 | 'host': { 639 | 'name': 'host-1' 640 | }, 641 | 'links': { 642 | 'hosts': 'https://fake/hosts' 643 | } 644 | } 645 | 646 | ], 647 | } 648 | ] 649 | 650 | targetResult = [ 651 | { 652 | 'service': listServices[0], 653 | 'target_dir': '/backup/stack-test/test2', 654 | 'commands': [ 655 | "sh -c 'mysqldump -h 10.1.0.2 -u root --all-databases > /backup/stack-test/test2/all-databases.dump'", 656 | ], 657 | 'environments': ['MYSQL_PWD:pass'], 658 | 'image': 'mariadb:latest' 659 | } 660 | ] 661 | 662 | result = backupService.searchDump('/backup', listServices) 663 | self.assertEqual(targetResult, result) 664 | 665 | # When user, password, root password and database are setted 666 | listServices = [ 667 | { 668 | 'type': 'service', 669 | 'name': 'test2', 670 | 'state': 'active', 671 | 'launchConfig': { 672 | 'imageUuid': 'test/mariadb:latest', 673 | 'environment': { 674 | 'MYSQL_USER': 'user', 675 | 'MYSQL_DATABASE': 'test', 676 | 'MYSQL_PASSWORD': 'pass', 677 | 'MYSQL_ROOT_PASSWORD': 'root_pass' 678 | } 679 | }, 680 | 'links': { 681 | 'stack': 'https://fake/stack', 682 | 'instances': 'https://fake/instances', 683 | }, 684 | 'stack': { 685 | 'name': 'stack-test' 686 | }, 687 | 'instances': [ 688 | { 689 | 'state': 'disabled', 690 | 'primaryIpAddress': '10.1.0.1', 691 | 'host': { 692 | 'name': 'host-1' 693 | }, 694 | 'links': { 695 | 'hosts': 'https://fake/hosts' 696 | } 697 | }, 698 | { 699 | 'state': 'running', 700 | 'primaryIpAddress': '10.1.0.2', 701 | 'host': { 702 | 'name': 'host-1' 703 | }, 704 | 'links': { 705 | 'hosts': 'https://fake/hosts' 706 | } 707 | }, 708 | { 709 | 'state': 'running', 710 | 'primaryIpAddress': '10.1.0.3', 711 | 'host': { 712 | 'name': 'host-1' 713 | }, 714 | 'links': { 715 | 'hosts': 'https://fake/hosts' 716 | } 717 | } 718 | 719 | ], 720 | } 721 | ] 722 | 723 | targetResult = [ 724 | { 725 | 'service': listServices[0], 726 | 'target_dir': '/backup/stack-test/test2', 727 | 'commands': [ 728 | "sh -c 'mysqldump -h 10.1.0.2 -u user test > /backup/stack-test/test2/test.dump'", 729 | ], 730 | 'environments': ['MYSQL_PWD:pass'], 731 | 'image': 'mariadb:latest' 732 | } 733 | ] 734 | 735 | result = backupService.searchDump('/backup', listServices) 736 | self.assertEqual(targetResult, result) 737 | 738 | 739 | 740 | def testTemplatePostgresql(self): 741 | backupService = Backup() 742 | 743 | # When user, password and database are setted 744 | listServices = [ 745 | { 746 | 'type': 'service', 747 | 'name': 'test', 748 | 'state': 'active', 749 | 'launchConfig': { 750 | 'imageUuid': 'test/postgres:latest', 751 | 'environment': { 752 | 'POSTGRES_USER': 'user', 753 | 'POSTGRES_DB': 'test', 754 | 'POSTGRES_PASSWORD': 'pass' 755 | } 756 | }, 757 | 'links': { 758 | 'stack': 'https://fake/stack', 759 | 'instances': 'https://fake/instances', 760 | }, 761 | 'stack': { 762 | 'name': 'stack-test' 763 | }, 764 | 'instances': [ 765 | { 766 | 'state': 'disabled', 767 | 'primaryIpAddress': '10.0.0.1', 768 | 'host': { 769 | 'name': 'host-1' 770 | }, 771 | 'links': { 772 | 'hosts': 'https://fake/hosts' 773 | } 774 | }, 775 | { 776 | 'state': 'running', 777 | 'primaryIpAddress': '10.0.0.2', 778 | 'host': { 779 | 'name': 'host-1' 780 | }, 781 | 'links': { 782 | 'hosts': 'https://fake/hosts' 783 | } 784 | }, 785 | { 786 | 'state': 'running', 787 | 'primaryIpAddress': '10.0.0.3', 788 | 'host': { 789 | 'name': 'host-1' 790 | }, 791 | 'links': { 792 | 'hosts': 'https://fake/hosts' 793 | } 794 | } 795 | 796 | ], 797 | } 798 | ] 799 | 800 | targetResult = [ 801 | { 802 | 'service': listServices[0], 803 | 'target_dir': '/tmp/backup/stack-test/test', 804 | 'commands': [ 805 | 'pg_dump -h 10.0.0.2 -U user -d test -f /tmp/backup/stack-test/test/test.dump' 806 | ], 807 | 'environments': ['PGPASSWORD:pass'], 808 | 'image': 'postgres:latest' 809 | } 810 | ] 811 | 812 | result = backupService.searchDump('/tmp/backup', listServices) 813 | self.assertEqual(targetResult, result) 814 | 815 | # When user, password are setted 816 | listServices = [ 817 | { 818 | 'type': 'service', 819 | 'name': 'test', 820 | 'state': 'active', 821 | 'launchConfig': { 822 | 'imageUuid': 'test/postgres:latest', 823 | 'environment': { 824 | 'POSTGRES_USER': 'user', 825 | 'POSTGRES_PASSWORD': 'pass' 826 | } 827 | }, 828 | 'links': { 829 | 'stack': 'https://fake/stack', 830 | 'instances': 'https://fake/instances', 831 | }, 832 | 'stack': { 833 | 'name': 'stack-test' 834 | }, 835 | 'instances': [ 836 | { 837 | 'state': 'disabled', 838 | 'primaryIpAddress': '10.0.0.1', 839 | 'host': { 840 | 'name': 'host-1' 841 | }, 842 | 'links': { 843 | 'hosts': 'https://fake/hosts' 844 | } 845 | }, 846 | { 847 | 'state': 'running', 848 | 'primaryIpAddress': '10.0.0.2', 849 | 'host': { 850 | 'name': 'host-1' 851 | }, 852 | 'links': { 853 | 'hosts': 'https://fake/hosts' 854 | } 855 | }, 856 | { 857 | 'state': 'running', 858 | 'primaryIpAddress': '10.0.0.3', 859 | 'host': { 860 | 'name': 'host-1' 861 | }, 862 | 'links': { 863 | 'hosts': 'https://fake/hosts' 864 | } 865 | } 866 | 867 | ], 868 | } 869 | ] 870 | 871 | targetResult = [ 872 | { 873 | 'service': listServices[0], 874 | 'target_dir': '/tmp/backup/stack-test/test', 875 | 'commands': [ 876 | 'pg_dump -h 10.0.0.2 -U user -d user -f /tmp/backup/stack-test/test/user.dump' 877 | ], 878 | 'environments': ['PGPASSWORD:pass'], 879 | 'image': 'postgres:latest' 880 | } 881 | ] 882 | 883 | result = backupService.searchDump('/tmp/backup', listServices) 884 | self.assertEqual(targetResult, result) 885 | 886 | # When password and database are setted 887 | listServices = [ 888 | { 889 | 'type': 'service', 890 | 'name': 'test', 891 | 'state': 'active', 892 | 'launchConfig': { 893 | 'imageUuid': 'test/postgres:latest', 894 | 'environment': { 895 | 'POSTGRES_DB': 'test', 896 | 'POSTGRES_PASSWORD': 'pass' 897 | } 898 | }, 899 | 'links': { 900 | 'stack': 'https://fake/stack', 901 | 'instances': 'https://fake/instances', 902 | }, 903 | 'stack': { 904 | 'name': 'stack-test' 905 | }, 906 | 'instances': [ 907 | { 908 | 'state': 'disabled', 909 | 'primaryIpAddress': '10.0.0.1', 910 | 'host': { 911 | 'name': 'host-1' 912 | }, 913 | 'links': { 914 | 'hosts': 'https://fake/hosts' 915 | } 916 | }, 917 | { 918 | 'state': 'running', 919 | 'primaryIpAddress': '10.0.0.2', 920 | 'host': { 921 | 'name': 'host-1' 922 | }, 923 | 'links': { 924 | 'hosts': 'https://fake/hosts' 925 | } 926 | }, 927 | { 928 | 'state': 'running', 929 | 'primaryIpAddress': '10.0.0.3', 930 | 'host': { 931 | 'name': 'host-1' 932 | }, 933 | 'links': { 934 | 'hosts': 'https://fake/hosts' 935 | } 936 | } 937 | 938 | ], 939 | } 940 | ] 941 | 942 | targetResult = [ 943 | { 944 | 'service': listServices[0], 945 | 'target_dir': '/tmp/backup/stack-test/test', 946 | 'commands': [ 947 | 'pg_dump -h 10.0.0.2 -U postgres -d test -f /tmp/backup/stack-test/test/test.dump' 948 | ], 949 | 'environments': ['PGPASSWORD:pass'], 950 | 'image': 'postgres:latest' 951 | } 952 | ] 953 | 954 | result = backupService.searchDump('/tmp/backup', listServices) 955 | self.assertEqual(targetResult, result) 956 | 957 | # When password are setted 958 | listServices = [ 959 | { 960 | 'type': 'service', 961 | 'name': 'test', 962 | 'state': 'active', 963 | 'launchConfig': { 964 | 'imageUuid': 'test/postgres:latest', 965 | 'environment': { 966 | 'POSTGRES_PASSWORD': 'pass' 967 | } 968 | }, 969 | 'links': { 970 | 'stack': 'https://fake/stack', 971 | 'instances': 'https://fake/instances', 972 | }, 973 | 'stack': { 974 | 'name': 'stack-test' 975 | }, 976 | 'instances': [ 977 | { 978 | 'state': 'disabled', 979 | 'primaryIpAddress': '10.0.0.1', 980 | 'host': { 981 | 'name': 'host-1' 982 | }, 983 | 'links': { 984 | 'hosts': 'https://fake/hosts' 985 | } 986 | }, 987 | { 988 | 'state': 'running', 989 | 'primaryIpAddress': '10.0.0.2', 990 | 'host': { 991 | 'name': 'host-1' 992 | }, 993 | 'links': { 994 | 'hosts': 'https://fake/hosts' 995 | } 996 | }, 997 | { 998 | 'state': 'running', 999 | 'primaryIpAddress': '10.0.0.3', 1000 | 'host': { 1001 | 'name': 'host-1' 1002 | }, 1003 | 'links': { 1004 | 'hosts': 'https://fake/hosts' 1005 | } 1006 | } 1007 | 1008 | ], 1009 | } 1010 | ] 1011 | 1012 | targetResult = [ 1013 | { 1014 | 'service': listServices[0], 1015 | 'target_dir': '/tmp/backup/stack-test/test', 1016 | 'commands': [ 1017 | 'pg_dumpall -h 10.0.0.2 -U postgres --clean -f /tmp/backup/stack-test/test/all-databases.dump' 1018 | ], 1019 | 'environments': ['PGPASSWORD:pass'], 1020 | 'image': 'postgres:latest' 1021 | } 1022 | ] 1023 | 1024 | result = backupService.searchDump('/tmp/backup', listServices) 1025 | self.assertEqual(targetResult, result) 1026 | 1027 | def testTemplateMongodb(self): 1028 | backupService = Backup() 1029 | 1030 | # When no user and password 1031 | listServices = [ 1032 | { 1033 | 'type': 'service', 1034 | 'name': 'test', 1035 | 'state': 'active', 1036 | 'launchConfig': { 1037 | 'imageUuid': 'mongo:latest', 1038 | 'environment': { 1039 | } 1040 | }, 1041 | 'links': { 1042 | 'stack': 'https://fake/stack', 1043 | 'instances': 'https://fake/instances', 1044 | }, 1045 | 'stack': { 1046 | 'name': 'stack-test' 1047 | }, 1048 | 'instances': [ 1049 | { 1050 | 'state': 'disabled', 1051 | 'primaryIpAddress': '10.0.0.1', 1052 | 'host': { 1053 | 'name': 'host-1' 1054 | }, 1055 | 'links': { 1056 | 'hosts': 'https://fake/hosts' 1057 | } 1058 | }, 1059 | { 1060 | 'state': 'running', 1061 | 'primaryIpAddress': '10.0.0.2', 1062 | 'host': { 1063 | 'name': 'host-1' 1064 | }, 1065 | 'links': { 1066 | 'hosts': 'https://fake/hosts' 1067 | } 1068 | }, 1069 | { 1070 | 'state': 'running', 1071 | 'primaryIpAddress': '10.0.0.3', 1072 | 'host': { 1073 | 'name': 'host-1' 1074 | }, 1075 | 'links': { 1076 | 'hosts': 'https://fake/hosts' 1077 | } 1078 | } 1079 | 1080 | ], 1081 | } 1082 | ] 1083 | 1084 | targetResult = [ 1085 | { 1086 | 'service': listServices[0], 1087 | 'target_dir': '/tmp/backup/stack-test/test', 1088 | 'commands': [ 1089 | 'mongodump --host 10.0.0.2 --gzip --archive > /tmp/backup/stack-test/test/databases.archive.gz' 1090 | ], 1091 | 'environments': [], 1092 | 'image': 'mongo:latest' 1093 | } 1094 | ] 1095 | 1096 | result = backupService.searchDump('/tmp/backup', listServices) 1097 | self.assertEqual(targetResult, result) 1098 | 1099 | 1100 | # When user and password 1101 | listServices = [ 1102 | { 1103 | 'type': 'service', 1104 | 'name': 'test', 1105 | 'state': 'active', 1106 | 'launchConfig': { 1107 | 'imageUuid': 'mongo:latest', 1108 | 'environment': { 1109 | 'MONGO_USER': 'user', 1110 | 'MONGO_PASS': 'pass' 1111 | } 1112 | }, 1113 | 'links': { 1114 | 'stack': 'https://fake/stack', 1115 | 'instances': 'https://fake/instances', 1116 | }, 1117 | 'stack': { 1118 | 'name': 'stack-test' 1119 | }, 1120 | 'instances': [ 1121 | { 1122 | 'state': 'disabled', 1123 | 'primaryIpAddress': '10.0.0.1', 1124 | 'host': { 1125 | 'name': 'host-1' 1126 | }, 1127 | 'links': { 1128 | 'hosts': 'https://fake/hosts' 1129 | } 1130 | }, 1131 | { 1132 | 'state': 'running', 1133 | 'primaryIpAddress': '10.0.0.2', 1134 | 'host': { 1135 | 'name': 'host-1' 1136 | }, 1137 | 'links': { 1138 | 'hosts': 'https://fake/hosts' 1139 | } 1140 | }, 1141 | { 1142 | 'state': 'running', 1143 | 'primaryIpAddress': '10.0.0.3', 1144 | 'host': { 1145 | 'name': 'host-1' 1146 | }, 1147 | 'links': { 1148 | 'hosts': 'https://fake/hosts' 1149 | } 1150 | } 1151 | 1152 | ], 1153 | } 1154 | ] 1155 | 1156 | targetResult = [ 1157 | { 1158 | 'service': listServices[0], 1159 | 'target_dir': '/tmp/backup/stack-test/test', 1160 | 'commands': [ 1161 | 'mongodump --host 10.0.0.2 -u user -p pass --gzip --archive > /tmp/backup/stack-test/test/databases.archive.gz' 1162 | ], 1163 | 'environments': [], 1164 | 'image': 'mongo:latest' 1165 | } 1166 | ] 1167 | 1168 | result = backupService.searchDump('/tmp/backup', listServices) 1169 | self.assertEqual(targetResult, result) 1170 | 1171 | def testTemplateElasticsearch(self): 1172 | backupService = Backup() 1173 | 1174 | listServices = [ 1175 | { 1176 | 'type': 'service', 1177 | 'name': 'test', 1178 | 'state': 'active', 1179 | 'launchConfig': { 1180 | 'imageUuid': 'elasticsearch:latest', 1181 | 'environment': { 1182 | } 1183 | }, 1184 | 'links': { 1185 | 'stack': 'https://fake/stack', 1186 | 'instances': 'https://fake/instances', 1187 | }, 1188 | 'stack': { 1189 | 'name': 'stack-test' 1190 | }, 1191 | 'instances': [ 1192 | { 1193 | 'state': 'disabled', 1194 | 'primaryIpAddress': '10.0.0.1', 1195 | 'host': { 1196 | 'name': 'host-1' 1197 | }, 1198 | 'links': { 1199 | 'hosts': 'https://fake/hosts' 1200 | } 1201 | }, 1202 | { 1203 | 'state': 'running', 1204 | 'primaryIpAddress': '10.0.0.2', 1205 | 'host': { 1206 | 'name': 'host-1' 1207 | }, 1208 | 'links': { 1209 | 'hosts': 'https://fake/hosts' 1210 | } 1211 | }, 1212 | { 1213 | 'state': 'running', 1214 | 'primaryIpAddress': '10.0.0.3', 1215 | 'host': { 1216 | 'name': 'host-1' 1217 | }, 1218 | 'links': { 1219 | 'hosts': 'https://fake/hosts' 1220 | } 1221 | } 1222 | 1223 | ], 1224 | } 1225 | ] 1226 | 1227 | targetResult = [ 1228 | { 1229 | 'service': listServices[0], 1230 | 'target_dir': '/tmp/backup/stack-test/test', 1231 | 'commands': [ 1232 | "-c 'rm -rf /tmp/backup/stack-test/test/*.json'", 1233 | "-c 'elasticdump --input=http://10.0.0.2:9200/ --output=/tmp/backup/stack-test/test/dump_mapping.json --type=mapping'", 1234 | "-c 'elasticdump --input=http://10.0.0.2:9200/ --output=/tmp/backup/stack-test/test/dump_data.json --type=data'", 1235 | ], 1236 | 'environments': [], 1237 | 'entrypoint': "/bin/sh", 1238 | 'image': 'taskrabbit/elasticsearch-dump:latest' 1239 | } 1240 | ] 1241 | 1242 | result = backupService.searchDump('/tmp/backup', listServices) 1243 | self.assertEqual(targetResult, result) 1244 | 1245 | 1246 | 1247 | if __name__ == '__main__': 1248 | unittest.main() -------------------------------------------------------------------------------- /backup/test/backup/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/disaster37/rancher-backup/3e589e6e8948309dbb9fce8423707a68db8e98cd/backup/test/backup/__init__.py -------------------------------------------------------------------------------- /root/etc/cont-init.d/00-welcome.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/with-contenv bash 2 | 3 | cat << "EOF" 4 | 5 | . `. 6 | -d+ +d. 7 | `...................................... yms........----------. hms 8 | :hdddddddddddddddddddddddddddddddddddddd:`dmmdddddddddmmmmmmmmddo `hmm` 9 | `ommmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm-:mmmmmmmmmmmmmmmmmmmmmmm-.::+hmmm- 10 | .ymmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmd /mmmmmmmmmmmmmmmmmmmmmmm-+mmmmmmm/ 11 | :hmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmh``/ooooooshmmmmmmmmmmmmmm-:oooooo/ 12 | /dmmmdmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmhssssssssydmmmmmmmmmmmmmm- 13 | ommdy//mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm- 14 | -so- :mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmdsmmmmmmmmmmmmmm- 15 | :mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm+:mmmmmmmmmmmmmm- 16 | :mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm/:mmmmmmmmmmmmmm- 17 | :mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm/:mmmmmmmmmmmmmm- 18 | :mmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmmm+`sdmmmmmmmmmmdo 19 | :mmmmmmmmmmmmmd+--------------------------:smmmmmm+..----------` 20 | :mmmmmmmmmmmmmo `mmmmmmmmmmmmmd 21 | :mmmmmmmmmmmmmo `mmmmmmmmmmmmmd 22 | :mmmmmmmmmmmmmo `mmmmmmmmmmmmmd 23 | :mmmmmmmmmmmmmo `mmmmmmmmmmmmmd 24 | .hmmmmmmmmmmmd/ ymmmmmmmmmmmms 25 | ````````````` ```````````` 26 | 27 | EOF 28 | cat << EOF 29 | Name: ${CONTAINER_NAME} 30 | Site: ${APP_WEB} 31 | Image Author: ${CONTAINER_AUHTOR} 32 | Image support: ${CONTAINER_SUPPORT} 33 | Rancher: http://rancher.com 34 | Try Rancher: https://try.rancher.com/ 35 | 36 | For french people, you can visit my blog http://blog.webcenter.fr about Docker and Rancher 37 | 38 | 39 | EOF 40 | cat << "EOF" 41 | .ooooooo+:` /ooo- /ooo: +oo -/osyso+- `+oo: +oo: +ooooooooo`ooooooo+:` 42 | -mmmhyydmmh- -mmmmh` smmmm: dmm``sdmdyosdmdo`dmmo hmmo dmmmyyyyyy.mmmhyydmmh- 43 | -mmm/ -mmmo `ymdhmmo smmmmd/ dmm`ymmm- /mdm.dmmo hmmo dmmh -mmm/ -mmmo 44 | -mmms//ymmh- +mmo:mmm- smmsmmm/ dmd`mmmy `--.`dmmhssssmmmo dmmdoooo+ -mmms//smmh- 45 | -mmmddmmm/` .mmd.`ymmh` smm-/mmm:hmd`mmms `dmmdyyyymmmo dmmdhhhhy -mmmddmmm+` 46 | -mmm+.smmh` `ymmdhhhmmmo smm- /dmdmmm`hmmd` `sss-dmmo hmmo dmmh -mmm+.smmh. 47 | -mmm/ `ymmh/smmy////ommm-smm- :dmmmm`-hmmy/::smmh`dmmo hmmo dmmd/////+.mmm/ `ymmh+- 48 | .hhh/ `ohhhhhh. yhhsshd- :hhhh` .+yhdddhy+``hhh+ yhh+ yhhhhhhhhd.hhh/ `ohhd: 49 | ``` ```` ``` `` ``` `..`` `` `` ````````` ``` 50 | Community ^_^ 51 | 52 | EOF 53 | -------------------------------------------------------------------------------- /root/etc/cont-init.d/01-gen-confd-config.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/with-contenv bash 2 | 3 | cat << EOF > ${CONFD_HOME}/etc/conf.d/rancher-backup.yml.toml 4 | [template] 5 | prefix = "${CONFD_PREFIX_KEY}" 6 | src = "rancher-backup.yml.tmpl" 7 | dest = "${APP_HOME}/config/rancher-backup.yml" 8 | uid = 1001 9 | gid = 1001 10 | mode = "0644" 11 | keys = [ 12 | "/duplicity", 13 | "/rancher", 14 | "/module", 15 | "/cron" 16 | ] 17 | EOF 18 | 19 | 20 | cat << EOF > ${CONFD_HOME}/etc/conf.d/cron.toml 21 | [template] 22 | prefix = "${CONFD_PREFIX_KEY}" 23 | src = "cron.tmpl" 24 | dest = "/etc/services.d/cron/run" 25 | mode = "0755" 26 | keys = [ 27 | "/cron", 28 | ] 29 | EOF 30 | -------------------------------------------------------------------------------- /root/etc/cont-init.d/02-init-confd.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/with-contenv bash 2 | 3 | if [ "${CONFD_NODES}X" == "X" ]; then 4 | NODE="" 5 | else 6 | NODE="-node ${CONFD_NODES}" 7 | fi 8 | 9 | ${CONFD_HOME}/bin/confd -confdir ${CONFD_HOME}/etc -onetime -backend ${CONFD_BACKEND} ${PREFIX} ${NODE} 10 | -------------------------------------------------------------------------------- /root/etc/cont-init.d/04-run-backup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/with-contenv bash 2 | 3 | su ${USER} -c "python \"${APP_HOME}/backup.py\"" -------------------------------------------------------------------------------- /root/etc/fix-attrs.d/01-backup-dir: -------------------------------------------------------------------------------- 1 | /backup true backup 0660 0770 2 | /opt/backup/config true backup 0660 0770 3 | /opt/backup/.gnupg true backup 0600 0700 4 | -------------------------------------------------------------------------------- /root/etc/services.d/cron/finish: -------------------------------------------------------------------------------- 1 | #!/usr/bin/execlineb 2 | 3 | s6-svscanctl -t /var/run/s6/services 4 | -------------------------------------------------------------------------------- /root/opt/confd/etc/templates/cron.tmpl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/with-contenv bash 2 | 3 | exec su ${USER} -c "/usr/local/bin/go-cron \"{{ getv "/cron/schedule" "0 0 0 * * *"}}\" /bin/bash -c \"cd ${APP_HOME} && python ${APP_HOME}/backup.py\"" 4 | -------------------------------------------------------------------------------- /root/opt/confd/etc/templates/rancher-backup.yml.tmpl: -------------------------------------------------------------------------------- 1 | # Rancher Api 2 | rancher: 3 | api: 4 | {{- if getenv "CATTLE_URL"}} 5 | url: "{{getenv "CATTLE_URL"}}" 6 | {{- else }} 7 | {{- if (exists "/rancher/api/url")}} 8 | url: "{{getv "/rancher/api/url" ""}}" 9 | {{- else}} 10 | url: 11 | {{- end}} 12 | {{- end }} 13 | {{- if getenv "CATTLE_ACCESS_KEY"}} 14 | key: "{{getenv "CATTLE_ACCESS_KEY"}}" 15 | {{- else }} 16 | {{- if (exists "/rancher/api/key")}} 17 | key: "{{getv "/rancher/api/key" ""}}" 18 | {{- else}} 19 | key: 20 | {{- end}} 21 | {{- end }} 22 | {{- if getenv "CATTLE_SECRET_KEY"}} 23 | secret: "{{getenv "CATTLE_SECRET_KEY"}}" 24 | {{- else }} 25 | {{- if (exists "/rancher/api/secret")}} 26 | secret: "{{getv "/rancher/api/secret" ""}}" 27 | {{- else}} 28 | secret: 29 | {{- end}} 30 | {{- end }} 31 | 32 | # Rancher database 33 | db: 34 | {{- if (exists "/rancher/db/host")}} 35 | host: "{{getv "/rancher/db/host"}}" 36 | {{- else}} 37 | host: 38 | {{- end}} 39 | port: {{getv "/rancher/db/port" "3306"}} 40 | user: "{{getv "/rancher/db/user" "rancher"}}" 41 | {{- if (exists "/rancher/db/password")}} 42 | password: "{{getv "/rancher/db/password"}}" 43 | {{- end}} 44 | name: "{{getv "/rancher/db/name" "rancher"}}" 45 | 46 | # Backup module 47 | module: 48 | databases: {{getv "/module/database" "true"}} 49 | stack: {{getv "/module/stack" "true"}} 50 | rancher-db: {{getv "/module/rancher-db" "true"}} 51 | 52 | # Duplicity Policy 53 | duplicity: 54 | source-path: "{{getv "/duplicity/source-path" "/backup"}}" 55 | target-path: "{{getv "/duplicity/target-path" "/"}}" 56 | {{- if (exists "/duplicity/url")}} 57 | url: "{{getv "/duplicity/url"}}" 58 | {{- else}} 59 | url: 60 | {{- end}} 61 | {{- if (exists "/duplicity/options")}} 62 | options: "{{getv "/duplicity/options"}}" 63 | {{- else}} 64 | options: 65 | {{- end}} 66 | {{- if (exists "/duplicity/encrypt-key")}} 67 | encrypt-key: "{{getv "/duplicity/encrypt-key"}}" 68 | {{- else}} 69 | encrypt-key: 70 | {{- end}} 71 | full-if-older-than: "{{getv "/duplicity/full-if-older-than" "7D"}}" 72 | remove-all-but-n-full: {{getv "/duplicity/remove-all-but-n-full" "3"}} 73 | remove-all-inc-of-but-n-full: {{getv "/duplicity/remove-all-inc-of-but-n-full" "1"}} 74 | volsize: {{getv "/duplicity/volsize" "200"}} 75 | 76 | # cron 77 | cron: 78 | schedule: "{{getv "/cron/schedule" "0 0 0 * * *"}}" 79 | 80 | --------------------------------------------------------------------------------