├── requirements.txt ├── README.md ├── backup.py └── restore.py /requirements.txt: -------------------------------------------------------------------------------- 1 | boto3==1.13.10 2 | botocore==1.16.10 3 | docutils==0.15.2 4 | jmespath==0.10.0 5 | psycopg2-binary==2.8.5 6 | python-dateutil==2.8.1 7 | pytz==2020.1 8 | s3transfer==0.3.3 9 | six==1.14.0 10 | termcolor==1.1.0 11 | urllib3==1.25.9 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Backup and restore PostgreSQL database with Yandex Object Storage (or any S3-compatible storage) 2 | === 3 | 4 | There are two Python scripts: `backup.py` and `restore.py`. The first one creates 5 | PostgreSQL dump, zip and encrypt it, and then upload to Yandex Object 6 | Storage (or any S3-compatible storage). The second one finds last dump in 7 | Yandex Object Storage, download, unzip and decrypt it, and then load to 8 | PostgreSQL database accordingly. 9 | 10 | [YouTube video about scripts (in Russian)](https://www.youtube.com/watch?v=30TBpI4lEPI) 11 | 12 | For both scripts you need: 13 | 14 | * Python3.6+ version, 15 | * installed pip packeges from `requirements.txt`, 16 | * Yandex Object Storage / AWS credentials in `~/.aws/credentials`, 17 | * files with public (`backup_key.pem.pub`) and private (`backup_key.pem`) keys for encrypting and decrypting dump. You can generate both with openssl: 18 | ```sh 19 | openssl req -x509 -nodes -days 1000000 -newkey rsa:4096 -keyout backup_key.pem\ 20 | -subj "/C=US/ST=Illinois/L=Chicago/O=IT/CN=www.example.com" \ 21 | -out backup_key.pem.pub 22 | ``` 23 | * file with PostgreSQL database password `~/.pgpass` with chmod 600, including, for example: 24 | ```sh 25 | localhost:5432:your_database:your_db_user:your_db_user_password 26 | ``` 27 | * check `check_hostname()` function in `restore.py` — it checks hostname of current server (kind of protection against drop database tables on production server). 28 | 29 | Example of backup database (substitute your values in the variables below, 30 | note, that here we need public key file for encrypting database): 31 | 32 | ```sh 33 | DB_HOSTNAME=localhost \ 34 | DB_NAME=your_database \ 35 | DB_USER=your_db_user \ 36 | BACKUP_KEY_PUB_FILE=/home/www/.backup_key.pem.pub \ 37 | S3_BUCKET_NAME=your_s3_bucket \ 38 | TIME_ZONE=Europe/Moscow \ 39 | python3 backup.py 40 | ``` 41 | 42 | Example of load database (note, that here we need private key file for 43 | decrypting database): 44 | 45 | ```sh 46 | DB_HOSTNAME=localhost \ 47 | DB_NAME=your_database \ 48 | DB_USER=your_db_user \ 49 | BACKUP_KEY_PRIVATE_FILE=/home/www/.backup_key.pem \ 50 | S3_BUCKET_NAME=your_s3_bucket \ 51 | TIME_ZONE=Europe/Moscow \ 52 | python3 restore.py 53 | ``` 54 | -------------------------------------------------------------------------------- /backup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Backup PostgreSQL database to Yandex Object Storage, that has S3 compatible 3 | API. 4 | """ 5 | import datetime 6 | import os 7 | from pathlib import Path 8 | import pytz 9 | 10 | from termcolor import colored 11 | import boto3 12 | 13 | 14 | DB_HOSTNAME = os.getenv("DB_HOSTNAME", "localhost") 15 | DB_NAME = os.getenv("DB_NAME") 16 | DB_USER = os.getenv("DB_USER") 17 | S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME") 18 | BACKUP_KEY_PUB_FILE = os.getenv("BACKUP_KEY_PUB_FILE") 19 | TIME_ZONE = os.getenv("TIME_ZONE", "Europe/Moscow") 20 | 21 | DB_FILENAME = "/tmp/backup_db.sql.gz.enc" 22 | 23 | 24 | def say_hello(): 25 | print(colored("Hi! This tool will dump PostgreSQL database, compress \n" 26 | "and encode it, and then send to Yandex Object Storage.\n", "cyan")) 27 | 28 | 29 | def get_now_datetime_str(): 30 | now = datetime.datetime.now(pytz.timezone(TIME_ZONE)) 31 | return now.strftime('%Y-%m-%d__%H-%M-%S') 32 | 33 | 34 | def check_key_file_exists(): 35 | if not Path(BACKUP_KEY_PUB_FILE).is_file(): 36 | exit( 37 | f"\U00002757 Public encrypt key ({BACKUP_KEY_PUB_FILE}) " 38 | f"not found. If you have no key – you need to generate it. " 39 | f"You can find help here: " 40 | f"https://www.imagescape.com/blog/2015/12/18/encrypted-postgres-backups/" 41 | ) 42 | 43 | 44 | def dump_database(): 45 | print("\U0001F4E6 Preparing database backup started") 46 | dump_db_operation_status = os.WEXITSTATUS(os.system( 47 | f"pg_dump -h {DB_HOSTNAME} -U {DB_USER} {DB_NAME} | gzip -c --best | \ 48 | openssl smime -encrypt -aes256 -binary -outform DEM \ 49 | -out {DB_FILENAME} {BACKUP_KEY_PUB_FILE}" 50 | )) 51 | if dump_db_operation_status != 0: 52 | exit(f"\U00002757 Dump database command exits with status " 53 | f"{dump_db_operation_status}.") 54 | print("\U0001F510 DB dumped, archieved and encoded") 55 | 56 | 57 | def get_s3_instance(): 58 | session = boto3.session.Session() 59 | return session.client( 60 | service_name='s3', 61 | endpoint_url='https://storage.yandexcloud.net' 62 | ) 63 | 64 | 65 | def upload_dump_to_s3(): 66 | print("\U0001F4C2 Starting upload to Object Storage") 67 | get_s3_instance().upload_file( 68 | Filename=DB_FILENAME, 69 | Bucket=S3_BUCKET_NAME, 70 | Key=f'db-{get_now_datetime_str()}.sql.gz.enc' 71 | ) 72 | print("\U0001f680 Uploaded") 73 | 74 | 75 | def remove_temp_files(): 76 | os.remove(DB_FILENAME) 77 | print(colored("\U0001F44D That's all!", "green")) 78 | 79 | 80 | if __name__ == "__main__": 81 | say_hello() 82 | check_key_file_exists() 83 | dump_database() 84 | upload_dump_to_s3() 85 | remove_temp_files() 86 | -------------------------------------------------------------------------------- /restore.py: -------------------------------------------------------------------------------- 1 | """ 2 | Renew database on current server, if hostname startswith loader* 3 | or ends with .local (can be modified in check_hostname function below). 4 | Script download last dump from S3 (Yandex Object Storage), decrypt 5 | and load it after clear current database state. 6 | """ 7 | import os 8 | from pathlib import Path 9 | import socket 10 | 11 | import boto3 12 | import psycopg2 13 | from termcolor import colored 14 | 15 | 16 | DB_HOSTNAME = os.getenv("DB_HOSTNAME", "localhost") 17 | DB_NAME = os.getenv("DB_NAME") 18 | DB_USER = os.getenv("DB_USER") 19 | S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME") 20 | BACKUP_KEY_PRIVATE_FILE = os.getenv("BACKUP_KEY_PRIVATE_FILE") 21 | 22 | DB_FILENAME = '/tmp/backup_db.sql.gz.enc' 23 | 24 | connection = psycopg2.connect( 25 | f"dbname={DB_NAME} user={DB_USER} host='{DB_HOSTNAME}'") 26 | cursor = connection.cursor() 27 | 28 | 29 | def say_hello(): 30 | print(colored( 31 | "This tool will download last database backup from Yandex Object " 32 | "Storage,\n decompress and unzip it, and then load to local " 33 | "database\n", 34 | "cyan")) 35 | 36 | 37 | def check_hostname(): 38 | hostname = socket.gethostname() 39 | if not hostname.startswith('loader-') and not hostname.endswith('.local'): 40 | exit(f"\U00002757 It seems this is not loader server " 41 | f"({colored(hostname, 'red')}), exit.") 42 | print(colored("We are on some loader or local server, ok\n", "green")) 43 | 44 | 45 | def check_key_file_exists(): 46 | if not Path(BACKUP_KEY_PRIVATE_FILE).is_file(): 47 | exit( 48 | f"""\U00002757 Private encrypt key ({BACKUP_KEY_PRIVATE_FILE}) " 49 | "not found. You can find help here: " 50 | "https://www.imagescape.com/blog/2015/12/18/encrypted-postgres-backups/""" 51 | ) 52 | 53 | 54 | def get_s3_instance(): 55 | session = boto3.session.Session() 56 | return session.client( 57 | service_name='s3', 58 | endpoint_url='https://storage.yandexcloud.net' 59 | ) 60 | 61 | 62 | def get_last_backup_filename(): 63 | s3 = get_s3_instance() 64 | dumps = s3.list_objects(Bucket=S3_BUCKET_NAME)['Contents'] 65 | dumps.sort(key=lambda x: x['LastModified']) 66 | last_backup_filename = dumps[-1] 67 | print(f"\U000023F3 Last backup in S3 is {last_backup_filename['Key']}, " 68 | f"{round(last_backup_filename['Size'] / (1024*1024))} MB, " 69 | f"download it") 70 | return last_backup_filename['Key'] 71 | 72 | 73 | def download_s3_file(filename: str): 74 | _silent_remove_file(filename) 75 | get_s3_instance().download_file(S3_BUCKET_NAME, filename, DB_FILENAME) 76 | print(f"\U0001f680 Downloaded") 77 | 78 | 79 | def unencrypt_database(): 80 | operation_status = os.WEXITSTATUS(os.system( 81 | f"""openssl smime -decrypt -in {DB_FILENAME} -binary \ 82 | -inform DEM -inkey {BACKUP_KEY_PRIVATE_FILE} \ 83 | -out /tmp/db.sql.gz""" 84 | )) 85 | if operation_status != 0: 86 | exit(f"\U00002757 Can not unecrypt db file, status " 87 | f"{operation_status}.") 88 | print(f"\U0001F511 Database unecnrypted") 89 | 90 | 91 | def unzip_database(): 92 | _silent_remove_file("/tmp/db.sql") 93 | operation_status = os.WEXITSTATUS(os.system( 94 | f"""gzip -d /tmp/db.sql.gz""" 95 | )) 96 | if operation_status != 0: 97 | exit(f"\U00002757 Can not unecrypt db file, status " 98 | f"{operation_status}.") 99 | print(f"\U0001F4E4 Database unzipped") 100 | 101 | 102 | def clear_database(): 103 | tables = _get_all_db_tables() 104 | if not tables: 105 | return 106 | with connection: 107 | with connection.cursor() as local_cursor: 108 | local_cursor.execute("\n".join([ 109 | f'drop table if exists "{table}" cascade;' 110 | for table in tables])) 111 | print(f"\U0001F633 Database cleared") 112 | 113 | 114 | def load_database(): 115 | print(f"\U0001F4A4 Database load started") 116 | operation_status = os.WEXITSTATUS(os.system( 117 | f"""psql -h {DB_HOSTNAME} -U {DB_USER} {DB_NAME} < /tmp/db.sql""" 118 | )) 119 | if operation_status != 0: 120 | exit(f"\U00002757 Can not load database, status {operation_status}.") 121 | print(f"\U0001F916 Database loaded") 122 | 123 | 124 | def remove_temp_files(): 125 | _silent_remove_file(DB_FILENAME) 126 | print(colored("\U0001F44D That's all!", "green")) 127 | 128 | 129 | def _get_all_db_tables(): 130 | cursor.execute("""SELECT table_name FROM information_schema.tables 131 | WHERE table_schema = 'public' order by table_name;""") 132 | results = cursor.fetchall() 133 | tables = [] 134 | for row in results: 135 | tables.append(row[0]) 136 | return tables 137 | 138 | def _silent_remove_file(filename: str): 139 | try: 140 | os.remove(filename) 141 | except FileNotFoundError: 142 | pass 143 | 144 | 145 | if __name__ == "__main__": 146 | say_hello() 147 | check_hostname() 148 | check_key_file_exists() 149 | download_s3_file(get_last_backup_filename()) 150 | unencrypt_database() 151 | unzip_database() 152 | clear_database() 153 | load_database() 154 | remove_temp_files() 155 | --------------------------------------------------------------------------------