├── uninstall_bucardo.sh ├── revoke_write_access_from_old_db.sql ├── vars.sh ├── compare_data.sh ├── uninstall.template ├── install.sh ├── setup_new_database.template └── configure.sh /uninstall_bucardo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | envsubst < uninstall.template > uninstall.sql 4 | 5 | psql -h ${BUCARDO_OLD_HOSTNAME} -U ${BUCARDO_OLD_USERNAME} -d ${BUCARDO_OLD_DATABASE} -f uninstall.sql 6 | -------------------------------------------------------------------------------- /revoke_write_access_from_old_db.sql: -------------------------------------------------------------------------------- 1 | revoke update, delete, insert on all tables in schema public from app1; 2 | revoke update, delete, insert on all tables in schema public from app2; 3 | revoke update on all sequences in schema public from app1; 4 | revoke update on all sequences in schema public from app2; 5 | -------------------------------------------------------------------------------- /vars.sh: -------------------------------------------------------------------------------- 1 | export BUCARDO_SYNC_NAME=database_migration_2021 2 | export BUCARDO_LOCAL_PASSWORD=supersecret 3 | export BUCARDO_OLD_HOSTNAME=old.myhost.com 4 | export BUCARDO_OLD_USERNAME=admin 5 | export BUCARDO_OLD_PASSWORD=12345 6 | export BUCARDO_OLD_DATABASE=bgdb 7 | export BUCARDO_NEW_HOSTNAME=new.myhost.com 8 | export BUCARDO_NEW_USERNAME=admin 9 | export BUCARDO_NEW_PASSWORD=12345 10 | export BUCARDO_NEW_DATABASE=bgdb 11 | export APP1_PASS= 12 | export APP1_RO_PASS= 13 | export APP2_PASS= 14 | export APP2_RO_PASS= 15 | -------------------------------------------------------------------------------- /compare_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if ! command -v pgdatadiff &> /dev/null 3 | then 4 | sudo apt install libpq-dev python3-dev python3-pip 5 | pip3 install git+https://github.com/andrikoz/pgdatadiff.git 6 | fi 7 | pgdatadiff \ 8 | --firstdb=postgres://$BUCARDO_OLD_USERNAME:$BUCARDO_OLD_PASSWORD@$BUCARDO_OLD_HOSTNAME/$BUCARDO_OLD_DATABASE \ 9 | --seconddb=postgres://$BUCARDO_NEW_USERNAME:$BUCARDO_NEW_PASSWORD@$BUCARDO_NEW_HOSTNAME/$BUCARDO_NEW_DATABASE \ 10 | --only-data \ 11 | --exclude-tables=table_foo,table_bar 12 | -------------------------------------------------------------------------------- /uninstall.template: -------------------------------------------------------------------------------- 1 | DROP TRIGGER bucardo_delta_namemaker ON bucardo.bucardo_delta_names; 2 | DROP TRIGGER bucardo_note_trunc_$BUCARDO_SYNC_NAME ON table_1; 3 | DROP TRIGGER bucardo_delta ON table_1; 4 | DROP TRIGGER bucardo_kick_$BUCARDO_SYNC_NAME ON table_1; 5 | DROP TRIGGER bucardo_note_trunc_$BUCARDO_SYNC_NAME ON table_2; 6 | DROP TRIGGER bucardo_delta ON table_2; 7 | DROP TRIGGER bucardo_kick_$BUCARDO_SYNC_NAME ON table_2; 8 | DROP TRIGGER bucardo_note_trunc_$BUCARDO_SYNC_NAME ON table_2_lock; 9 | DROP TRIGGER bucardo_delta ON table_2_lock; 10 | DROP TRIGGER bucardo_kick_$BUCARDO_SYNC_NAME ON table_2_lock; 11 | DROP TRIGGER bucardo_note_trunc_$BUCARDO_SYNC_NAME ON payment; 12 | DROP TRIGGER bucardo_delta ON payment; 13 | DROP TRIGGER bucardo_kick_$BUCARDO_SYNC_NAME ON payment; 14 | DROP TRIGGER bucardo_note_trunc_$BUCARDO_SYNC_NAME ON booking; 15 | DROP TRIGGER bucardo_delta ON booking; 16 | DROP TRIGGER bucardo_kick_$BUCARDO_SYNC_NAME ON booking; 17 | DROP SCHEMA bucardo CASCADE; 18 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -ex 2 | 3 | # 4 | # Note: Need to run this as root 5 | # 6 | 7 | [ -z $BUCARDO_LOCAL_PASSWORD ] && echo "Plase set BUCARDO_LOCAL_PASSWORD" && exit 1 8 | 9 | # Install prerequisites 10 | apt update 11 | apt install -y \ 12 | cpanminus \ 13 | gcc \ 14 | libdbi-perl \ 15 | libpq-dev \ 16 | make \ 17 | postgresql \ 18 | postgresql-client-common \ 19 | postgresql-plperl-12 20 | 21 | # Create bucardo user in the local Postgresql 22 | sudo -u postgres psql -w postgres < ~/.pgpass 25 | cat >> ~/.pgpass < setup_new_database.sql 34 | psql -h ${BUCARDO_NEW_HOSTNAME} -U ${BUCARDO_NEW_USERNAME} -f setup_new_database.sql postgres 35 | 36 | # Setup Bucardo multi-master replication 37 | # XXX: Unfortunately Bucardo does not read the .pgpass file, 38 | # so we must explicitly specify the password in the command. 39 | bucardo_cmd add db source_db \ 40 | dbhost=$BUCARDO_OLD_HOSTNAME \ 41 | dbport=5432 \ 42 | dbname=$BUCARDO_OLD_DATABASE \ 43 | dbuser=$BUCARDO_OLD_USERNAME \ 44 | dbpass=$BUCARDO_OLD_PASSWORD 45 | bucardo_cmd add db target_db \ 46 | dbhost=$BUCARDO_NEW_HOSTNAME \ 47 | dbport=5432 \ 48 | dbname=$BUCARDO_NEW_DATABASE \ 49 | dbuser=$BUCARDO_NEW_USERNAME \ 50 | dbpass=$BUCARDO_NEW_PASSWORD 51 | 52 | # List the bucardo databases 53 | echo "The bucardo databases are:" 54 | bucardo_cmd list databases 55 | 56 | # Setup bucardo to replicate all tables and all sequences 57 | bucardo_cmd add table all --db=source_db --herd=my_herd 58 | bucardo_cmd add sequence all --db=source_db --herd=my_herd 59 | 60 | # Remove any unused tables or the ones that don't have indexes 61 | bucardo_cmd remove table public.table_foo 62 | bucardo_cmd remove table public.table_bar 63 | 64 | # Setup bucardo dbgroups 65 | bucardo_cmd add dbgroup my_group 66 | bucardo_cmd add dbgroup my_group source_db:source 67 | bucardo_cmd add dbgroup my_group target_db:source 68 | 69 | # Transfer the database schema 70 | pg_dump -v -h $BUCARDO_OLD_HOSTNAME -U $BUCARDO_OLD_USERNAME --schema-only $BUCARDO_OLD_DATABASE --file=schema.sql 71 | psql -h $BUCARDO_NEW_HOSTNAME -U $BUCARDO_NEW_USERNAME -d $BUCARDO_NEW_DATABASE -f schema.sql 72 | 73 | # Setup bucardo sync (autokick=0) ensures that nothing is transfered yet 74 | bucardo_cmd add sync $BUCARDO_SYNC_NAME herd=my_herd dbs=my_group autokick=0 75 | 76 | # Migrate the data using compression in order to minimize file size. Make sure 77 | # that you don't transfer Bucardo's data or anything else that is not managed by 78 | # you. 79 | echo "Dumping data to compressed file" 80 | time pg_dump -v \ 81 | -U $BUCARDO_OLD_USERNAME \ 82 | -h $BUCARDO_OLD_HOSTNAME \ 83 | --file=$DUMPFILE \ 84 | -N bucardo -N schema_baz \ 85 | -Fc \ 86 | $BUCARDO_OLD_DATABASE 87 | echo "Restoring data from compressed file" 88 | # Treat the new database as replica until the data restoration is over, to avoid 89 | # re-running triggers. `-j 8` is the parallelization factor, you can change it 90 | # according to your system's number of CPUs. 91 | export PGOPTIONS='-c session_replication_role=replica' 92 | time pg_restore -v \ 93 | -U $BUCARDO_NEW_USERNAME \ 94 | -h $BUCARDO_NEW_HOSTNAME \ 95 | -j 8 \ 96 | --data-only \ 97 | -d $BUCARDO_NEW_DATABASE \ 98 | $DUMPFILE 99 | export PGOPTIONS='' 100 | 101 | # Reset autokick flag and start continuous multi-master replication 102 | echo "Starting Bucardo" 103 | bucardo_cmd start 104 | bucardo_cmd update sync $BUCARDO_SYNC_NAME autokick=1 105 | bucardo_cmd reload config 106 | bucardo_cmd restart 107 | --------------------------------------------------------------------------------