├── README.md ├── config.yml └── mongo-sync /README.md: -------------------------------------------------------------------------------- 1 | [mongo-sync](https://sheharyar.me/blog/sync-mongodb-local-and-production-heroku/) 2 | ================================================================================= 3 | 4 | > _Sync Remote and Local MongoDB Databases in Bash. Works with Heroku too!_ 5 | 6 | For all the Rubyists out there, I've converted this in to a [Ruby Gem](https://github.com/sheharyarn/mongo-sync-ruby) as well. 7 | 8 | ![mongo-sync demo gif](http://i.imgur.com/hg6hwLk.gif) 9 | 10 | 11 | ## Usage 12 | 13 | - Download / Clone the script 14 | 15 | ```bash 16 | git clone https://github.com/sheharyarn/mongo-sync.git 17 | cd mongo-sync 18 | ``` 19 | 20 | - Edit `config.yml` and insert your configuration details 21 | 22 | - Use the script like this: 23 | 24 | ```bash 25 | ./mongo-sync push [options] # Push DB to Remote 26 | ./mongo-sync pull [options] # Pull DB to Local 27 | ``` 28 | - Options 29 | 30 | ``` 31 | -y # Skip confirmation 32 | --config alternate-config-file.yml 33 | ``` 34 | 35 | ## Notes 36 | 37 | - `mongo-sync` requires `mongodump` and `mongorestore` binaries to be installed in your system. If you have [`mongodb`](http://docs.mongodb.org/manual/tutorial/#getting-started) installed, then you probably already have them 38 | - Pushing/Pulling ***overwrites*** the Target DB 39 | - It's a good idea to keep your `config.yml` in `.gitignore` if you're using it inside some other project 40 | 41 | 42 | ## TODO 43 | 44 | - Add a `--no-overwrite` flag+feature that doesn't drop the target db before restoring it, and *actually* tries to sync it 45 | - Add a `backup` command and an `--auto-backup` feature 46 | - Add more options for Local DB in `config.yml` 47 | 48 | 49 | ## Contributing 50 | 51 | 1. [Fork it](https://github.com/sheharyarn/mongo-sync/fork) 52 | 2. Create your feature/fix branch (`git checkout -b feature/my-feature`) 53 | 3. Commit your changes (`git commit -am 'Add some feature'`) 54 | 4. Push to the branch (`git push origin feature/my-feature`) 55 | 5. Create a new Pull Request 56 | 57 | 58 | ## License 59 | 60 | Copyright (c) 2015 Sheharyar Naseer 61 | 62 | MIT License 63 | 64 | Permission is hereby granted, free of charge, to any person obtaining 65 | a copy of this software and associated documentation files (the 66 | "Software"), to deal in the Software without restriction, including 67 | without limitation the rights to use, copy, modify, merge, publish, 68 | distribute, sublicense, and/or sell copies of the Software, and to 69 | permit persons to whom the Software is furnished to do so, subject to 70 | the following conditions: 71 | 72 | The above copyright notice and this permission notice shall be 73 | included in all copies or substantial portions of the Software. 74 | 75 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 76 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 77 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 78 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 79 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 80 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 81 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 82 | 83 | -------------------------------------------------------------------------------- /config.yml: -------------------------------------------------------------------------------- 1 | # Mongo-Sync Configurations 2 | 3 | local: 4 | db: 'local_db_name' 5 | host: 6 | port: 27017 7 | access: 8 | username: 'local_mongo_user' 9 | password: 'local_mongo_pass' 10 | 11 | remote: 12 | db: 'remote_db_name' 13 | host: 14 | url: 'some.remoteurl.com' 15 | port: 27017 16 | access: 17 | username: 'remote_mongo_user' 18 | password: 'remote_mongo_pass' 19 | 20 | tunnel: 21 | on: false 22 | access: 23 | username: 'remote_ssh_user' 24 | port: 22 25 | 26 | # For now :local just 27 | # assumes DB is at localhost:port 28 | 29 | # All of the parameters above are required, 30 | # without them, the script will fail 31 | -------------------------------------------------------------------------------- /mongo-sync: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e # exit on error 4 | set -o pipefail # trace ERR through pipes 5 | set -o errtrace # trace ERR through 'time command' and other functions 6 | set -o errexit # exit the script if any statement returns a non-true return value 7 | 8 | function cleanup { 9 | echo "Cleaning up..." 10 | if [ -z ${TMPDIR+x} ] ; then 11 | echo -n 12 | else 13 | rm -rf $TMPDIR 14 | unset TMPDIR 15 | fi 16 | unset TUNNEL_PORT 17 | unset TUNNEL_CREDENTIALS 18 | unset LOCAL_CREDENTIALS 19 | unset REMOTE_CREDENTIALS 20 | } 21 | 22 | function error { 23 | local parent_lineno="$1" 24 | local message="$2" 25 | local code="${3:-1}" 26 | if [[ -n "$message" ]] ; then 27 | echo "Error on or near line ${parent_lineno}: ${message}; exiting with status ${code}" 28 | else 29 | echo "Error on or near line ${parent_lineno}; exiting with status ${code}" 30 | fi 31 | cleanup 32 | exit "${code}" 33 | } 34 | trap 'error ${LINENO}' ERR 35 | 36 | function usage_error { 37 | echo mongo-sync: "$1" 38 | echo "usage: mongo-sync push|pull" 39 | echo "" 40 | exit 41 | } 42 | 43 | function config_not_found { 44 | echo "failed: '$1' not found, it needs to be in the same dir" 45 | echo "aborting..." 46 | echo "" 47 | exit 48 | } 49 | 50 | function parse_yaml { 51 | local prefix=$2 52 | local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034') 53 | sed -ne "s|^\($s\):|\1|" \ 54 | -e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)[\"']$s\$|\1$fs\2$fs\3|p" \ 55 | -e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" "$1" | 56 | awk -F$fs '{ 57 | indent = length($1)/2; 58 | vname[indent] = $2; 59 | for (i in vname) {if (i > indent) {delete vname[i]}} 60 | if (length($3) > 0) { 61 | vn=""; for (i=0; i /dev/null 69 | local SCRIPT_PATH="${BASH_SOURCE[0]}" 70 | 71 | if ([ -h "${SCRIPT_PATH}" ]) then 72 | while ([ -h "${SCRIPT_PATH}" ]) do cd `dirname "$SCRIPT_PATH"`; SCRIPT_PATH=`readlink "${SCRIPT_PATH}"`; done 73 | fi 74 | 75 | cd `dirname ${SCRIPT_PATH}` > /dev/null 76 | local SCRIPT_PATH=`pwd`; 77 | popd > /dev/null 78 | 79 | echo $SCRIPT_PATH 80 | } 81 | 82 | function get_confirmation() { 83 | read -p "Are you sure you want to $1? Enter 'yes': " mongo_confr 84 | 85 | case $mongo_confr in 86 | [yY][Ee][Ss] ) ;; 87 | *) echo "Incorrect input, aborting..."; exit;; 88 | esac 89 | } 90 | 91 | function load_configs { 92 | echo "load_configs" 93 | if [ -z "$CONFIG_FILE" ] ; then 94 | CONFIG_FILE="config.yml" 95 | fi 96 | local config=$CONFIG_FILE 97 | echo "Parsing '$config'..." 98 | 99 | DIR=$(get_script_dir) 100 | local FILE="$DIR/$config" 101 | 102 | if [ -f "$FILE" ]; then 103 | eval $(parse_yaml "$FILE") 104 | success_msg 105 | else 106 | config_not_found $config 107 | fi 108 | 109 | # Loads: 110 | # - $local_db 111 | # - $local_host_port 112 | # - $local_access_username 113 | # - $local_access_password 114 | # - $remote_db 115 | # - $remote_host_url 116 | # - $remote_host_port 117 | # - $remote_access_username 118 | # - $remote_access_password 119 | # - $tunnel_on 120 | # - $tunnel_access_username 121 | # - $tunnel_access_port 122 | 123 | LOCAL_CREDENTIALS="" 124 | if [[ ! -z $local_access_username ]] ; then 125 | LOCAL_CREDENTIALS="-u $local_access_username -p $local_access_password" 126 | fi 127 | 128 | REMOTE_CREDENTIALS="" 129 | if [[ ! -z $remote_access_username ]] ; then 130 | REMOTE_CREDENTIALS="-u $remote_access_username -p $remote_access_password" 131 | fi 132 | 133 | TUNNEL_CREDENTIALS="$tunnel_access_username@$remote_host_url" 134 | TUNNEL_PORT=27018 135 | 136 | TMPDIR=/tmp/"$local_db"/dump 137 | } 138 | 139 | function banner { 140 | echo mongo-sync: 141 | echo ----------- 142 | } 143 | 144 | function success_msg { 145 | echo "Success!" 146 | echo 147 | } 148 | 149 | function done_msg { 150 | echo "Done!" 151 | echo 152 | } 153 | 154 | function open_tunnel { 155 | echo "Connecting through SSH tunnel..." 156 | ssh -fqTNM -S db-sync-socket -L $TUNNEL_PORT:127.0.0.1:$remote_host_port $TUNNEL_CREDENTIALS -p $tunnel_access_port 157 | } 158 | 159 | function close_tunnel { 160 | echo "Disconnecting from SSH tunnel..." 161 | ssh -S db-sync-socket -O exit $TUNNEL_CREDENTIALS 2> /dev/null 162 | } 163 | 164 | 165 | function pull { 166 | banner 167 | if [ "$1" == false ] ; then 168 | get_confirmation 'pull' 169 | fi 170 | load_configs 171 | 172 | if [ "$tunnel_on" == true ] ; then 173 | open_tunnel 174 | remote_host_url="localhost" 175 | remote_host_port=$TUNNEL_PORT 176 | fi 177 | 178 | echo "Dumping Remote DB to $TMPDIR... " 179 | mongodump \ 180 | -h "$remote_host_url":"$remote_host_port" \ 181 | -d "$remote_db" \ 182 | $REMOTE_CREDENTIALS \ 183 | -o "$TMPDIR" > /dev/null 184 | success_msg 185 | 186 | echo "Overwriting Local DB with Dump... " 187 | mongorestore \ 188 | --port "$local_host_port" \ 189 | -d "$local_db" \ 190 | $LOCAL_CREDENTIALS \ 191 | "$TMPDIR"/"$remote_db" \ 192 | --drop > /dev/null 193 | success_msg 194 | 195 | if [ "$tunnel_on" == true ] ; then 196 | close_tunnel 197 | fi 198 | 199 | cleanup 200 | done_msg 201 | } 202 | 203 | function push { 204 | banner 205 | if [ "$1" == false ] ; then 206 | get_confirmation 'push' 207 | fi 208 | load_configs 209 | 210 | if [ "$tunnel_on" == true ] ; then 211 | open_tunnel 212 | remote_host_url="localhost" 213 | remote_host_port=$TUNNEL_PORT 214 | fi 215 | 216 | echo "Dumping Local DB to $TMPDIR... " 217 | mongodump \ 218 | --port "$local_host_port" \ 219 | -d "$local_db" \ 220 | $LOCAL_CREDENTIALS \ 221 | -o "$TMPDIR" > /dev/null 222 | success_msg 223 | 224 | echo "Overwriting Remote DB with Dump... " 225 | mongorestore \ 226 | -h "$remote_host_url":"$remote_host_port" \ 227 | -d "$remote_db" \ 228 | $REMOTE_CREDENTIALS \ 229 | "$TMPDIR"/"$local_db" --drop > /dev/null 230 | success_msg 231 | 232 | if [ "$tunnel_on" == true ] ; then 233 | close_tunnel 234 | fi 235 | 236 | cleanup 237 | done_msg 238 | } 239 | 240 | 241 | ## MAIN 242 | ## ==== 243 | 244 | # TODO: Add --overwrite flag 245 | 246 | if [[ $# -eq 0 ]] ; then 247 | usage_error "no arguments provided" 248 | else 249 | action="$1" 250 | skip=false 251 | shift 252 | 253 | while [ "${1+defined}" ]; do 254 | if [ "$1" == '--config' ] ; then 255 | shift 256 | if [ -e "$1" ] ; then 257 | CONFIG_FILE="$1" 258 | else 259 | usage_error "No config file named \"$1\"" 260 | fi 261 | elif [ "$1" == '-y' ] ; then 262 | skip=true 263 | else 264 | usage_error "unrecognized option \"$1\"" 265 | fi 266 | shift 267 | done 268 | 269 | if [[ "$action" == 'push' ]] ; then 270 | push $skip 271 | elif [[ "$action" == 'pull' ]] ; then 272 | pull $skip 273 | else 274 | usage_error "unrecognized command \"$action\"" 275 | fi 276 | fi 277 | --------------------------------------------------------------------------------