├── .drone.yml ├── Dockerfile ├── README.md └── scripts ├── backup.sh ├── check.sh ├── compact.sh ├── delete.sh ├── entry.sh ├── list.sh ├── notif-test.sh ├── restore.sh ├── scripts_common.sh └── setup.sh /.drone.yml: -------------------------------------------------------------------------------- 1 | kind: pipeline 2 | type: docker 3 | name: default 4 | 5 | steps: 6 | - name: publish-image 7 | image: plugins/docker 8 | settings: 9 | username: 10 | from_secret: public_docker_username 11 | password: 12 | from_secret: public_docker_password 13 | auto_tag: true 14 | repo: layr/borg-mysql-backup 15 | 16 | trigger: 17 | ref: 18 | - refs/heads/master 19 | - refs/heads/develop 20 | - refs/heads/feature/* 21 | - refs/tags/* 22 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3 2 | 3 | ENV LANG=C.UTF-8 \ 4 | BORG_VERSION=1.4.0-r0 5 | 6 | ADD scripts/* /usr/local/sbin/ 7 | 8 | RUN apk update && \ 9 | apk add --no-cache \ 10 | grep curl bash mariadb-client postgresql-client ca-certificates tzdata msmtp logrotate \ 11 | openssh-client \ 12 | openssh-keygen \ 13 | borgbackup=$BORG_VERSION \ 14 | docker-cli && \ 15 | chown -R root:root /usr/local/sbin/ && \ 16 | chmod -R 755 /usr/local/sbin/ && \ 17 | mkdir /root/.ssh && \ 18 | ln -s /usr/local/sbin/setup.sh /setup.sh && \ 19 | ln -s /usr/local/sbin/backup.sh /backup.sh && \ 20 | ln -s /usr/local/sbin/scripts_common.sh /scripts_common.sh && \ 21 | rm -rf /var/cache/apk/* /tmp/* 22 | 23 | VOLUME ["/root/.cache/borg", "/root/.config/borg"] 24 | ENTRYPOINT ["/usr/local/sbin/entry.sh"] 25 | 26 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # borg-mysql-backup 2 | 3 | This image is for backing up mysql or postgres dumps to local and/or remote 4 | [borg](https://github.com/borgbackup/borg) repos. 5 | Other files&dirs may be included in the backup, and database dumps can be excluded 6 | altogether. 7 | 8 | The container features `backup`, `restore`, `list`, `delete` and `notif-test` scripts that can 9 | either be ran as one-off jobs or by cron - latter being the preferred method for `backup`. 10 | 11 | For cron and/or remote borg usage, you also need to mount container configuration 12 | at `/config`, containing crontab file (named `crontab`) and/or ssh key (named `id_rsa`). 13 | Note ssh key, when provided, is expected to be passwordless. 14 | 15 | Both remote & local repositories need to be 16 | [initialised](https://borgbackup.readthedocs.io/en/stable/usage/init.html) manually 17 | beforehand. You still may use this image to do so - just start a container in an 18 | interactive mode and use the shell. 19 | 20 | In case some containers need to be stopped for the backup (eg to ensure there won't 21 | be any mismatch between data and database), you can specify those container names to 22 | the `backup` script (see below). Note that this requires mounting docker socket with 23 | `-v /var/run/docker.sock:/var/run/docker.sock`, but keep in mind it has security 24 | implications - borg-mysql-backup will have essentially root permissions on the host. 25 | 26 | To synchronize container tz with that of host's, then also add following mount: 27 | `-v /etc/localtime:/etc/localtime:ro`. You'll likely want to do this for cron times 28 | to match the host time. 29 | 30 | It's possible to get notified of _any_ errors that occur during backups. 31 | Currently supported notification methods are 32 | 33 | - sending mail via SMTP 34 | - sending [pushover](https://pushover.net/) notifications 35 | - posting to [healthchecks.io](https://healthchecks.io/) `/fail` endpoint (see 36 | note on healtchchecks in following paragraphs) 37 | 38 | If you wish to provide your own msmtprc config file instead of defining `SMTP_*` env 39 | vars, create it at the `/config` mount, named `msmtprc`. 40 | 41 | Dead man's switch support is provided via healthchecks; healthcheck provider will 42 | always be pinged when backup job runs - regardless of the outcome; ie it's only 43 | there to monitor the scheduled backup is being executed, not that it succeeds. 44 | For error notifications you still need to configure notifications (see `ERR_NOTIF`). 45 | Note if you've configured healthchecks.io as your healthcheck provider, then you 46 | may also use it for error notifications (see above). 47 | 48 | Additionally, following bindings are _strongly_ recommended: 49 | `-v /host/borg-conf/.borg/cache:/root/.cache/borg` 50 | `-v /host/borg-conf/.borg/config:/root/.config/borg` 51 | You might want to change where borg conf & cache are located via 52 | `BORG_CONFIG_DIR` & `BORG_CACHE_DIR` env vars as described [in docs](https://borgbackup.readthedocs.io/en/stable/usage/general.html?highlight=BORG_CACHE_DIR#environment-variables) 53 | 54 | You might also wish to expose the logs: 55 | `-v /host/borg-conf/logs:/var/log` 56 | 57 | Every time any config is changed in `/config`, container needs to be restarted for 58 | the changes to get picked up. 59 | 60 | 61 | ## Repo init examples: 62 | 63 | ### For rsync.net (or other remote location): 64 | 65 | Running command from your computer: `BORG_REMOTE_PATH=borg14 borg init 66 | --encryption=repokey 12345@ch-s010.rsync.net:dir/path` 67 | 68 | ### Local repo: 69 | 70 | `borg init --encryption=repokey /path/to/local/repo` 71 | 72 | 73 | ## Be careful 74 | 75 | **Please make sure to verify you're able to access your offsite (ie remote) 76 | backups without your local repo/config! You don't want to find yourself unable 77 | to access remote backups in case local configuration/repository gets nuked.** 78 | Refer to [brog docs](https://borgbackup.readthedocs.io/en/stable/quickstart.html#repository-encryption) 79 | as to how and what to back up, depending on the encryption mode used. 80 | 81 | You should be able to access your offsite backups from _any_ system. 82 | 83 | Remember - "Untested backup is no backup at all" 84 | 85 | 86 | ## Container Parameters 87 | 88 | Note all `BORG_`-prefixed env vars are [borg native ones](https://borgbackup.readthedocs.io/en/stable/usage/general.html#environment-variables). 89 | 90 | MYSQL_HOST the host/ip of your mysql database 91 | MYSQL_PORT the port number of your mysql database 92 | MYSQL_USER the username of your mysql database 93 | MYSQL_PASS the password of your mysql database 94 | MYSQL_FAIL_FATAL whether unsuccessful db dump should abort backup, 95 | defaults to 'true'; 96 | MYSQL_EXTRA_OPTS the extra options to pass to 'mariadb-dump' command; optional 97 | 98 | POSTGRES_HOST the host/ip of your postgresql database 99 | POSTGRES_PORT the port number of your postgresql database 100 | POSTGRES_USER the username of your postgresql database 101 | POSTGRES_PASS the password of your postgresql database 102 | POSTGRES_FAIL_FATAL whether unsuccessful db dump should abort backup, 103 | defaults to 'true'; 104 | POSTGRES_EXTRA_OPTS the extra options to pass to 'pg_dump' or 'pg_dumpall' commands; optional 105 | mysql & postgres env variables are only required if you intend to back up databases 106 | 107 | 108 | HOST_ID host identifier to include in the borg archive name 109 | REMOTE remote connection - user & host with optional port; eg for 110 | rsync.net it'd be something like '12345@ch-s010.rsync.net[:1234]' 111 | optional - can be omitted when only backing up to local 112 | borg repo, or if providing value via script 113 | REMOTE_REPO path to repo on remote host, eg '/backup/repo' 114 | optional - can be omitted when only backing up to local 115 | borg repo, or if providing value via script 116 | LOCAL_REPO path to local borg repo; optional - can be omitted 117 | when only backing up to remote borg repo, or if 118 | providing value via script 119 | COMMON_OPTS additional borg params to be used with _all_ borg 120 | commands. see 121 | https://borgbackup.readthedocs.io/en/stable/usage/general.html#common-options 122 | CREATE_OPTS additional borg params to the borg backup command 123 | (for both local & remote borg commands); optional 124 | LOCAL_CREATE_OPTS additional borg params for local borg backup command; optional 125 | REMOTE_CREATE_OPTS additional borg params for remote borg backup command; optional 126 | BORG_REMOTE_PATH remote borg executable path; eg with rsync.net 127 | you'd want to use value 'borg14'; optional 128 | BORG_PASSPHRASE borg repo password 129 | PRUNE_OPTS options for borg prune (both local and remote); not 130 | required when it's defined by backup script -P param 131 | (which overrides this container env var) 132 | LOCAL_PRUNE_OPTS prune options for local borg repo; overrides PRUNE_OPTS; 133 | REMOTE_PRUNE_OPTS prune options for remote borg repo; overrides PRUNE_OPTS; 134 | SCRIPT_FAIL_FATAL whether failure of custom script execution should abort 135 | backup, defaults to 'true'; 136 | NOTIF_MISSING_EXCL_PTH set to non-empty value if any of the configured excluded 137 | file paths should raise a warning notification; 138 | this is to find potential misconfigurations 139 | 140 | HC_URL healthcheck url to ping upon script completion; may contain 141 | {id} placeholder to define general template and provide the 142 | unique/id value via backup script option HC_ID 143 | ERR_NOTIF comma separated error notification methods; supported values 144 | are {mail,pushover,healthchecksio}; optional 145 | NOTIF_SUBJECT notifications' subject/title; defaults to '{p}: backup error on {h}' 146 | ADD_NOTIF_TAIL whether all error messages should contain the 147 | trailing block of additional info; defaults to 'true'; 148 | NOTIF_TAIL_MSG replaces the default contents of trailing error 149 | notifications; only in effect if ADD_NOTIF_TAIL=true 150 | 151 | following {MAIL,SMTP}_* params are only used if ERR_NOTIF value contains 'mail'; 152 | also note all SMTP_* env vars besides SMTP_ACCOUNT are ignored if you've 153 | provided smtp config file at /config/msmtprc 154 | MAIL_TO address to send notifications to 155 | MAIL_FROM name of the notification sender; defaults to '{h} backup reporter' 156 | SMTP_HOST smtp server host; only required if MSMTPRC file not provided 157 | SMTP_USER login user to the smtp account; only required if MSMTPRC file not provided 158 | SMTP_PASS login password to the smtp account; only required if MSMTPRC file not provided 159 | SMTP_PORT smtp server port; defaults to 587 160 | SMTP_AUTH defaults to 'on' 161 | SMTP_TLS defaults to 'on' 162 | SMTP_STARTTLS defaults to 'on' 163 | SMTP_ACCOUNT smtp account to use for sending mail; 164 | makes sense only if you've provided your own MSMTPRC 165 | config at /config/msmtprc that defines multiple accounts 166 | 167 | following params are only used/required if ERR_NOTIF value contains 'pushover': 168 | PUSHOVER_USER_KEY your pushover account key 169 | PUSHOVER_APP_TOKEN token of a registered app to send notifications from 170 | PUSHOVER_PRIORITY defaults to 1 171 | PUSHOVER_RETRY only in use if priority=2; defaults to 60 172 | PUSHOVER_EXPIRE only in use if priority=2; defaults to 3600 173 | 174 | ## Script usage 175 | 176 | Container incorporates `backup`, `compact`, `restore`, `list`, `delete`, 177 | `check` and `notif-test` scripts. 178 | 179 | ### backup.sh 180 | 181 | `backup` script is mostly intended to be ran by cron, but can also be executed 182 | as a one-off command for a single backup. 183 | 184 | usage: backup [-h] [-d MYSQL_DBS] [-g POSTGRES_DBS] [-c CONTAINERS] [-rl] 185 | [-P PRUNE_OPTS] [-B|-Z CREATE_OPTS] [-E EXCLUDE_PATHS] 186 | [-L LOCAL_REPO] [-e ERR_NOTIF] [-A SMTP_ACCOUNT] [-D MYSQL_FAIL_FATAL] 187 | [-G POSTGRES_FAIL_FATAL] [-S SCRIPT_FAIL_FATAL] [-R REMOTE] 188 | [-T REMOTE_REPO] [-C] [-H HC_ID] -p PREFIX [NODES_TO_BACK_UP...] 189 | 190 | Create new archive 191 | 192 | arguments: 193 | -h show help and exit 194 | -d MYSQL_DBS comma-separated mysql database names to back up; use value of 195 | __all__ to back up all dbs on the server 196 | -g POSTGRES_DBS comma-separated postgresql database names to back up; use value of 197 | __all__ to back up all dbs on the server 198 | -c CONTAINERS comma-separated container names to stop for the backup process; 199 | requires mounting the docker socket (-v /var/run/docker.sock:/var/run/docker.sock); 200 | note containers will be stopped in given order; after backup 201 | completion, containers are started in reverse order; only containers 202 | that were stopped by the script will be re-started afterwards 203 | -r only back to remote borg repo (remote-only) 204 | -l only back to local borg repo (local-only) 205 | -P PRUNE_OPTS overrides container env var of same name; only required when 206 | container var is not defined or needs to be overridden; 207 | -1 LOCAL_PRUNE_OPTS prune options for local borg repo; overrides PRUNE_OPTS (& -P); 208 | -2 REMOTE_PRUNE_OPTS prune options for remote borg repo; overrides PRUNE_OPTS (& -P); 209 | -B CREATE_OPTS additional borg params; note it doesn't overwrite the 210 | env var of same name, but extends it; 211 | -Z CREATE_OPTS additional borg params; note it _overrides_ the env 212 | var of same name; 213 | -E EXCLUDE_PATHS comma-separated paths to exclude from backup; 214 | [-E '/p1,/p2'] would be equivalent to [-B '-e /p1 -e /p2'] 215 | -L LOCAL_REPO overrides container env var of same name; 216 | -e ERR_NOTIF overrides container env var of same name; 217 | -A SMTP_ACCOUNT overrides container env var of same name; 218 | -D MYSQL_FAIL_FATAL overrides container env var of same name; 219 | -G POSTGRES_FAIL_FATAL overrides container env var of same name; 220 | -S SCRIPT_FAIL_FATAL overrides container env var of same name; 221 | -R REMOTE overrides container env var of same name; 222 | -T REMOTE_REPO overrides container env var of same name; 223 | -C run `compact` command against repo after backup/prune; 224 | -H HC_ID the unique/id part of healthcheck url, replacing the '{id}' 225 | placeholder in HC_URL; may also provide new full url to call 226 | instead, overriding the env var HC_URL 227 | -p PREFIX borg archive name prefix. note that the full archive name already 228 | contains HOST_ID env var and timestamp, so omit those. 229 | NODES_TO_BACK_UP... last arguments to backup.sh are files&directories to be 230 | included in the backup 231 | 232 | #### Usage examples 233 | 234 | ##### Back up App1 & App2 mysql databases and app1's data directory /app1-data daily at 05:15 to both local and remote borg repos 235 | 236 | docker run -d \ 237 | -e MYSQL_HOST=mysql.host \ 238 | -e MYSQL_PORT=27017 \ 239 | -e MYSQL_USER=admin \ 240 | -e MYSQL_PASS=password \ 241 | -e HOST_ID=hostname-to-use-in-archive-prefix \ 242 | -e REMOTE=remoteuser@server.com \ 243 | -e REMOTE_REPO=repo/location \ 244 | -e LOCAL_REPO=/backup/repo \ 245 | -e CREATE_OPTS='--compression zlib,5 --lock-wait 60' \ 246 | -e BORG_PASSPHRASE=borgrepopassword \ 247 | -e PRUNE_OPTS='--keep-daily=7 --keep-weekly=4' \ 248 | -v /etc/localtime:/etc/localtime:ro \ 249 | -v /host/backup:/backup \ 250 | -v /host/borg-conf:/config:ro \ 251 | -v /host/borg-conf/.borg/cache:/root/.cache/borg \ 252 | -v /host/borg-conf/.borg/config:/root/.config/borg \ 253 | -v /host/borg-conf/logs:/var/log \ 254 | -v /app1-data-on-host:/app1-data:ro \ 255 | layr/borg-mysql-backup 256 | 257 | `/config/crontab` contents: 258 | 259 | 15 05 * * * /backup.sh -p app1-app2 -d "App1,App2" /app1-data 260 | 261 | ##### Back up all postgres databases daily at 04:10 and 16:10 to local&remote borg repos, stopping containers myapp1 & myapp2 for the process 262 | 263 | docker run -d \ 264 | -e POSTGRES_HOST=postgre.host \ 265 | -e POSTGRES_PORT=5432 \ 266 | -e POSTGRES_USER=postgres \ 267 | -e POSTGRES_PASS=password \ 268 | -e HOST_ID=hostname-to-use-in-archive-prefix \ 269 | -e REMOTE=remoteuser@server.com \ 270 | -e REMOTE_REPO=repo/location \ 271 | -e LOCAL_REPO=/backup/repo \ 272 | -e BORG_PASSPHRASE=borgrepopassword \ 273 | -e PRUNE_OPTS='--keep-daily=7 --keep-weekly=4' \ 274 | -v /var/run/docker.sock:/var/run/docker.sock \ 275 | -v /etc/localtime:/etc/localtime:ro \ 276 | -v /host/backup:/backup \ 277 | -v /host/borg-conf:/config:ro \ 278 | -v /host/borg-conf/.borg/cache:/root/.cache/borg \ 279 | -v /host/borg-conf/.borg/config:/root/.config/borg \ 280 | -v /host/borg-conf/logs:/var/log \ 281 | layr/borg-mysql-backup 282 | 283 | `/config/crontab` contents: 284 | 285 | 10 04,16 * * * /backup.sh -p myapp-prefix -g __all__ -c "myapp1,myapp2" 286 | 287 | ##### Back up directories /app1 & /app2 every 6 hours to local borg repo (ie remote is excluded) 288 | 289 | docker run -d \ 290 | -e HOST_ID=hostname-to-use-in-archive-prefix \ 291 | -e LOCAL_REPO=/backup/repo \ 292 | -e BORG_PASSPHRASE=borgrepopassword \ 293 | -e PRUNE_OPTS='--keep-daily=7 --keep-weekly=4' \ 294 | -e HC_URL='https://hc-ping.com/{id}' \ 295 | -v /host/backup:/backup \ 296 | -v /host/borg-conf:/config:ro \ 297 | -v /host/borg-conf/.borg/cache:/root/.cache/borg \ 298 | -v /host/borg-conf/.borg/config:/root/.config/borg \ 299 | -v /host/borg-conf/logs:/var/log \ 300 | -v /app1:/app1:ro \ 301 | -v /app2:/app2:ro \ 302 | layr/borg-mysql-backup 303 | 304 | `/config/crontab` contents: 305 | 306 | 0 */6 * * * /backup.sh -l -p my_app_prefix -H eb095278-f28d-448d-87fb-7b75c171a6aa /app1 /app2 307 | 308 | Note we didn't need to define mysql- or remote borg repo related docker env vars. 309 | Also there's no need to have ssh key in `/config`, as we're not connecting to a remote server. 310 | Additionally, there was no need to mount `/etc/localtime`, as cron doesn't 311 | define absolute time, but simply an interval. 312 | Note also how we define the healthcheck url HC_URL template, whose {id} placeholder 313 | is replaced by -H value provided by backup.sh 314 | 315 | ##### Same as above, but report errors via mail 316 | 317 | Use same docker command as above, with following env vars added: 318 | 319 | -e ERR_NOTIF=mail \ 320 | -e MAIL_TO=receiver@example.com \ 321 | -e NOTIF_SUBJECT='{i} backup error' \ 322 | -e SMTP_HOST='smtp.gmail.com' \ 323 | -e SMTP_USER='your.google.username' \ 324 | -e SMTP_PASS='your-google-app-password-you-created-for-this' \ 325 | 326 | Same as the example before, but we've also opted to get notified of any backup 327 | errors via email. 328 | 329 | ##### Back up directory /emby once to remote borg repo (ie local is excluded) 330 | 331 | docker run -it --rm \ 332 | -e HOST_ID=hostname-to-use-in-archive-prefix \ 333 | -e REMOTE=remoteuser@server.com \ 334 | -e REMOTE_REPO=repo/location \ 335 | -e BORG_PASSPHRASE=borgrepopassword \ 336 | -e PRUNE_OPTS='--keep-daily=7 --keep-weekly=4' \ 337 | -e HC_URL='https://hc-ping.com/eb095278-f28d-448d-87fb-7b75c171a6aa' \ 338 | -v /host/borg-conf:/config:ro \ 339 | -v /host/borg-conf/.borg/cache:/root/.cache/borg \ 340 | -v /host/borg-conf/.borg/config:/root/.config/borg \ 341 | -v /host/borg-conf/logs:/var/log \ 342 | -v /host/emby/dir:/emby:ro \ 343 | layr/borg-mysql-backup backup.sh -r -p emby /emby 344 | 345 | Note there's no need to have a crontab file in `/config`, as we're executing this 346 | command just once, after which container exits and is removed (ie we're not using 347 | scheduled backups). Also note there's no `/backup` mount for local borg repo as 348 | we're operating only against the remote borg repo. 349 | Note also the healtcheck url that will get pinged. 350 | 351 | ### restore.sh 352 | 353 | `restore` script should be executed directly with docker in interactive mode. All data 354 | will be extracted into `/$RESTORE_DIR/restored-{archive_name}`. 355 | 356 | Note none of the data is 357 | copied/moved automatically - user is expected to carry this operation out on their own. 358 | Only db will be restored from a dump, given the option is provided to the script. 359 | 360 | usage: restore [-h] [-d] [-g] [-c CONTAINERS] [-rl] [-B BORG_OPTS] [-L LOCAL_REPO] 361 | [-R REMOTE] [-T REMOTE_REPO] -O RESTORE_DIR -a ARCHIVE_NAME 362 | 363 | Restore data from borg archive 364 | 365 | arguments: 366 | -h show help and exit 367 | -d automatically restore mysql database from dumped file; if this 368 | option is provided and archive doesn't contain exactly one dump-file, 369 | it's an error; be careful, this is a destructive operation! 370 | -g automatically restore postgresql database from dumped file; if this 371 | option is provided and archive contains no sql dumps, it's an error; 372 | be careful, this is a destructive operation! 373 | -c CONTAINERS comma-separated container names to stop before the restore begins; 374 | note they won't be started afterwards, as there might be need 375 | to restore other data (only sql dumps are restored automatically); 376 | requires mounting the docker socket (-v /var/run/docker.sock:/var/run/docker.sock) 377 | -r restore from remote borg repo 378 | -l restore from local borg repo 379 | -B BORG_OPTS additional borg params to pass to extract command 380 | -L LOCAL_REPO overrides container env var of same name 381 | -R REMOTE overrides container env var of same name 382 | -T REMOTE_REPO overrides container env var of same name 383 | -O RESTORE_DIR path to directory where archive will get extracted to 384 | -a ARCHIVE_NAME full name of the borg archive to extract data from 385 | 386 | #### Usage examples 387 | 388 | ##### Restore archive from remote borg repo & restore mysql with restored dumpfile 389 | 390 | docker run -it --rm \ 391 | -e MYSQL_HOST=mysql.host \ 392 | -e MYSQL_PORT=27017 \ 393 | -e MYSQL_USER=admin \ 394 | -e MYSQL_PASS=password \ 395 | -e REMOTE=remoteuser@server.com \ 396 | -e REMOTE_REPO=repo/location \ 397 | -e BORG_PASSPHRASE=borgrepopassword \ 398 | -v /host/backup:/backup \ 399 | -v /host/borg-conf:/config:ro \ 400 | -v /host/borg-conf/.borg/cache:/root/.cache/borg \ 401 | -v /host/borg-conf/.borg/config:/root/.config/borg \ 402 | -v /host/borg-conf/logs:/var/log \ 403 | layr/borg-mysql-backup restore.sh -r -d -O /backup -a my_prefix-HOSTNAME-2017-02-27-160159 404 | 405 | ##### Restore archive from local borg repo & stop container app1 beforehand 406 | 407 | docker run -it --rm \ 408 | -e LOCAL_REPO=/backup/repo \ 409 | -e BORG_PASSPHRASE=borgrepopassword \ 410 | -v /var/run/docker.sock:/var/run/docker.sock \ 411 | -v /host/backup:/backup \ 412 | -v /host/borg-conf/.borg/cache:/root/.cache/borg \ 413 | -v /host/borg-conf/.borg/config:/root/.config/borg \ 414 | -v /host/borg-conf/logs:/var/log \ 415 | layr/borg-mysql-backup restore.sh -l -c app1 -O /backup -a my_prefix-HOSTNAME-2017-02-27-160159 416 | 417 | Note there's no need to mount `/config`, as we're not using cron nor connecting to remote borg. 418 | Also we're not providing mysql env vars, as script isn't invoked with `-d` option, meaning 419 | db won't be automatically restored with the included .sql dumpfile (if there was one). 420 | 421 | ##### Restore archive from overridden local borg repo 422 | 423 | docker run -it --rm \ 424 | -e LOCAL_REPO=/backup/repo \ 425 | -v /host/backup:/backup \ 426 | -v /host/borg-conf/.borg/cache:/root/.cache/borg \ 427 | -v /host/borg-conf/.borg/config:/root/.config/borg \ 428 | -v /host/borg-conf/logs:/var/log \ 429 | layr/borg-mysql-backup restore.sh -l -L /backup/otherrepo -O /backup -a my_prefix-HOSTNAME-2017-02-27-160159 430 | 431 | Data will be restored from a local borg repo `/backup/otherrepo` that overrides the 432 | env-var-configured value `/backup/repo`. Also note missing 433 | env variable `BORG_PASSPHRASE`, which will be required to be typed in manually. 434 | 435 | Note the `CREATE_OPTS`, `LOCAL_CREATE_OPTS`, `REMOTE_CREATE_OPTS` env 436 | variables are not usable with `restore`. 437 | 438 | ### list.sh 439 | 440 | `list` script is for listing archives in a borg repo or contents of an archive. 441 | 442 | usage: list [-h] [-rl] [-p ARCHIVE_PREFIX] [-B BORG_OPTS] [-L LOCAL_REPO] 443 | [-R REMOTE] [-T REMOTE_REPO] [-a ARCHIVE_NAME] 444 | 445 | List archives in a borg repository or contents of an archive 446 | 447 | arguments: 448 | -h show help and exit 449 | -r list remote borg repo 450 | -l list local borg repo 451 | -p ARCHIVE_PREFIX list archives with given prefix; same as providing 452 | [-B '--glob-archives ARCHIVE_PREFIX*'] 453 | -B BORG_OPTS additional borg params to pass to borg list command 454 | -L LOCAL_REPO overrides container env var of same name 455 | -R REMOTE overrides container env var of same name 456 | -T REMOTE_REPO overrides container env var of same name 457 | -a ARCHIVE_NAME full name of the borg archive whose contents to list 458 | 459 | #### Usage examples 460 | 461 | ##### List the local repository contents 462 | 463 | docker run -it --rm \ 464 | -e BORG_PASSPHRASE=borgrepopassword \ 465 | -v /host/backup:/backup \ 466 | -v /host/borg-conf/.borg/cache:/root/.cache/borg \ 467 | -v /host/borg-conf/.borg/config:/root/.config/borg \ 468 | -v /host/borg-conf/logs:/var/log \ 469 | layr/borg-mysql-backup list.sh -l -L /backup/repo 470 | 471 | ##### List the remote repository contents 472 | 473 | docker run -it --rm \ 474 | -e REMOTE=remoteuser@server.com \ 475 | -e BORG_PASSPHRASE=borgrepopassword \ 476 | -v /host/borg-conf:/config:ro \ 477 | -v /host/borg-conf/.borg/cache:/root/.cache/borg \ 478 | -v /host/borg-conf/.borg/config:/root/.config/borg \ 479 | -v /host/borg-conf/logs:/var/log \ 480 | layr/borg-mysql-backup list.sh -r -T repo/location -p my-prefix 481 | 482 | Note the `CREATE_OPTS`, `LOCAL_CREATE_OPTS`, `REMOTE_CREATE_OPTS` env 483 | variables are not usable with `list`. 484 | 485 | ### delete.sh 486 | 487 | `delete` script is for deleting archives in a borg repo (or whole repo itself) 488 | 489 | usage: delete [-h] [-rl] [-p ARCHIVE_PREFIX] [-a ARCHIVE] [-B BORG_OPTS] 490 | [-L LOCAL_REPO] [-R REMOTE] [-T REMOTE_REPO] 491 | 492 | Delete whole borg repository or archives in it 493 | 494 | arguments: 495 | -h show help and exit 496 | -r only delete from remote borg repo (remote-only) 497 | -l only delete from local borg repo (local-only) 498 | -p ARCHIVE_PREFIX delete archives with given prefix; same as providing 499 | -B '--glob-archives ARCHIVE_PREFIX*' 500 | -a ARCHIVE archive name to delete; -p & -a are mutually exclusive 501 | -B BORG_OPTS additional borg params to pass to borg delete command 502 | -L LOCAL_REPO overrides container env var of same name 503 | -R REMOTE overrides container env var of same name 504 | -T REMOTE_REPO overrides container env var of same name 505 | 506 | #### Usage examples 507 | 508 | ##### Delete the local archives starting with 'prefix-HOST' 509 | 510 | docker run -it --rm \ 511 | -e BORG_PASSPHRASE=borgrepopassword \ 512 | -v /host/backup:/backup \ 513 | -v /host/borg-conf/.borg/cache:/root/.cache/borg \ 514 | -v /host/borg-conf/.borg/config:/root/.config/borg \ 515 | -v /host/borg-conf/logs:/var/log \ 516 | layr/borg-mysql-backup delete.sh -l -p 'prefix-HOST' 517 | 518 | ##### Delete a specific archive from local repository 519 | 520 | docker run -it --rm \ 521 | -e BORG_PASSPHRASE=borgrepopassword \ 522 | -v /host/backup:/backup \ 523 | -v /host/borg-conf/.borg/cache:/root/.cache/borg \ 524 | -v /host/borg-conf/.borg/config:/root/.config/borg \ 525 | -v /host/borg-conf/logs:/var/log \ 526 | layr/borg-mysql-backup delete.sh -l -a 'prefix-HOST-timestamp' 527 | 528 | ##### Delete the contents of a whole remote repository 529 | 530 | docker run -it --rm \ 531 | -e REMOTE=remoteuser@server.com \ 532 | -e BORG_PASSPHRASE=borgrepopassword \ 533 | -v /host/borg-conf:/config:ro \ 534 | -v /host/borg-conf/.borg/cache:/root/.cache/borg \ 535 | -v /host/borg-conf/.borg/config:/root/.config/borg \ 536 | -v /host/borg-conf/logs:/var/log \ 537 | layr/borg-mysql-backup delete.sh -r -T repo/location 538 | 539 | Note the `CREATE_OPTS`, `LOCAL_CREATE_OPTS`, `REMOTE_CREATE_OPTS` env 540 | variables are not usable with `delete`. 541 | 542 | ### compact.sh 543 | 544 | `compact` script is for freeing repository space by compacting segments. 545 | 546 | usage: compact [-h] [-rl] [-B BORG_OPTS] [-L LOCAL_REPO] 547 | [-R REMOTE] [-T REMOTE_REPO] 548 | 549 | 550 | Compact borg repository 551 | 552 | arguments: 553 | -h show help and exit 554 | -r compact only remote borg repo (remote-only) 555 | -l compact only local borg repo (local-only) 556 | -B BORG_OPTS additional borg params to pass to borg compact command 557 | -L LOCAL_REPO overrides container env var of same name 558 | -R REMOTE overrides container env var of same name 559 | -T REMOTE_REPO overrides container env var of same name 560 | 561 | 562 | #### Usage examples 563 | 564 | ##### Compact the local repository 565 | 566 | docker run -it --rm \ 567 | -e BORG_PASSPHRASE=borgrepopassword \ 568 | -v /host/backup:/backup \ 569 | -v /host/borg-conf/.borg/cache:/root/.cache/borg \ 570 | -v /host/borg-conf/.borg/config:/root/.config/borg \ 571 | -v /host/borg-conf/logs:/var/log \ 572 | layr/borg-mysql-backup compact.sh -l 573 | 574 | ### check.sh 575 | 576 | `check` script is for verifying repo/archive integrity and optionally 577 | attempting to fix any issues. 578 | 579 | usage: check [-h] [-rlF] [-p ARCHIVE_PREFIX] [-a ARCHIVE] [-B BORG_OPTS] 580 | [-L LOCAL_REPO] [-R REMOTE] [-T REMOTE_REPO] 581 | 582 | Verify the consistency of a repo and its archives. 583 | 584 | arguments: 585 | -h show help and exit 586 | -r only check remote borg repo (remote-only) 587 | -l only check local borg repo (local-only) 588 | -F attempt to repair/fix inconsistencies; dangerous, see docs! 589 | -p ARCHIVE_PREFIX check archives with given prefix; same as providing 590 | -B '--glob-archives ARCHIVE_PREFIX*' 591 | -a ARCHIVE archive name to check; -p & -a are mutually exclusive 592 | -B BORG_OPTS additional borg params to pass to borg check command 593 | -L LOCAL_REPO overrides container env var of same name 594 | -R REMOTE overrides container env var of same name 595 | -T REMOTE_REPO overrides container env var of same name 596 | 597 | 598 | #### Usage examples 599 | 600 | ##### Check/verify the local repository 601 | 602 | docker run -it --rm \ 603 | -e BORG_PASSPHRASE=borgrepopassword \ 604 | -v /host/backup:/backup \ 605 | -v /host/borg-conf/.borg/cache:/root/.cache/borg \ 606 | -v /host/borg-conf/.borg/config:/root/.config/borg \ 607 | -v /host/borg-conf/logs:/var/log \ 608 | layr/borg-mysql-backup check.sh -l 609 | 610 | ### notif-test.sh 611 | 612 | `notif-test` script is for testing your configured notifications. 613 | 614 | usage: notif-test.sh [-hpIHsTFAmef] 615 | 616 | Test configured notifications. Running it will fire notification via each of 617 | the configured channels. 618 | 619 | arguments: 620 | -p ARCHIVE_PREFIX 621 | -I HOST_ID 622 | -H HC_ID (id to replace in healthcheck url) 623 | -s NOTIF_SUBJECT 624 | -T MAIL_TO 625 | -F MAIL_FROM 626 | -A SMTP_ACCOUNT 627 | -m MSG 628 | -e ERR_NOTIF 629 | -f marks the error as fatal (ie halting the script) 630 | 631 | #### Usage examples 632 | 633 | ##### Simulate an error message to test all the notifications at once 634 | 635 | docker run -it --rm \ 636 | -e PUSHOVER_USER_KEY='key' \ 637 | -e PUSHOVER_APP_TOKEN='token' \ 638 | -e MAIL_TO='your@mail.com' \ 639 | -e SMTP_HOST='smtp.gmail.com' \ 640 | -e SMTP_USER='your.google.username' \ 641 | -e SMTP_PASS='your-google-app-password-you-created-for-this' \ 642 | -e ERR_NOTIF='mail,pushover' \ 643 | -e HOST_ID='our-hostname' \ 644 | layr/borg-mysql-backup notif-test.sh -p 'my-prefix' [-f] 645 | 646 | 647 | ## See also/recommended 648 | - [restic](https://github.com/restic/restic) 649 | - [duplicacy](https://github.com/gilbertchen/duplicacy) - alternatives to borg. lock-free! 650 | - [docker-db-backup](https://github.com/tiredofit/docker-db-backup) - similar service; supports multiple dbs 651 | - [this blog](https://ifnull.org/articles/borgbackup_rsyncnet/) for borg setup 652 | - [borgmatic](https://github.com/witten/borgmatic) - declarative borg config 653 | - [this dockerised borgmatic](https://hub.docker.com/r/b3vis/borgmatic/) - provides same as this service, and more 654 | - main offsite hostings: [rsync.net](https://www.rsync.net) & [BorgBase](https://www.borgbase.com/) 655 | - [vorta](https://github.com/borgbase/vorta) - mac & linux desktop client; by borgbase 656 | - for backups from k8s: 657 | - [velero](https://github.com/vmware-tanzu/velero) 658 | - [k8up](https://github.com/vshn/k8up) - based on restic 659 | - [stash](https://github.com/stashed/stash) 660 | - [web interface for borg](https://github.com/borgbackup/borgweb/) 661 | 662 | -------------------------------------------------------------------------------- /scripts/backup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # backs up mysql/postgres dump and/or other data to local and/or remote borg repository 4 | 5 | readonly SELF="${0##*/}" 6 | 7 | readonly usage=" 8 | usage: $SELF [-h] [-d MYSQL_DBS] [-g POSTGRES_DBS] [-c CONTAINERS] [-rl] 9 | [-P PRUNE_OPTS] [-B|-Z CREATE_OPTS] [-E EXCLUDE_PATHS] 10 | [-L LOCAL_REPO] [-e ERR_NOTIF] [-A SMTP_ACCOUNT] [-D MYSQL_FAIL_FATAL] 11 | [-G POSTGRES_FAIL_FATAL] [-S SCRIPT_FAIL_FATAL] [-R REMOTE] 12 | [-T REMOTE_REPO] [-C] [-H HC_ID] -p PREFIX [NODES_TO_BACK_UP...] 13 | 14 | Create new archive 15 | 16 | arguments: 17 | -h show help and exit 18 | -d MYSQL_DBS comma-separated mysql database names to back up; use value of 19 | __all__ to back up all dbs on the server 20 | -g POSTGRES_DBS comma-separated postgresql database names to back up; use value of 21 | __all__ to back up all dbs on the server 22 | -c CONTAINERS comma-separated container names to stop for the backup process; 23 | requires mounting the docker socket (-v /var/run/docker.sock:/var/run/docker.sock); 24 | note containers will be stopped in given order; after backup 25 | completion, containers are started in reverse order; only containers 26 | that were stopped by the script will be re-started afterwards 27 | -r only back to remote borg repo (remote-only) 28 | -l only back to local borg repo (local-only) 29 | -P PRUNE_OPTS overrides container env var of same name; only required when 30 | container var is not defined or needs to be overridden; 31 | -1 LOCAL_PRUNE_OPTS prune options for local borg repo; overrides PRUNE_OPTS (& -P); 32 | -2 REMOTE_PRUNE_OPTS prune options for remote borg repo; overrides PRUNE_OPTS (& -P); 33 | -B CREATE_OPTS additional borg params; note it doesn't overwrite the 34 | env var of same name, but extends it; 35 | -Z CREATE_OPTS additional borg params; note it _overrides_ the env 36 | var of same name; 37 | -E EXCLUDE_PATHS comma-separated paths to exclude from backup; 38 | [-E '/p1,/p2'] would be equivalent to [-B '-e /p1 -e /p2'] 39 | -L LOCAL_REPO overrides container env var of same name; 40 | -e ERR_NOTIF overrides container env var of same name; 41 | -A SMTP_ACCOUNT overrides container env var of same name; 42 | -D MYSQL_FAIL_FATAL overrides container env var of same name; 43 | -G POSTGRES_FAIL_FATAL overrides container env var of same name; 44 | -S SCRIPT_FAIL_FATAL overrides container env var of same name; 45 | -R REMOTE overrides container env var of same name; 46 | -T REMOTE_REPO overrides container env var of same name; 47 | -C run compact command against repo after backup/prune; 48 | -H HC_ID the unique/id part of healthcheck url, replacing the '{id}' 49 | placeholder in HC_URL; may also provide new full url to call 50 | instead, overriding the env var HC_URL 51 | -p PREFIX borg archive name prefix. note that the full archive name already 52 | contains HOST_ID env var and timestamp, so omit those. 53 | NODES_TO_BACK_UP... last arguments to $SELF are files&directories to be 54 | included in the backup 55 | " 56 | 57 | # expands the $NODES_TO_BACK_UP with files in $TMP/, if there are any 58 | expand_nodes_to_back_up() { 59 | local i 60 | 61 | is_dir_empty "$TMP" && return 0 62 | 63 | while IFS= read -r -d $'\0' i; do 64 | i="$(basename -- "$i")" # note relative path; we don't want borg archive to contain "$TMP_ROOT" path 65 | contains "$i" "${NODES_TO_BACK_UP[@]}" && continue 66 | NODES_TO_BACK_UP+=("$i") 67 | done < <(find "$TMP" -mindepth 1 -maxdepth 1 -print0) 68 | } 69 | 70 | 71 | # dumps selected mariadb/mysql db(s) to $TMP 72 | dump_mysql() { 73 | local output_filename dbs dbs_log err_code start_timestamp t s 74 | 75 | [[ "${#MYSQL_DB[@]}" -eq 0 ]] && return 0 # no db specified, meaning db dump not required 76 | 77 | if [[ "${MYSQL_DB[*]}" == "$ALL_DBS_MARKER" ]]; then 78 | dbs_log='all databases' 79 | output_filename='all-dbs' 80 | dbs=('--all-databases') 81 | else 82 | dbs_log="databases [${MYSQL_DB[*]}]" 83 | output_filename="$(tr '[:blank:]' '+' <<< "${MYSQL_DB[*]}")" # let the filename reflect which dbs it contains 84 | dbs=('--databases' "${MYSQL_DB[@]}") 85 | fi 86 | 87 | log "=> starting mysql db dump for ${dbs_log}..." 88 | start_timestamp="$(date +%s)" 89 | 90 | # TODO: add following column-stats option back once mariadb-dump from alpine accepts it: 91 | #--column-statistics=0 \ 92 | # TODO: add --routines ? 93 | # TODO: add --add-locks ? 94 | mariadb-dump \ 95 | --add-drop-database \ 96 | --max-allowed-packet=512M \ 97 | --host="${MYSQL_HOST}" \ 98 | --port="${MYSQL_PORT}" \ 99 | --user="${MYSQL_USER}" \ 100 | --password="${MYSQL_PASS}" \ 101 | ${MYSQL_EXTRA_OPTS} \ 102 | "${dbs[@]}" > "$TMP/mysql:${output_filename}.sql" 2> >(tee -a "$LOG" >&2) 103 | 104 | err_code="$?" 105 | if [[ "$err_code" -ne 0 ]]; then 106 | local msg 107 | msg="mysql db dump step for input args [${MYSQL_DB[*]}] failed w/ [$err_code]" 108 | [[ "${MYSQL_FAIL_FATAL:-true}" == true ]] && fail "${msg}; aborting" || err "${msg}; not aborting" 109 | fi 110 | 111 | t="$(( $(date +%s) - start_timestamp ))" 112 | [[ "$err_code" -eq 0 ]] && s='succeeded ' 113 | log "=> mysql db dump ${s}in $(print_time "$t")" 114 | } 115 | 116 | 117 | # dumps selected postgres db(s) to $TMP 118 | # https://www.postgresql.org/docs/15/backup-dump.html 119 | dump_postgres() { 120 | local err_code start_timestamp t d s 121 | 122 | [[ "${#POSTGRES_DB[@]}" -eq 0 ]] && return 0 # no db specified, meaning db dump not required 123 | 124 | start_timestamp="$(date +%s)" 125 | export PGPASSWORD="$POSTGRES_PASS" 126 | 127 | if [[ "${POSTGRES_DB[*]}" == "$ALL_DBS_MARKER" ]]; then 128 | log "=> starting postgres db dump for all databases..." 129 | 130 | pg_dumpall \ 131 | --clean \ 132 | --if-exists \ 133 | --quote-all-identifiers \ 134 | -h "$POSTGRES_HOST" \ 135 | -p "$POSTGRES_PORT" \ 136 | -U "$POSTGRES_USER" \ 137 | ${POSTGRES_EXTRA_OPTS} > "$TMP/postgres:all-dbs.sql" 2> >(tee -a "$LOG" >&2) # !! restore.sh references filename 138 | 139 | err_code="$?" 140 | else 141 | log "=> starting postgres db dump for databases [${POSTGRES_DB[*]}]..." 142 | err_code=0 143 | 144 | for d in "${POSTGRES_DB[@]}"; do 145 | pg_dump \ 146 | --clean \ 147 | --if-exists \ 148 | --quote-all-identifiers \ 149 | --create \ 150 | -h "$POSTGRES_HOST" \ 151 | -p "$POSTGRES_PORT" \ 152 | -U "$POSTGRES_USER" \ 153 | ${POSTGRES_EXTRA_OPTS} \ 154 | -d "$d" > "$TMP/postgres:${d}.sql" 2> >(tee -a "$LOG" >&2) || { err_code=$?; err "pg_dump for db [$d] failed w/ $err_code"; } 155 | done 156 | fi 157 | 158 | if [[ "$err_code" -ne 0 ]]; then 159 | local msg 160 | msg="postgres db dump step for input args [${POSTGRES_DB[*]}] failed w/ [$err_code]" 161 | [[ "${POSTGRES_FAIL_FATAL:-true}" == true ]] && fail "${msg}; aborting" || err "${msg}; not aborting" 162 | fi 163 | 164 | t="$(( $(date +%s) - start_timestamp ))" 165 | [[ "$err_code" -eq 0 ]] && s='succeeded ' 166 | log "=> postgres db dump ${s}in $(print_time "$t")" 167 | } 168 | 169 | 170 | # TODO: should we err() or fail() from here, as they are backgrounded anyway? 171 | _backup_common() { 172 | local l_or_r repo extra_opts start_timestamp err_code opts t s 173 | 174 | l_or_r="$1" 175 | repo="$2" 176 | extra_opts="$3" 177 | 178 | opts="$(join -s ' ' -- "$COMMON_OPTS" "$CREATE_OPTS" "$extra_opts")" 179 | 180 | log "=> starting $l_or_r backup to [$repo]..." 181 | log "=> effective $l_or_r create opts = [$opts]" 182 | start_timestamp="$(date +%s)" 183 | 184 | borg create --stats --show-rc \ 185 | $opts \ 186 | "${repo}::${ARCHIVE_NAME}" \ 187 | "${NODES_TO_BACK_UP[@]}" > >(tee -a "$LOG") 2> >(tee -a "$LOG" >&2) || { err "$l_or_r borg create exited w/ [$?]"; err_code=1; } 188 | 189 | t="$(( $(date +%s) - start_timestamp ))" 190 | [[ -z "$err_code" ]] && s='succeeded ' 191 | log "=> $l_or_r backup ${s}in $(print_time "$t")" 192 | 193 | return "${err_code:-0}" 194 | } 195 | 196 | 197 | # TODO: should we err() or fail() from here, as they are backgrounded anyway? 198 | _prune_common() { 199 | local l_or_r repo prune_opts start_timestamp err_code opts t s 200 | 201 | l_or_r="$1" 202 | repo="$2" 203 | prune_opts="$3" # local/remote specific prune opts, optional 204 | 205 | opts="$(join -s ' ' -- "$COMMON_OPTS" "${prune_opts:-$PRUNE_OPTS}")" 206 | 207 | log "=> starting $l_or_r prune from [$repo]..." 208 | log "=> effective $l_or_r prune opts = [$opts]" 209 | start_timestamp="$(date +%s)" 210 | 211 | borg prune --show-rc \ 212 | $opts \ 213 | --glob-archives "${PREFIX_WITH_HOSTNAME}*" \ 214 | "$repo" > >(tee -a "$LOG") 2> >(tee -a "$LOG" >&2) || { err "$l_or_r borg prune exited w/ [$?]"; err_code=1; } 215 | 216 | t="$(( $(date +%s) - start_timestamp ))" 217 | [[ -z "$err_code" ]] && s='succeeded ' 218 | log "=> $l_or_r prune ${s}in $(print_time "$t")" 219 | 220 | return "${err_code:-0}" 221 | } 222 | 223 | 224 | backup_local() { 225 | _backup_common local "${LOCAL_REPO}" "$LOCAL_CREATE_OPTS" 226 | } 227 | 228 | 229 | backup_remote() { 230 | _backup_common remote "${REMOTE}" "$REMOTE_CREATE_OPTS" 231 | } 232 | 233 | 234 | prune_local() { 235 | _prune_common local "${LOCAL_REPO}" "$LOCAL_PRUNE_OPTS" 236 | } 237 | 238 | 239 | prune_remote() { 240 | _prune_common remote "${REMOTE}" "$REMOTE_PRUNE_OPTS" 241 | } 242 | 243 | 244 | # backup selected data 245 | # note the borg processes are executed in a sub-shell, so local & remote backup could be 246 | # run in parallel 247 | # 248 | # TODO: should we skip prune if create exits w/ code >=2? 249 | do_backup() { 250 | local started_pids start_timestamp i err_ t 251 | 252 | declare -a started_pids=() 253 | 254 | log "=> Backup started" 255 | log "=> ARCHIVE_NAME = [$ARCHIVE_NAME]" 256 | 257 | start_timestamp="$(date +%s)" 258 | 259 | run_scripts before-mysql-dump 260 | dump_mysql 261 | run_scripts after-mysql-dump 262 | 263 | run_scripts before-postgres-dump 264 | dump_postgres 265 | run_scripts after-postgres-dump 266 | 267 | expand_nodes_to_back_up # adds dump sql (and any possible custom script additions) to NODES_TO_BACK_UP 268 | 269 | run_scripts before-backup 270 | 271 | expand_nodes_to_back_up # once again, in case any of the custom scripts added files 272 | [[ "${#NODES_TO_BACK_UP[@]}" -eq 0 ]] && fail "no items selected for backup" 273 | 274 | pushd -- "$TMP" &> /dev/null || fail "unable to pushd into [$TMP]" # cd there because files in $TMP are added without full path (to avoid "$TMP_ROOT" prefix in borg repo) 275 | 276 | # note! log files/types out _after_ pushd to $TMP, otherwise some files would not resolve 277 | log "following ${#NODES_TO_BACK_UP[@]} file(s) will be backed up:" 278 | for i in "${NODES_TO_BACK_UP[@]}"; do 279 | log " - $i (type: $(file_type "$i"))" 280 | done 281 | 282 | if [[ "$REMOTE_ONLY" -ne 1 ]]; then 283 | backup_local & 284 | started_pids+=("$!") 285 | fi 286 | 287 | if [[ "$LOCAL_ONLY" -ne 1 ]]; then 288 | backup_remote & 289 | started_pids+=("$!") 290 | fi 291 | 292 | for i in "${started_pids[@]}"; do 293 | wait "$i" || err_=TRUE 294 | done 295 | 296 | popd &> /dev/null 297 | 298 | run_scripts after-backup 299 | 300 | # backup is done, we can go ahead and start the containers while pruning: 301 | # TODO: should start_containers() be called when we errored? 302 | ( 303 | run_scripts before-start-containers 304 | start_containers "${CONTAINERS_TO_START[@]}" 305 | run_scripts after-start-containers 306 | ) & 307 | CONTAINERS_TO_START=() # empty so no secondary start attempts would be made after 308 | 309 | started_pids=() # reset 310 | 311 | run_scripts before-prune 312 | 313 | if [[ "$REMOTE_ONLY" -ne 1 ]]; then 314 | prune_local & 315 | started_pids+=("$!") 316 | fi 317 | 318 | if [[ "$LOCAL_ONLY" -ne 1 ]]; then 319 | prune_remote & 320 | started_pids+=("$!") 321 | fi 322 | 323 | for i in "${started_pids[@]}"; do 324 | wait "$i" || err_=TRUE 325 | done 326 | 327 | run_scripts after-prune 328 | 329 | if [[ -n "$COMPACT" ]]; then 330 | run_scripts before-compact 331 | compact_repos || err_=TRUE 332 | run_scripts after-compact 333 | fi 334 | 335 | t="$(( $(date +%s) - start_timestamp ))" 336 | log "=> Backup+prune${COMPACT:++compact} finished, duration $(print_time "$t")${err_:+; at least one step failed or produced warning}" 337 | 338 | return 0 339 | } 340 | 341 | 342 | validate_config() { 343 | local i vars 344 | 345 | validate_config_common 346 | 347 | declare -a vars=( 348 | ARCHIVE_PREFIX 349 | BORG_PASSPHRASE 350 | HOST_ID 351 | ) 352 | [[ -n "${MYSQL_DB[*]}" ]] && vars+=( 353 | MYSQL_HOST 354 | MYSQL_PORT 355 | MYSQL_USER 356 | MYSQL_PASS 357 | ) 358 | [[ -n "${POSTGRES_DB[*]}" ]] && vars+=( 359 | POSTGRES_HOST 360 | POSTGRES_PORT 361 | POSTGRES_USER 362 | POSTGRES_PASS 363 | ) 364 | [[ "$LOCAL_ONLY" -ne 1 ]] && vars+=(REMOTE REMOTE_REPO) 365 | [[ "$REMOTE_ONLY" -ne 1 ]] && vars+=(LOCAL_REPO) 366 | 367 | # validate prune options: 368 | if [[ "$REMOTE_ONLY" -ne 1 && "$LOCAL_ONLY" -ne 1 ]]; then 369 | if [[ -z "$REMOTE_PRUNE_OPTS" && -z "$LOCAL_PRUNE_OPTS" ]]; then 370 | vars+=(PRUNE_OPTS) 371 | elif [[ -z "$REMOTE_PRUNE_OPTS" || -z "$LOCAL_PRUNE_OPTS" ]] && [[ -z "$PRUNE_OPTS" ]]; then 372 | fail "prune options for remote and/or local repo(s) undefined" 373 | fi 374 | elif [[ "$REMOTE_ONLY" -eq 1 && -z "$REMOTE_PRUNE_OPTS" ]] || [[ "$LOCAL_ONLY" -eq 1 && -z "$LOCAL_PRUNE_OPTS" ]]; then 375 | vars+=(PRUNE_OPTS) 376 | fi 377 | 378 | vars_defined "${vars[@]}" 379 | 380 | if [[ "${#NODES_TO_BACK_UP[@]}" -gt 0 ]]; then 381 | for i in "${NODES_TO_BACK_UP[@]}"; do 382 | [[ -e "$i" ]] || err "node [$i] to back up does not exist; missing mount?" 383 | done 384 | elif [[ "${#MYSQL_DB[@]}" -eq 0 || -z "${MYSQL_DB[*]}" ]] && [[ "${#POSTGRES_DB[@]}" -eq 0 || -z "${POSTGRES_DB[*]}" ]]; then 385 | fail "no databases nor nodes selected for backup - nothing to do!" 386 | fi 387 | 388 | [[ "$REMOTE_OR_LOCAL_OPT_COUNTER" -gt 1 ]] && fail "-r & -l options are exclusive" 389 | [[ "$BORG_OTPS_COUNTER" -gt 1 ]] && fail "-B & -Z options are exclusive" 390 | [[ "$REMOTE_ONLY" -ne 1 ]] && [[ ! -d "$LOCAL_REPO" || ! -w "$LOCAL_REPO" ]] && fail "[$LOCAL_REPO] does not exist or is not writable; missing mount?" 391 | 392 | if [[ "$LOCAL_ONLY" -ne 1 && "$-" != *i* ]]; then 393 | [[ -f "$SSH_KEY" && -s "$SSH_KEY" ]] || fail "[$SSH_KEY] is not a file; is /config mounted?" 394 | fi 395 | } 396 | 397 | 398 | create_dirs() { 399 | mkdir -p -- "$TMP" || fail "dir [$TMP] creation failed w/ [$?]" 400 | } 401 | 402 | 403 | cleanup() { 404 | [[ -d "$TMP" ]] && rm -rf -- "$TMP" 405 | [[ -d "$TMP_ROOT" ]] && is_dir_empty "$TMP_ROOT" && rm -rf -- "$TMP_ROOT" 406 | 407 | if [[ "${#CONTAINERS_TO_START[@]}" -gt 0 ]]; then 408 | run_scripts before-start-containers 409 | start_containers "${CONTAINERS_TO_START[@]}" # do not background here 410 | run_scripts after-start-containers 411 | fi 412 | 413 | # TODO: shouldn't we ping healthcheck the very first thing in cleanup()? ie it should fire regardles of the outcome of other calls in here 414 | ping_healthcheck 415 | log "==> backup script end" 416 | } 417 | 418 | 419 | # ================ 420 | # Entry 421 | # ================ 422 | source /scripts_common.sh || { echo -e " ERROR: failed to import /scripts_common.sh" >&2; exit 1; } 423 | REMOTE_OR_LOCAL_OPT_COUNTER=0 424 | BORG_OTPS_COUNTER=0 425 | BORG_EXCLUDE_PATHS=() 426 | 427 | unset MYSQL_DB POSTGRES_DB ARCHIVE_PREFIX CONTAINERS COMPACT HC_ID # just in case 428 | 429 | while getopts 'd:g:p:c:rlP:1:2:B:Z:E:L:e:A:D:G:S:R:T:H:Ch' opt; do 430 | case "$opt" in 431 | d) IFS="$SEPARATOR" read -ra MYSQL_DB <<< "$OPTARG" 432 | ;; 433 | g) IFS="$SEPARATOR" read -ra POSTGRES_DB <<< "$OPTARG" 434 | ;; 435 | p) readonly ARCHIVE_PREFIX="$OPTARG" # be careful w/ var rename! eg run_scripts() depends on many var names 436 | JOB_ID="${OPTARG}-$$" 437 | ;; 438 | c) IFS="$SEPARATOR" read -ra CONTAINERS <<< "$OPTARG" 439 | ;; 440 | r) REMOTE_ONLY=1 441 | let REMOTE_OR_LOCAL_OPT_COUNTER+=1 442 | ;; 443 | l) LOCAL_ONLY=1 444 | let REMOTE_OR_LOCAL_OPT_COUNTER+=1 445 | ;; 446 | P) PRUNE_OPTS="$OPTARG" # overrides env var of same name 447 | ;; 448 | 1) LOCAL_PRUNE_OPTS="$OPTARG" # overrides env var of same name 449 | ;; 450 | 2) REMOTE_PRUNE_OPTS="$OPTARG" # overrides env var of same name 451 | ;; 452 | B) CREATE_OPTS+=" $OPTARG" # _extends_ env var of same name 453 | let BORG_OTPS_COUNTER+=1 454 | ;; 455 | Z) CREATE_OPTS="$OPTARG" # overrides env var of same name 456 | let BORG_OTPS_COUNTER+=1 457 | ;; 458 | E) IFS="$SEPARATOR" read -ra BORG_EXCLUDE_PATHS <<< "$OPTARG" 459 | ;; 460 | L) LOCAL_REPO="$OPTARG" # overrides env var of same name 461 | ;; 462 | e) ERR_NOTIF="$OPTARG" # overrides env var of same name 463 | ;; 464 | A) SMTP_ACCOUNT="$OPTARG" 465 | ;; 466 | D) MYSQL_FAIL_FATAL="$OPTARG" 467 | ;; 468 | G) POSTGRES_FAIL_FATAL="$OPTARG" 469 | ;; 470 | S) SCRIPT_FAIL_FATAL="$OPTARG" 471 | ;; 472 | R) REMOTE="$OPTARG" # overrides env var of same name 473 | ;; 474 | T) REMOTE_REPO="$OPTARG" # overrides env var of same name 475 | ;; 476 | C) COMPACT=TRUE 477 | ;; 478 | H) HC_ID="$OPTARG" 479 | ;; 480 | h) echo -e "$usage" 481 | exit 0 482 | ;; 483 | *) fail "$SELF called with unsupported flag(s)" 484 | ;; 485 | esac 486 | done 487 | shift "$((OPTIND-1))" 488 | 489 | trap -- 'cleanup; exit' EXIT HUP INT QUIT PIPE TERM 490 | 491 | NODES_TO_BACK_UP=("$@") 492 | JOB_SCRIPT_ROOT="$SCRIPT_ROOT/jobs/$ARCHIVE_PREFIX" 493 | 494 | readonly TMP_ROOT="/tmp/${SELF}.tmp" 495 | readonly TMP="$TMP_ROOT/${ARCHIVE_PREFIX}-$RANDOM" 496 | 497 | [[ -f "$ENV_ROOT/${ARCHIVE_PREFIX}.conf" ]] && source "$ENV_ROOT/${ARCHIVE_PREFIX}.conf" # load job-specific config if avail 498 | 499 | readonly PREFIX_WITH_HOSTNAME="${ARCHIVE_PREFIX}-${HOST_ID}-" # used for pruning 500 | readonly ARCHIVE_NAME="$PREFIX_WITH_HOSTNAME"'{now:%Y-%m-%d-%H%M%S}' 501 | 502 | validate_config 503 | 504 | # make sure these are processed _after_ sourcing job-specific config: 505 | if [[ "${#BORG_EXCLUDE_PATHS[@]}" -gt 0 ]]; then 506 | for i in "${BORG_EXCLUDE_PATHS[@]}"; do 507 | if [[ -n "$NOTIF_MISSING_EXCL_PTH" && ! -e "$i" ]]; then 508 | err "excluded path [$i] does not exist; not aborting" 509 | fi 510 | BORG_EXCLUDE_OPTS+=" --exclude $i" 511 | done 512 | unset BORG_EXCLUDE_PATHS i 513 | fi 514 | 515 | [[ -n "$BORG_EXCLUDE_OPTS" ]] && CREATE_OPTS+=" $BORG_EXCLUDE_OPTS" 516 | unset BORG_EXCLUDE_OPTS 517 | 518 | 519 | process_remote # note this overwrites global REMOTE var 520 | create_dirs 521 | 522 | run_scripts before 523 | 524 | run_scripts before-stop-containers 525 | stop_containers 526 | run_scripts after-stop-containers 527 | 528 | do_backup 529 | 530 | run_scripts after 531 | 532 | exit 0 533 | 534 | -------------------------------------------------------------------------------- /scripts/check.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # check local and/or remote borg repositories 4 | 5 | readonly SELF="${0##*/}" 6 | JOB_ID="check-$$" 7 | 8 | readonly usage=" 9 | usage: $SELF [-h] [-rlF] [-p ARCHIVE_PREFIX] [-a ARCHIVE] [-B BORG_OPTS] 10 | [-L LOCAL_REPO] [-R REMOTE] [-T REMOTE_REPO] 11 | 12 | Verify the consistency of a repo and its archives. 13 | 14 | arguments: 15 | -h show help and exit 16 | -r only check remote borg repo (remote-only) 17 | -l only check local borg repo (local-only) 18 | -F attempt to repair/fix inconsistencies; dangerous, see docs! 19 | -p ARCHIVE_PREFIX check archives with given prefix; same as providing 20 | -B '--glob-archives ARCHIVE_PREFIX*' 21 | -a ARCHIVE archive name to check; -p & -a are mutually exclusive 22 | -B BORG_OPTS additional borg params to pass to borg check command 23 | -L LOCAL_REPO overrides container env var of same name 24 | -R REMOTE overrides container env var of same name 25 | -T REMOTE_REPO overrides container env var of same name 26 | " 27 | 28 | 29 | _check_common() { 30 | local l_or_r repo start_timestamp err_code t s 31 | 32 | l_or_r="$1" 33 | repo="$2" 34 | 35 | log "=> starting check operation on $l_or_r repo [$repo]..." 36 | start_timestamp="$(date +%s)" 37 | 38 | borg check --verify-data $REPAIR --show-rc \ 39 | $COMMON_OPTS \ 40 | $BORG_OPTS \ 41 | "${repo}${ARCHIVE:+::$ARCHIVE}" > >(tee -a "$LOG") 2> >(tee -a "$LOG" >&2) || { err "check operation on $l_or_r repo [$repo] failed w/ [$?]"; err_code=1; } 42 | 43 | t="$(( $(date +%s) - start_timestamp ))" 44 | [[ -z "$err_code" ]] && s='succeeded ' 45 | log "=> $l_or_r repo check ${s}in $(print_time "$t")" 46 | 47 | return "${err_code:-0}" 48 | } 49 | 50 | 51 | check() { 52 | if [[ "$REMOTE_ONLY" -ne 1 ]]; then 53 | _check_common local "$LOCAL_REPO" # do not background; even remote check takes resources locally 54 | fi 55 | 56 | if [[ "$LOCAL_ONLY" -ne 1 ]]; then 57 | _check_common remote "$REMOTE" # do not background; even remote check takes resources locally 58 | fi 59 | } 60 | 61 | 62 | validate_config() { 63 | local vars 64 | 65 | declare -a vars 66 | 67 | [[ "$LOCAL_ONLY" -ne 1 ]] && vars+=(REMOTE REMOTE_REPO) 68 | [[ "$REMOTE_ONLY" -ne 1 ]] && vars+=(LOCAL_REPO) 69 | 70 | vars_defined "${vars[@]}" 71 | 72 | [[ "$REMOTE_OR_LOCAL_OPT_COUNTER" -gt 1 ]] && fail "-r & -l options are exclusive" 73 | [[ "$REMOTE_ONLY" -ne 1 ]] && [[ ! -d "$LOCAL_REPO" || ! -w "$LOCAL_REPO" ]] && fail "[$LOCAL_REPO] does not exist or is not writable; missing mount?" 74 | [[ "$ARCHIVE_OR_PREFIX_OPT_COUNTER" -gt 1 ]] && fail "defining both archive prefix & full archive name are mutually exclusive" 75 | } 76 | 77 | # ================ 78 | # Entry 79 | # ================ 80 | NO_NOTIF=true # do not notify errors 81 | source /scripts_common.sh || { echo -e " ERROR: failed to import /scripts_common.sh" >&2; exit 1; } 82 | REMOTE_OR_LOCAL_OPT_COUNTER=0 83 | ARCHIVE_OR_PREFIX_OPT_COUNTER=0 84 | 85 | unset ARCHIVE ARCHIVE_PREFIX BORG_OPTS REPAIR # just in case 86 | 87 | while getopts 'rlFp:a:B:L:R:T:h' opt; do 88 | case "$opt" in 89 | r) REMOTE_ONLY=1 90 | let REMOTE_OR_LOCAL_OPT_COUNTER+=1 91 | ;; 92 | l) LOCAL_ONLY=1 93 | let REMOTE_OR_LOCAL_OPT_COUNTER+=1 94 | ;; 95 | F) REPAIR='--repair' 96 | ;; 97 | p) ARCHIVE_PREFIX="$OPTARG" 98 | let ARCHIVE_OR_PREFIX_OPT_COUNTER+=1 99 | ;; 100 | a) ARCHIVE="$OPTARG" 101 | let ARCHIVE_OR_PREFIX_OPT_COUNTER+=1 102 | ;; 103 | B) BORG_OPTS="$OPTARG" 104 | ;; 105 | L) LOCAL_REPO="$OPTARG" # overrides env var of same name 106 | ;; 107 | R) REMOTE="$OPTARG" # overrides env var of same name 108 | ;; 109 | T) REMOTE_REPO="$OPTARG" # overrides env var of same name 110 | ;; 111 | h) echo -e "$usage" 112 | exit 0 113 | ;; 114 | *) fail "$SELF called with unsupported flag(s)" 115 | ;; 116 | esac 117 | done 118 | 119 | validate_config 120 | process_remote # note this overwrites global REMOTE var 121 | 122 | [[ -n "$ARCHIVE_PREFIX" ]] && BORG_OPTS+=" --glob-archives ${ARCHIVE_PREFIX}*" 123 | check 124 | 125 | exit 0 126 | 127 | -------------------------------------------------------------------------------- /scripts/compact.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # compact local and/or remote borg repository 4 | 5 | readonly SELF="${0##*/}" 6 | JOB_ID="compact-$$" 7 | 8 | readonly usage=" 9 | usage: $SELF [-h] [-rl] [-B BORG_OPTS] [-L LOCAL_REPO] 10 | [-R REMOTE] [-T REMOTE_REPO] 11 | 12 | 13 | Compact borg repository 14 | 15 | arguments: 16 | -h show help and exit 17 | -r compact only remote borg repo (remote-only) 18 | -l compact only local borg repo (local-only) 19 | -B BORG_OPTS additional borg params to pass to borg compact command 20 | -L LOCAL_REPO overrides container env var of same name 21 | -R REMOTE overrides container env var of same name 22 | -T REMOTE_REPO overrides container env var of same name 23 | " 24 | 25 | 26 | validate_config() { 27 | local vars 28 | 29 | declare -a vars 30 | 31 | [[ "$LOCAL_ONLY" -ne 1 ]] && vars+=(REMOTE REMOTE_REPO) 32 | [[ "$REMOTE_ONLY" -ne 1 ]] && vars+=(LOCAL_REPO) 33 | 34 | vars_defined "${vars[@]}" 35 | 36 | [[ "$REMOTE_OR_LOCAL_OPT_COUNTER" -gt 1 ]] && fail "-r & -l options are exclusive" 37 | [[ "$REMOTE_ONLY" -ne 1 ]] && [[ ! -d "$LOCAL_REPO" || ! -w "$LOCAL_REPO" ]] && fail "[$LOCAL_REPO] does not exist or is not writable; missing mount?" 38 | } 39 | 40 | # ================ 41 | # Entry 42 | # ================ 43 | NO_NOTIF=true # do not notify errors 44 | source /scripts_common.sh || { echo -e " ERROR: failed to import /scripts_common.sh" >&2; exit 1; } 45 | REMOTE_OR_LOCAL_OPT_COUNTER=0 46 | 47 | unset BORG_OPTS # just in case 48 | 49 | while getopts 'rlB:L:R:T:h' opt; do 50 | case "$opt" in 51 | r) REMOTE_ONLY=1 52 | let REMOTE_OR_LOCAL_OPT_COUNTER+=1 53 | ;; 54 | l) LOCAL_ONLY=1 55 | let REMOTE_OR_LOCAL_OPT_COUNTER+=1 56 | ;; 57 | B) BORG_OPTS="$OPTARG" 58 | ;; 59 | L) LOCAL_REPO="$OPTARG" # overrides env var of same name 60 | ;; 61 | R) REMOTE="$OPTARG" # overrides env var of same name 62 | ;; 63 | T) REMOTE_REPO="$OPTARG" # overrides env var of same name 64 | ;; 65 | h) echo -e "$usage" 66 | exit 0 67 | ;; 68 | *) fail "$SELF called with unsupported flag(s)" 69 | ;; 70 | esac 71 | done 72 | 73 | validate_config 74 | process_remote # note this overwrites global REMOTE var 75 | 76 | compact_repos 77 | 78 | exit 0 79 | 80 | -------------------------------------------------------------------------------- /scripts/delete.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # delete archive or whole repository 4 | 5 | readonly SELF="${0##*/}" 6 | JOB_ID="delete-$$" 7 | 8 | readonly usage=" 9 | usage: $SELF [-h] [-rl] [-p ARCHIVE_PREFIX] [-a ARCHIVE] [-B BORG_OPTS] 10 | [-L LOCAL_REPO] [-R REMOTE] [-T REMOTE_REPO] 11 | 12 | Delete whole borg repository or archives in it 13 | 14 | arguments: 15 | -h show help and exit 16 | -r only delete from remote borg repo (remote-only) 17 | -l only delete from local borg repo (local-only) 18 | -p ARCHIVE_PREFIX delete archives with given prefix; same as providing 19 | -B '--glob-archives ARCHIVE_PREFIX*' 20 | -a ARCHIVE archive name to delete; -p & -a are mutually exclusive 21 | -B BORG_OPTS additional borg params to pass to borg delete command 22 | -L LOCAL_REPO overrides container env var of same name 23 | -R REMOTE overrides container env var of same name 24 | -T REMOTE_REPO overrides container env var of same name 25 | " 26 | 27 | 28 | _del_common() { 29 | local l_or_r repo start_timestamp err_code t s 30 | 31 | l_or_r="$1" 32 | repo="$2" 33 | 34 | log "=> starting delete operation on $l_or_r repo [$repo]..." 35 | start_timestamp="$(date +%s)" 36 | 37 | borg delete --stats --show-rc \ 38 | $COMMON_OPTS \ 39 | $BORG_OPTS \ 40 | "${repo}${ARCHIVE:+::$ARCHIVE}" > >(tee -a "$LOG") 2> >(tee -a "$LOG" >&2) || { err "delete operation on $l_or_r repo [$repo] failed w/ [$?]"; err_code=1; } 41 | 42 | t="$(( $(date +%s) - start_timestamp ))" 43 | [[ -z "$err_code" ]] && s='succeeded ' 44 | log "=> $l_or_r repo delete ${s}in $(print_time "$t")" 45 | 46 | return "${err_code:-0}" 47 | } 48 | 49 | 50 | delete() { 51 | local started_pids start_timestamp i err_ t 52 | 53 | declare -a started_pids=() 54 | start_timestamp="$(date +%s)" 55 | 56 | if [[ "$REMOTE_ONLY" -ne 1 ]]; then 57 | _del_common local "$LOCAL_REPO" & 58 | started_pids+=("$!") 59 | fi 60 | 61 | if [[ "$LOCAL_ONLY" -ne 1 ]]; then 62 | _del_common remote "$REMOTE" & 63 | started_pids+=("$!") 64 | fi 65 | 66 | for i in "${started_pids[@]}"; do 67 | wait "$i" || err_=TRUE 68 | done 69 | 70 | t="$(( $(date +%s) - start_timestamp ))" 71 | log "=> Delete finished, duration $(print_time "$t")${err_:+; at least one repo produced a warning}" 72 | 73 | return 0 74 | } 75 | 76 | 77 | validate_config() { 78 | local vars 79 | 80 | declare -a vars 81 | 82 | [[ "$LOCAL_ONLY" -ne 1 ]] && vars+=(REMOTE REMOTE_REPO) 83 | [[ "$REMOTE_ONLY" -ne 1 ]] && vars+=(LOCAL_REPO) 84 | 85 | vars_defined "${vars[@]}" 86 | 87 | [[ "$REMOTE_OR_LOCAL_OPT_COUNTER" -gt 1 ]] && fail "-r & -l options are exclusive" 88 | [[ "$REMOTE_ONLY" -ne 1 ]] && [[ ! -d "$LOCAL_REPO" || ! -w "$LOCAL_REPO" ]] && fail "[$LOCAL_REPO] does not exist or is not writable; missing mount?" 89 | [[ "$ARCHIVE_OR_PREFIX_OPT_COUNTER" -gt 1 ]] && fail "defining both archive prefix & full archive name are mutually exclusive" 90 | } 91 | 92 | # ================ 93 | # Entry 94 | # ================ 95 | NO_NOTIF=true # do not notify errors 96 | source /scripts_common.sh || { echo -e " ERROR: failed to import /scripts_common.sh" >&2; exit 1; } 97 | REMOTE_OR_LOCAL_OPT_COUNTER=0 98 | ARCHIVE_OR_PREFIX_OPT_COUNTER=0 99 | 100 | unset ARCHIVE ARCHIVE_PREFIX BORG_OPTS # just in case 101 | 102 | while getopts 'rlp:a:B:L:R:T:h' opt; do 103 | case "$opt" in 104 | r) REMOTE_ONLY=1 105 | let REMOTE_OR_LOCAL_OPT_COUNTER+=1 106 | ;; 107 | l) LOCAL_ONLY=1 108 | let REMOTE_OR_LOCAL_OPT_COUNTER+=1 109 | ;; 110 | p) ARCHIVE_PREFIX="$OPTARG" 111 | let ARCHIVE_OR_PREFIX_OPT_COUNTER+=1 112 | ;; 113 | a) ARCHIVE="$OPTARG" 114 | let ARCHIVE_OR_PREFIX_OPT_COUNTER+=1 115 | ;; 116 | B) BORG_OPTS="$OPTARG" 117 | ;; 118 | L) LOCAL_REPO="$OPTARG" # overrides env var of same name 119 | ;; 120 | R) REMOTE="$OPTARG" # overrides env var of same name 121 | ;; 122 | T) REMOTE_REPO="$OPTARG" # overrides env var of same name 123 | ;; 124 | h) echo -e "$usage" 125 | exit 0 126 | ;; 127 | *) fail "$SELF called with unsupported flag(s)" 128 | ;; 129 | esac 130 | done 131 | 132 | validate_config 133 | process_remote # note this overwrites global REMOTE var 134 | 135 | [[ -n "$ARCHIVE_PREFIX" ]] && BORG_OPTS+=" --glob-archives ${ARCHIVE_PREFIX}*" 136 | delete 137 | 138 | exit 0 139 | 140 | -------------------------------------------------------------------------------- /scripts/entry.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # alpine-linux entry 3 | 4 | /setup.sh || exit 1 5 | 6 | 7 | if [ $# -ne 0 ]; then 8 | [ "$1" = '--' ] && shift 9 | [ $# -eq 0 ] && exit 1 10 | exec "$@" 11 | else 12 | # start cron 13 | /usr/sbin/crond -f -l 8 -L /dev/stdout -c /var/spool/cron/crontabs 14 | fi 15 | -------------------------------------------------------------------------------- /scripts/list.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # lists contents of local or remote archive 4 | 5 | readonly SELF="${0##*/}" 6 | JOB_ID="list-$$" 7 | 8 | readonly usage=" 9 | usage: $SELF [-h] [-rl] [-p ARCHIVE_PREFIX] [-B BORG_OPTS] [-L LOCAL_REPO] 10 | [-R REMOTE] [-T REMOTE_REPO] [-a ARCHIVE_NAME] 11 | 12 | 13 | List archives in a borg repository or contents of an archive 14 | 15 | arguments: 16 | -h show help and exit 17 | -r list remote borg repo 18 | -l list local borg repo 19 | -p ARCHIVE_PREFIX list archives with given prefix; same as providing 20 | [-B '--glob-archives ARCHIVE_PREFIX*'] 21 | -B BORG_OPTS additional borg params to pass to borg list command 22 | -L LOCAL_REPO overrides container env var of same name 23 | -R REMOTE overrides container env var of same name 24 | -T REMOTE_REPO overrides container env var of same name 25 | -a ARCHIVE_NAME full name of the borg archive whose contents to list 26 | " 27 | 28 | 29 | # TODO: do not fail() if err code <=1? 30 | _list_common() { 31 | local l_or_r repo 32 | 33 | l_or_r="$1" 34 | repo="$2" 35 | 36 | borg list --show-rc \ 37 | $COMMON_OPTS \ 38 | $BORG_OPTS \ 39 | "${repo}${ARCHIVE_NAME:+::$ARCHIVE_NAME}" > >(tee -a "$LOG") 2> >(tee -a "$LOG" >&2) || fail "listing $l_or_r repo [$repo] failed w/ [$?]" 40 | } 41 | 42 | 43 | list_repos() { 44 | 45 | if [[ "$LOC" -eq 1 ]]; then 46 | _list_common local "$LOCAL_REPO" 47 | elif [[ "$REM" -eq 1 ]]; then 48 | _list_common remote "$REMOTE" 49 | else 50 | fail "need to select local or remote repo" 51 | fi 52 | } 53 | 54 | 55 | validate_config() { 56 | local vars 57 | 58 | declare -a vars 59 | 60 | [[ "$REM" -eq 1 ]] && vars+=(REMOTE REMOTE_REPO) 61 | [[ "$LOC" -eq 1 ]] && vars+=(LOCAL_REPO) 62 | 63 | vars_defined "${vars[@]}" 64 | 65 | [[ "$REMOTE_OR_LOCAL_OPT_COUNTER" -ne 1 ]] && fail "need to select whether to list local or remote repo" 66 | [[ "$LOC" -eq 1 ]] && [[ ! -d "$LOCAL_REPO" || ! -w "$LOCAL_REPO" ]] && fail "[$LOCAL_REPO] does not exist or is not writable; missing mount?" 67 | [[ -n "$ARCHIVE_PREFIX" && -n "$ARCHIVE_NAME" ]] && fail "ARCHIVE_NAME & ARCHIVE_PREFIX options are mutually exclusive" 68 | } 69 | 70 | # ================ 71 | # Entry 72 | # ================ 73 | NO_NOTIF=true # do not notify errors 74 | source /scripts_common.sh || { echo -e " ERROR: failed to import /scripts_common.sh" >&2; exit 1; } 75 | REMOTE_OR_LOCAL_OPT_COUNTER=0 76 | 77 | unset ARCHIVE_PREFIX BORG_OPTS ARCHIVE_NAME # just in case 78 | 79 | while getopts 'rlp:B:L:R:T:a:h' opt; do 80 | case "$opt" in 81 | r) REM=1 82 | let REMOTE_OR_LOCAL_OPT_COUNTER+=1 83 | ;; 84 | l) LOC=1 85 | let REMOTE_OR_LOCAL_OPT_COUNTER+=1 86 | ;; 87 | p) ARCHIVE_PREFIX="$OPTARG" 88 | ;; 89 | B) BORG_OPTS="$OPTARG" 90 | ;; 91 | L) LOCAL_REPO="$OPTARG" # overrides env var of same name 92 | ;; 93 | R) REMOTE="$OPTARG" # overrides env var of same name 94 | ;; 95 | T) REMOTE_REPO="$OPTARG" # overrides env var of same name 96 | ;; 97 | a) ARCHIVE_NAME="$OPTARG" 98 | ;; 99 | h) echo -e "$usage" 100 | exit 0 101 | ;; 102 | *) fail "$SELF called with unsupported flag(s)" 103 | ;; 104 | esac 105 | done 106 | 107 | validate_config 108 | process_remote # note this overwrites global REMOTE var 109 | 110 | [[ -n "$ARCHIVE_PREFIX" ]] && BORG_OPTS+=" --glob-archives ${ARCHIVE_PREFIX}*" 111 | list_repos 112 | 113 | exit 0 114 | 115 | -------------------------------------------------------------------------------- /scripts/notif-test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # tests configured notification system(s) 4 | 5 | readonly SELF="${0##*/}" 6 | JOB_ID="notif-test-$$" # just for logging; will be overwritten before notification(s) are triggered 7 | 8 | readonly usage=" 9 | usage: $SELF [-hpIHsTFAmef] 10 | 11 | Test configured notifications. Running it will fire notification via each of 12 | the configured channels. 13 | 14 | arguments: 15 | -p ARCHIVE_PREFIX 16 | -I HOST_ID 17 | -H HC_ID (id to replace in healthcheck url) 18 | -s NOTIF_SUBJECT 19 | -T MAIL_TO 20 | -F MAIL_FROM 21 | -A SMTP_ACCOUNT 22 | -m MSG 23 | -e ERR_NOTIF 24 | -f marks the error as fatal (ie halting the script) 25 | " 26 | 27 | # ================ 28 | # Entry 29 | # ================ 30 | NO_NOTIF=true 31 | source /scripts_common.sh || { echo -e " ERROR: failed to import /scripts_common.sh" >&2; exit 1; } 32 | export LOG=/dev/null # override LOG 33 | 34 | while getopts 'p:I:H:s:T:F:A:m:e:fh' opt; do 35 | case "$opt" in 36 | p) ARCHIVE_PREFIX="$OPTARG" 37 | ;; 38 | I) HOST_ID="$OPTARG" # overrides env var of same name 39 | ;; 40 | H) HC_ID="$OPTARG" 41 | ;; 42 | s) NOTIF_SUBJECT="$OPTARG" # overrides env var of same name 43 | ;; 44 | T) MAIL_TO="$OPTARG" # overrides env var of same name 45 | ;; 46 | F) MAIL_FROM="$OPTARG" # overrides env var of same name 47 | ;; 48 | A) SMTP_ACCOUNT="$OPTARG" # overrides env var of same name 49 | ;; 50 | m) MSG="$OPTARG" 51 | ;; 52 | e) ERR_NOTIF="$OPTARG" # overrides env var of same name 53 | ;; 54 | f) FATAL=1 55 | ;; 56 | h) echo -e "$usage" 57 | exit 0 58 | ;; 59 | *) fail "$SELF called with unsupported flag(s)" 60 | ;; 61 | esac 62 | done 63 | 64 | [[ -z "$ERR_NOTIF" ]] && fail "[ERR_NOTIF] is undefined - nothing to test here" 65 | log "ERR_NOTIF: [$ERR_NOTIF]" 66 | validate_config_common 67 | 68 | [[ -z "$ARCHIVE_PREFIX" ]] && ARCHIVE_PREFIX='dummy-prefix' 69 | JOB_ID="${ARCHIVE_PREFIX}-$$" 70 | [[ -z "$HOST_ID" ]] && HOST_ID='dummy-host' 71 | 72 | [[ -z "$MSG" ]] && MSG='Test error message' 73 | 74 | unset NO_NOTIF 75 | [[ "$FATAL" -eq 1 ]] && fail "$MSG" || err "$MSG" 76 | 77 | exit 0 78 | -------------------------------------------------------------------------------- /scripts/restore.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # restores selected borg archive from either local or remote repo to $RESTORE_DIR 4 | 5 | readonly SELF="${0##*/}" 6 | JOB_ID="restore-$$" 7 | 8 | readonly usage=" 9 | usage: $SELF [-h] [-d] [-g] [-c CONTAINERS] [-rl] [-B BORG_OPTS] [-L LOCAL_REPO] 10 | [-R REMOTE] [-T REMOTE_REPO] -O RESTORE_DIR -a ARCHIVE_NAME 11 | 12 | Restore data from borg archive 13 | 14 | arguments: 15 | -h show help and exit 16 | -d automatically restore mysql database from dumped file; if this 17 | option is provided and archive doesn't contain exactly one dump-file, 18 | it's an error; be careful, this is a destructive operation! 19 | -g automatically restore postgresql database from dumped file; if this 20 | option is provided and archive contains no sql dumps, it's an error; 21 | be careful, this is a destructive operation! 22 | -c CONTAINERS comma-separated container names to stop before the restore begins; 23 | note they won't be started afterwards, as there might be need 24 | to restore other data (only sql dumps are restored automatically); 25 | requires mounting the docker socket (-v /var/run/docker.sock:/var/run/docker.sock) 26 | -r restore from remote borg repo 27 | -l restore from local borg repo 28 | -B BORG_OPTS additional borg params to pass to extract command 29 | -L LOCAL_REPO overrides container env var of same name 30 | -R REMOTE overrides container env var of same name 31 | -T REMOTE_REPO overrides container env var of same name 32 | -O RESTORE_DIR path to directory where archive will get extracted to 33 | -a ARCHIVE_NAME full name of the borg archive to extract data from 34 | " 35 | 36 | 37 | # TODO: currently user-included sql files would also be picked up by find! 38 | restore_db() { 39 | local mysql_files postgre_files i j d 40 | 41 | declare -a mysql_files postgre_files 42 | 43 | while IFS= read -r -d $'\0' i; do 44 | j="$(basename -- "$i")" 45 | 46 | if [[ "$j" == 'mysql:'* ]]; then 47 | mysql_files+=("$i") 48 | elif [[ "$j" == 'postgres:'* ]]; then 49 | postgre_files+=("$i") 50 | else 51 | err "unrecognized SQL file [$i], ignoring it..." 52 | fi 53 | done < <(find "$RESTORE_DIR" -mindepth 1 -maxdepth 1 -type f -name '*.sql' -print0) 54 | unset i 55 | 56 | if [[ "$RESTORE_MYSQL_DB" == 1 ]]; then 57 | [[ "${#mysql_files[@]}" -ne 1 ]] && fail "expected to find exactly 1 mysql .sql file in the root of [$RESTORE_DIR], but found ${#mysql_files[@]}" 58 | if confirm "restore db from mysql dump [${mysql_files[*]}]?"; then 59 | mariadb \ 60 | --host="${MYSQL_HOST}" \ 61 | --port="${MYSQL_PORT}" \ 62 | --user="${MYSQL_USER}" \ 63 | --password="${MYSQL_PASS}" < "${mysql_files[@]}" 2> >(tee -a "$LOG" >&2) || fail "restoring db from [${mysql_files[*]}] failed w/ [$?]" 64 | else 65 | log "skip restoring mysql db..." 66 | fi 67 | fi 68 | 69 | # https://www.postgresql.org/docs/15/backup-dump.html#BACKUP-DUMP-RESTORE 70 | if [[ "$RESTORE_POSTGRES_DB" == 1 ]]; then 71 | [[ "${#postgre_files[@]}" -eq 0 ]] && fail "expected to find at least 1 postgres .sql file in the root of [$RESTORE_DIR], but found none" 72 | if confirm "restore db from postgres dump(s) [${postgre_files[*]}]?"; then 73 | export PGPASSWORD="$POSTGRES_PASS" 74 | 75 | for i in "${postgre_files[@]}"; do 76 | d="$(basename -- "$i")" 77 | [[ "$d" == 'postgres:all-dbs.sql' ]] && d=postgres || d="$(grep -Po '^postgres:\K.*(?=\.sql$)' <<< "$d")" 78 | # TODO2: restoring from all-dbs.sql, then the cluster should really be empty! 79 | psql \ 80 | -h "$POSTGRES_HOST" \ 81 | -p "$POSTGRES_PORT" \ 82 | -U "$POSTGRES_USER" \ 83 | -d "$d" \ 84 | -f "$i" 2> >(tee -a "$LOG" >&2) || fail "restoring [$d] postgres db from [$i] failed w/ [$?]" 85 | done 86 | else 87 | log "skip restoring postgres db..." 88 | fi 89 | fi 90 | } 91 | 92 | 93 | # TODO: do not fail() if err code <=1? 94 | _restore_common() { 95 | local l_or_r repo start_timestamp t 96 | 97 | l_or_r="$1" 98 | repo="$2" 99 | 100 | pushd -- "$RESTORE_DIR" &> /dev/null || fail "unable to pushd into [$RESTORE_DIR]" 101 | 102 | log "=> Restore from $l_or_r repo [${repo}::${ARCHIVE_NAME}] started..." 103 | start_timestamp="$(date +%s)" 104 | 105 | borg extract -v --list --show-rc \ 106 | $COMMON_OPTS \ 107 | $BORG_OPTS \ 108 | "${repo}::${ARCHIVE_NAME}" > >(tee -a "$LOG") 2> >(tee -a "$LOG" >&2) || fail "=> extracting $l_or_r repo failed w/ [$?] (duration $(print_time "$(( $(date +%s) - start_timestamp ))"))" 109 | 110 | 111 | t="$(( $(date +%s) - start_timestamp ))" 112 | log "=> Extract from $l_or_r repo succeeded in $(print_time "$t")" 113 | 114 | popd &> /dev/null 115 | KEEP_DIR=1 # from this point onward, we should not delete $RESTORE_DIR on failure 116 | restore_db 117 | 118 | t="$(( $(date +%s) - start_timestamp ))" 119 | log "=> Restore finished OK in $(print_time "$t"), contents are in [$RESTORE_DIR]" 120 | } 121 | 122 | 123 | do_restore() { 124 | 125 | if [[ "$LOC" -eq 1 ]]; then 126 | _restore_common local "$LOCAL_REPO" 127 | elif [[ "$REM" -eq 1 ]]; then 128 | _restore_common remote "$REMOTE" 129 | fi 130 | } 131 | 132 | 133 | validate_config() { 134 | local vars 135 | 136 | declare -a vars=( 137 | ARCHIVE_NAME 138 | RESTORE_DIR 139 | ) 140 | [[ "$RESTORE_MYSQL_DB" == 1 ]] && vars+=( 141 | MYSQL_HOST 142 | MYSQL_PORT 143 | MYSQL_USER 144 | MYSQL_PASS 145 | ) 146 | [[ "$RESTORE_POSTGRES_DB" == 1 ]] && vars+=( 147 | POSTGRES_HOST 148 | POSTGRES_PORT 149 | POSTGRES_USER 150 | POSTGRES_PASS 151 | ) 152 | [[ "$REM" -eq 1 ]] && vars+=(REMOTE REMOTE_REPO) 153 | [[ "$LOC" -eq 1 ]] && vars+=(LOCAL_REPO) 154 | 155 | vars_defined "${vars[@]}" 156 | 157 | [[ "$REMOTE_OR_LOCAL_OPT_COUNTER" -ne 1 ]] && fail "need to select whether to restore from local or remote repo" 158 | [[ -d "$RESTORE_DIR" && -w "$RESTORE_DIR" ]] || fail "[$RESTORE_DIR] is not mounted or not writable; missing mount?" 159 | [[ "$LOC" -eq 1 ]] && [[ ! -d "$LOCAL_REPO" || ! -w "$LOCAL_REPO" ]] && fail "[$LOCAL_REPO] does not exist or is not writable; missing mount?" 160 | } 161 | 162 | 163 | create_dirs() { 164 | mkdir -p -- "$RESTORE_DIR" || fail "dir [$RESTORE_DIR] creation failed w/ [$?]" 165 | } 166 | 167 | 168 | cleanup() { 169 | [[ "$KEEP_DIR" -ne 1 && -d "$RESTORE_DIR" ]] && rm -r -- "$RESTORE_DIR" 170 | [[ -d "$RESTORE_DIR" ]] && log "\n\n -> restored files are in [$RESTORE_DIR]" 171 | 172 | log "==> restore script end" 173 | } 174 | 175 | 176 | # ================ 177 | # Entry 178 | # ================ 179 | trap -- 'cleanup; exit' EXIT HUP INT QUIT PIPE TERM 180 | NO_NOTIF=true # do not notify errors 181 | source /scripts_common.sh || { echo -e " ERROR: failed to import /scripts_common.sh" >&2; exit 1; } 182 | REMOTE_OR_LOCAL_OPT_COUNTER=0 183 | 184 | unset RESTORE_MYSQL_DB RESTORE_POSTGRES_DB CONTAINERS REM LOC BORG_OPTS RESTORE_DIR ARCHIVE_NAME # just in case 185 | 186 | while getopts 'dgc:rlB:L:R:T:O:a:h' opt; do 187 | case "$opt" in 188 | d) RESTORE_MYSQL_DB=1 189 | ;; 190 | g) RESTORE_POSTGRES_DB=1 191 | ;; 192 | c) IFS="$SEPARATOR" read -ra CONTAINERS <<< "$OPTARG" 193 | ;; 194 | r) REM=1 195 | let REMOTE_OR_LOCAL_OPT_COUNTER+=1 196 | ;; 197 | l) LOC=1 198 | let REMOTE_OR_LOCAL_OPT_COUNTER+=1 199 | ;; 200 | B) BORG_OPTS="$OPTARG" 201 | ;; 202 | L) LOCAL_REPO="$OPTARG" # overrides env var of same name 203 | ;; 204 | R) REMOTE="$OPTARG" # overrides env var of same name 205 | ;; 206 | T) REMOTE_REPO="$OPTARG" # overrides env var of same name 207 | ;; 208 | O) RESTORE_DIR="$OPTARG" # dir where selected borg archive will be restored into 209 | ;; 210 | a) ARCHIVE_NAME="$OPTARG" 211 | ;; 212 | h) echo -e "$usage" 213 | exit 0 214 | ;; 215 | *) fail "$SELF called with unsupported flag(s)" 216 | ;; 217 | esac 218 | done 219 | 220 | 221 | validate_config 222 | process_remote # note this overwrites global REMOTE var 223 | 224 | readonly RESTORE_DIR="$RESTORE_DIR/restored-${ARCHIVE_NAME}" # define & test after validation, as we're re-defining the arg 225 | [[ -e "$RESTORE_DIR" ]] && fail "[$RESTORE_DIR] already exists, abort" 226 | create_dirs 227 | 228 | stop_containers 229 | do_restore 230 | # do not start containers, so we'd have time to manualy move the data files back, if any 231 | 232 | exit 0 233 | 234 | -------------------------------------------------------------------------------- /scripts/scripts_common.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # common vars & functions 4 | 5 | set -o noglob 6 | set -o pipefail 7 | 8 | readonly CONF_ROOT='/config' 9 | readonly LOG_ROOT="$CONF_ROOT/logs" # note path is also tied to logrotate config 10 | readonly ENV_ROOT="$CONF_ROOT/env" 11 | readonly SCRIPT_ROOT="$CONF_ROOT/scripts" 12 | export LOG="$LOG_ROOT/${SELF}.log" # note SELF is defined by importing file 13 | 14 | [[ "$SEPARATOR" == space ]] && SEPARATOR=' ' 15 | [[ "$SEPARATOR" == comma ]] && SEPARATOR=',' 16 | [[ "$SEPARATOR" == colon ]] && SEPARATOR=':' 17 | [[ "$SEPARATOR" == semicolon ]] && SEPARATOR=';' 18 | readonly SEPARATOR="${SEPARATOR:-,}" # default to comma 19 | 20 | readonly CRON_FILE="$CONF_ROOT/crontab" 21 | readonly MSMTPRC="$CONF_ROOT/msmtprc" 22 | readonly LOGROTATE_CONF="$CONF_ROOT/logrotate.conf" 23 | readonly SSH_KEY="$CONF_ROOT/id_rsa" 24 | LOG_TIMESTAMP_FORMAT='+%F %T' 25 | readonly DEFAULT_NOTIF_TAIL_MSG='\n 26 | ---------------- 27 | host: {h} 28 | archive prefix: {p} 29 | job id: {i} 30 | fatal?: {f}' 31 | readonly ALL_DBS_MARKER='__all__' 32 | 33 | 34 | DEFAULT_MAIL_FROM='{h} backup reporter' 35 | DEFAULT_NOTIF_SUBJECT='{p}: backup error on {h}' 36 | # make sure CURL_FLAGS don't contain $SEPARATOR! (as we join the array into string to export) 37 | CURL_FLAGS=( 38 | -w '\n' 39 | --output /dev/null 40 | --max-time 6 41 | --connect-timeout 3 42 | -s -S --fail -L 43 | ) 44 | declare -A CONTAINER_TO_RUNNING_STATE=() # container_name->is_running mappings at the beginning of script 45 | declare -a CONTAINERS_TO_START=() # list of containers that were stopped by this script and should be started back up upon completion 46 | 47 | # https://borgbackup.readthedocs.io/en/stable/usage/notes.html#ssh-batch-mode 48 | if [[ -n "$BORG_RSH" ]]; then 49 | export BORG_RSH 50 | elif [[ -n "$RSH_EXTRA_OPTS" ]]; then 51 | export BORG_RSH="ssh -o BatchMode=yes -o StrictHostKeyChecking=no $RSH_EXTRA_OPTS" 52 | else 53 | export BORG_RSH='ssh -o BatchMode=yes -o StrictHostKeyChecking=no' # default 54 | fi 55 | 56 | 57 | # No one can answer if Borg asks these questions, it is better to just fail quickly 58 | # instead of hanging: (from https://borgbackup.readthedocs.io/en/stable/deployment/automated-local.html#configuring-the-system) 59 | export BORG_RELOCATED_REPO_ACCESS_IS_OK="${BORG_RELOCATED_REPO_ACCESS_IS_OK:-no}" 60 | export BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK="${BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK:-no}" 61 | 62 | 63 | stop_containers() { 64 | local c 65 | 66 | [[ "${#CONTAINERS[@]}" -eq 0 ]] && return 0 # no containers defined, return 67 | 68 | log "going to stop following containers before starting with backup job: [${CONTAINERS[*]}]" 69 | for c in "${CONTAINERS[@]}"; do 70 | if [[ "${CONTAINER_TO_RUNNING_STATE[$c]}" == true ]]; then 71 | log "=> stopping container [$c]..." 72 | docker stop "$c" 2> >(tee -a "$LOG" >&2) || fail "stopping container [$c] failed w/ [$?]" 73 | CONTAINERS_TO_START+=("$c") 74 | else 75 | log "=> container [$c] already stopped" 76 | fi 77 | done 78 | 79 | log "=> all containers stopped" 80 | 81 | return 0 82 | } 83 | 84 | 85 | # caller might be backgrounding! that's why the container names are passed explicitly. 86 | start_containers() { 87 | local containers c idx err_ 88 | 89 | containers=("$@") 90 | [[ "${#containers[@]}" -eq 0 ]] && return 0 # no containers given, return 91 | 92 | log "going to start following containers that were previously stopped by this job: [${containers[*]}]" 93 | for (( idx=${#containers[@]}-1 ; idx>=0 ; idx-- )); do 94 | c="${containers[idx]}" 95 | log "=> starting container [$c]..." 96 | docker start "$c" 2> >(tee -a "$LOG" >&2) || { err "starting container [$c] failed w/ [$?]"; err_='at least one container failed to start'; } 97 | done 98 | 99 | log "=> ${err_:-all containers started}" 100 | 101 | return 0 102 | } 103 | 104 | 105 | # note this fun is exported 106 | # note _running_dock_by_name() is not strictly needed, as container name would suffice! 107 | # note commands need to block 108 | # 109 | # note with -t option stderr gets merged w/ stdout! 110 | dex() { 111 | docker exec "$(_running_dock_by_name "$1")" "${@:2}" 2> >(tee -a "$LOG" >&2) || { err "running [${*:2}] on container [$1] failed w/ [$?]"; return 1; } 112 | } 113 | 114 | 115 | # find running container ID by container name 116 | # note this fun is exported 117 | _running_dock_by_name() { 118 | local input name_to_id name line 119 | 120 | input="$*" 121 | 122 | declare -A name_to_id 123 | 124 | while read -r line; do 125 | name="$(cut -d' ' -f2- <<< "$line")" 126 | name_to_id[$name]="$(cut -d' ' -f1 <<< "$line")" 127 | done < <(docker ps --no-trunc --format '{{.ID}} {{.Names}}' | grep -i "$input") # note we don't use docker-ps's --filter option, as using grep gives us case-insensitivity 128 | 129 | [[ "${#name_to_id[@]}" -eq 1 ]] || return 1 130 | echo -n "${name_to_id[@]}" 131 | } 132 | 133 | 134 | _compact_common() { 135 | local l_or_r repo start_timestamp t err_code s 136 | 137 | l_or_r="$1" 138 | repo="$2" 139 | 140 | log "=> starting compact operation on $l_or_r repo [$repo]..." 141 | start_timestamp="$(date +%s)" 142 | 143 | borg compact --show-rc \ 144 | $COMMON_OPTS \ 145 | $BORG_OPTS \ 146 | "$repo" > >(tee -a "$LOG") 2> >(tee -a "$LOG" >&2) || { err "$l_or_r borg compact exited w/ [$?]"; err_code=1; } 147 | 148 | t="$(( $(date +%s) - start_timestamp ))" 149 | [[ -z "$err_code" ]] && s='succeeded ' 150 | log "=> $l_or_r compact ${s}in $(print_time "$t")" 151 | 152 | return "${err_code:-0}" 153 | } 154 | 155 | 156 | # note this is called both by compact.sh & backup.sh, so be careful by changing global vars! 157 | compact_repos() { 158 | local started_pids start_timestamp i err_code t 159 | 160 | declare -a started_pids=() 161 | 162 | start_timestamp="$(date +%s)" 163 | 164 | if [[ "$REMOTE_ONLY" -ne 1 ]]; then 165 | _compact_common local "$LOCAL_REPO" & 166 | started_pids+=("$!") 167 | fi 168 | 169 | if [[ "$LOCAL_ONLY" -ne 1 ]]; then 170 | _compact_common remote "$REMOTE" & 171 | started_pids+=("$!") 172 | fi 173 | 174 | for i in "${started_pids[@]}"; do 175 | wait "$i" || err_code="$?" 176 | done 177 | 178 | t="$(( $(date +%s) - start_timestamp ))" 179 | log "=> Compact finished, duration $(print_time "$t")${err_code:+; at least one step failed or produced warning}" 180 | 181 | return "${err_code:-0}" 182 | } 183 | 184 | 185 | # dir existence needs to be verified by the caller! 186 | # 187 | # note this fun is exported 188 | is_dir_empty() { 189 | local dir 190 | 191 | readonly dir="$1" 192 | 193 | [[ -d "$dir" ]] || fail "[$dir] is not a valid dir." 194 | find -L "$dir" -mindepth 1 -maxdepth 1 -print -quit | grep -q . 195 | [[ $? -eq 0 ]] && return 1 || return 0 196 | } 197 | 198 | 199 | confirm() { 200 | local msg yno 201 | 202 | readonly msg="$1" 203 | 204 | while : ; do 205 | [[ -n "$msg" ]] && log "$msg" 206 | read -r yno 207 | case "${yno^^}" in 208 | Y | YES ) 209 | log "Ok, continuing..."; 210 | return 0 211 | ;; 212 | N | NO ) 213 | log "Abort."; 214 | return 1 215 | ;; 216 | *) 217 | err -N "incorrect answer; try again. (y/n accepted)" 218 | ;; 219 | esac 220 | done 221 | } 222 | 223 | 224 | # note this fun is exported 225 | fail() { 226 | err -F "$@" 227 | err -N " - ABORTING -" 228 | exit 1 229 | } 230 | 231 | 232 | # info lvl logging 233 | # 234 | # note this fun is exported 235 | log() { 236 | local msg 237 | readonly msg="$1" 238 | echo -e "[$(date "$LOG_TIMESTAMP_FORMAT")] [$JOB_ID]\tINFO $msg" | tee -a "$LOG" 239 | return 0 240 | } 241 | 242 | 243 | # 244 | # note this fun is exported 245 | err() { 246 | local opt msg f no_notif OPTIND no_mail_orig 247 | 248 | no_mail_orig="$NO_SEND_MAIL" 249 | 250 | while getopts 'FNM' opt; do 251 | case "$opt" in 252 | F) f='-F' # only to be provided by fail(), ie do not pass -F flag to err() yourself! 253 | ;; 254 | N) no_notif=1 255 | ;; 256 | M) NO_SEND_MAIL=true # note this would be redundant if -N is already given 257 | ;; 258 | *) fail -N "$FUNCNAME called with unsupported flag(s) [$opt]" 259 | ;; 260 | esac 261 | done 262 | shift "$((OPTIND-1))" 263 | 264 | readonly msg="$1" 265 | echo -e "[$(date "$LOG_TIMESTAMP_FORMAT")] [$JOB_ID]\t ERROR $msg" | tee -a "$LOG" >&2 266 | [[ "$no_notif" -ne 1 ]] && notif $f "$msg" 267 | 268 | NO_SEND_MAIL="$no_mail_orig" # reset to previous value 269 | } 270 | 271 | 272 | # note no notifications are generated if shell is in interactive mode 273 | # 274 | # note this fun is exported 275 | notif() { 276 | local msg f msg_tail 277 | 278 | [[ "$1" == '-F' ]] && { f='-F'; shift; } 279 | [[ "$-" == *i* || "$NO_NOTIF" == true ]] && return 0 280 | 281 | msg="$1" 282 | 283 | if [[ "${ADD_NOTIF_TAIL:-true}" == true ]]; then 284 | msg_tail="$(echo -e "${NOTIF_TAIL_MSG:-$DEFAULT_NOTIF_TAIL_MSG}")" 285 | msg+="$msg_tail" 286 | fi 287 | 288 | # if this function was called from a script that accesses this function via exported vars: 289 | [[ -n "${ERR_NOTIF[*]}" && "${#ERR_NOTIF[@]}" -eq 1 && "${ERR_NOTIF[*]}" == *"$SEPARATOR"* ]] && IFS="$SEPARATOR" read -ra ERR_NOTIF <<< "$ERR_NOTIF" 290 | 291 | if contains mail "${ERR_NOTIF[@]}" && [[ "$NO_SEND_MAIL" != true ]]; then 292 | mail $f -t "$MAIL_TO" -f "$MAIL_FROM" -s "$NOTIF_SUBJECT" -a "$SMTP_ACCOUNT" -b "$msg" & 293 | fi 294 | 295 | if contains pushover "${ERR_NOTIF[@]}"; then 296 | pushover $f -s "$NOTIF_SUBJECT" -b "$msg" & 297 | fi 298 | 299 | if contains healthchecksio "${ERR_NOTIF[@]}"; then 300 | hcio $f -b "$msg" & 301 | fi 302 | } 303 | 304 | 305 | # 306 | # note this fun is exported 307 | mail() { 308 | local opt to from subj acc body is_fail err_code account OPTIND 309 | 310 | while getopts "Ft:f:s:b:a:" opt; do 311 | case "$opt" in 312 | F) is_fail=true 313 | ;; 314 | t) to="$OPTARG" 315 | ;; 316 | f) from="$OPTARG" 317 | ;; 318 | s) subj="$OPTARG" 319 | ;; 320 | b) body="$OPTARG" 321 | ;; 322 | a) acc="$OPTARG" 323 | ;; 324 | *) fail -M "$FUNCNAME called with unsupported flag(s) [$opt]" 325 | ;; 326 | esac 327 | done 328 | shift "$((OPTIND-1))" 329 | 330 | [[ -n "$acc" ]] && declare -a account=('-a' "$acc") 331 | 332 | msmtp "${account[@]}" --read-envelope-from -t <> "$known_f" || fail "adding host [$host_w_port] to $known_f failed w/ [$?]" 458 | fi 459 | } 460 | 461 | 462 | # note this validation can't be called for anything else than backup & notif-test scripts; 463 | # eg list/extract scripts shouldn't validate existence of SMTP_* env vars. 464 | # !! it's also called by setup.sh, but with -i flag to skip some checks. 465 | # 466 | # also note we expand the ERR_NOTIF env var into an array here! 467 | validate_config_common() { 468 | local i vars init 469 | 470 | [[ "$1" == -i ]] && init=1 471 | 472 | declare -a vars 473 | 474 | if [[ "$init" -ne 1 ]]; then 475 | if [[ -n "$HC_ID" ]]; then 476 | if is_valid_url "$HC_ID"; then 477 | HC_URL="$HC_ID" 478 | elif [[ "$HC_ID" == disable* ]]; then 479 | unset HC_URL 480 | elif [[ -z "$HC_URL" ]]; then 481 | err "[HC_ID] given, but no healthcheck url template provided" 482 | elif [[ "$HC_URL" != *'{id}'* ]]; then 483 | err "[HC_URL] template does not contain id placeholder [{id}]" 484 | else 485 | HC_URL="$(sed "s/{id}/$HC_ID/g" <<< "$HC_URL")" 486 | fi 487 | fi 488 | 489 | if [[ "$HC_URL" == *'{id}'* ]]; then 490 | err "[HC_URL] with {id} placeholder defined, but no replacement value provided" 491 | fi 492 | fi 493 | 494 | IFS="$SEPARATOR" read -ra ERR_NOTIF <<< "$ERR_NOTIF" 495 | 496 | if [[ "${#ERR_NOTIF[@]}" -gt 0 ]]; then 497 | for i in "${ERR_NOTIF[@]}"; do 498 | [[ "$i" =~ ^(mail|pushover|healthchecksio)$ ]] || fail "unsupported [ERR_NOTIF] value: [$i]" 499 | done 500 | 501 | if contains mail "${ERR_NOTIF[@]}"; then 502 | vars+=(MAIL_TO) 503 | 504 | [[ -f "$MSMTPRC" && -s "$MSMTPRC" ]] || vars+=( 505 | SMTP_HOST 506 | SMTP_USER 507 | SMTP_PASS 508 | ) 509 | fi 510 | 511 | contains pushover "${ERR_NOTIF[@]}" && vars+=( 512 | PUSHOVER_APP_TOKEN 513 | PUSHOVER_USER_KEY 514 | ) 515 | 516 | if [[ "$init" -ne 1 ]] && contains healthchecksio "${ERR_NOTIF[@]}"; then 517 | #vars+=(HC_URL) 518 | 519 | local hcio_rgx='^https?://hc-ping.com/[-a-z0-9]+/?$' 520 | if ! [[ "$HC_URL" =~ $hcio_rgx ]]; then 521 | err "healthchecksio selected for notifications, but configured HC_URL [$HC_URL] does not match expected healthchecks.io url pattern [$hcio_rgx]" 522 | fi 523 | fi 524 | fi 525 | 526 | vars_defined "${vars[@]}" 527 | 528 | vars=() # reset 529 | [[ -n "$MYSQL_FAIL_FATAL" ]] && vars+=(MYSQL_FAIL_FATAL) 530 | [[ -n "$POSTGRES_FAIL_FATAL" ]] && vars+=(POSTGRES_FAIL_FATAL) 531 | [[ -n "$ADD_NOTIF_TAIL" ]] && vars+=(ADD_NOTIF_TAIL) 532 | [[ -n "$SCRIPT_FAIL_FATAL" ]] && vars+=(SCRIPT_FAIL_FATAL) 533 | validate_true_false "${vars[@]}" 534 | 535 | validate_containers 536 | validate_remote 537 | [[ -n "$BORG_RSH" && -n "$RSH_EXTRA_OPTS" ]] && fail "[BORG_RSH] & [RSH_EXTRA_OPTS] are mutually exclusive" 538 | } 539 | 540 | 541 | validate_remote() { 542 | local host port 543 | 544 | if [[ -n "$REMOTE" ]]; then 545 | IFS=':' read -r host port <<< "$REMOTE" 546 | if [[ -n "$port" ]] && ! is_digit "$port"; then 547 | fail "port in REMOTE:PORT, if defined, needs to be digit, but was [$port]" 548 | fi 549 | fi 550 | } 551 | 552 | 553 | process_remote() { 554 | if [[ -n "$REMOTE" ]]; then 555 | validate_remote 556 | 557 | #add_remote_to_known_hosts_if_missing "$REMOTE" 558 | if [[ "$REMOTE" == *:* ]]; then 559 | readonly REMOTE+="$REMOTE_REPO" # define after validation, as we're re-defining the arg 560 | else 561 | readonly REMOTE+=":$REMOTE_REPO" # define after validation, as we're re-defining the arg 562 | fi 563 | fi 564 | } 565 | 566 | 567 | # validate valid container names have been given 568 | # and store their current running state globally 569 | validate_containers() { 570 | local c running 571 | 572 | [[ "${#CONTAINERS[@]}" -eq 0 ]] && return 0 573 | 574 | for c in "${CONTAINERS[@]}"; do 575 | running="$(docker container inspect -f '{{.State.Running}}' "$c")" 576 | if [[ "$?" -ne 0 ]]; then 577 | # TODO: should we fail here instead? 578 | err "container [$c] inspection failed - does the container exist?" 579 | else 580 | is_true_false "$running" || err "container [$c] inspection result not true|false: [$running]" 581 | CONTAINER_TO_RUNNING_STATE[$c]="$running" 582 | fi 583 | done 584 | } 585 | 586 | 587 | vars_defined() { 588 | local i val 589 | 590 | for i in "$@"; do 591 | val="$(eval echo "\$$i")" || fail "evaling [echo \"\$$i\"] failed w/ [$?]" 592 | [[ -z "$val" ]] && fail "[$i] is not defined" 593 | done 594 | } 595 | 596 | 597 | validate_true_false() { 598 | local i val 599 | 600 | for i in "$@"; do 601 | val="$(eval echo "\$$i")" || fail "evaling [echo \"\$$i\"] failed w/ [$?]" 602 | is_true_false "$val" || fail "$i value, when given, can be either [true] or [false]" 603 | done 604 | } 605 | 606 | 607 | is_true_false() { 608 | [[ "$*" =~ ^(true|false)$ ]] 609 | } 610 | 611 | 612 | # 613 | # note this fun is exported 614 | expand_placeholders() { 615 | local m is_fatal 616 | 617 | m="$1" 618 | is_fatal="${2:-false}" # true|false; indicates whether given error caused job to abort/exit 619 | 620 | m="$(sed "s/{h}/$HOST_ID/g" <<< "$m")" 621 | m="$(sed "s/{p}/$ARCHIVE_PREFIX/g" <<< "$m")" 622 | m="$(sed "s/{i}/$JOB_ID/g" <<< "$m")" 623 | m="$(sed "s/{f}/$is_fatal/g" <<< "$m")" 624 | 625 | echo "$m" 626 | } 627 | 628 | 629 | file_type() { 630 | if [[ -h "$*" ]]; then 631 | echo symlink 632 | elif [[ -f "$*" ]]; then 633 | echo file 634 | elif [[ -d "$*" ]]; then 635 | echo dir 636 | elif [[ -p "$*" ]]; then 637 | echo 'named pipe' 638 | elif [[ -c "$*" ]]; then 639 | echo 'character special' 640 | elif [[ -b "$*" ]]; then 641 | echo 'block special' 642 | elif [[ -S "$*" ]]; then 643 | echo socket 644 | elif ! [[ -e "$*" ]]; then 645 | echo 'does not exist' 646 | else 647 | echo UNKNOWN 648 | fi 649 | } 650 | 651 | 652 | # Checks whether given url is a valid one. 653 | # 654 | # @param {string} url url which validity to test. 655 | # 656 | # @returns {bool} true, if provided url was a valid url. 657 | # 658 | # note this fun is exported 659 | is_valid_url() { 660 | local regex 661 | 662 | readonly regex='^(https?|ftp|file)://[-A-Za-z0-9\+&@#/%?=~_|!:,.;]*[-A-Za-z0-9\+&@#/%=~_|]' 663 | 664 | [[ "$1" =~ $regex ]] 665 | } 666 | 667 | 668 | ping_healthcheck() { 669 | [[ -z "$HC_URL" ]] && return 0 670 | 671 | curl "${CURL_FLAGS[@]}" \ 672 | --retry 5 \ 673 | --user-agent "$HOST_ID" \ 674 | "$HC_URL" || err "pinging healthcheck service at [$HC_URL] failed w/ [$?]" 675 | } 676 | 677 | 678 | run_scripts() { 679 | local stage dir flags start_timestamp t msg 680 | 681 | stage="$1" 682 | 683 | flags=() 684 | [[ "${SCRIPT_FAIL_FATAL:-true}" == true ]] && flags+=('--exit-on-error') 685 | flags+=( 686 | -a "$stage" 687 | -a "$(join -- "${CONTAINERS[@]}")" 688 | -a "$(join -- "${NODES_TO_BACK_UP[@]}")" 689 | ) 690 | 691 | # export all the necessary args & functions that child processes might want/need to use: 692 | export LOG LOG_TIMESTAMP_FORMAT ARCHIVE_PREFIX HOST_ID SEPARATOR 693 | export NO_SEND_MAIL NO_NOTIF ADD_NOTIF_TAIL NOTIF_TAIL_MSG DEFAULT_NOTIF_TAIL_MSG 694 | export MAIL_TO MAIL_FROM DEFAULT_MAIL_FROM NOTIF_SUBJECT DEFAULT_NOTIF_SUBJECT SMTP_ACCOUNT 695 | export PUSHOVER_USER_KEY PUSHOVER_APP_TOKEN PUSHOVER_PRIORITY PUSHOVER_EXPIRE 696 | export HC_URL TMP TMP_ROOT CONF_ROOT 697 | #export ERR_NOTIF CURL_FLAGS CONTAINERS NODES_TO_BACK_UP # arrays, need to be joined and passed directly to the command below 698 | export -f fail err log notif mail pushover hcio expand_placeholders contains is_dir_empty print_time is_valid_url 699 | export -f dex _running_dock_by_name is_digit 700 | 701 | 702 | # TODO: deprecate /each? realistically there won't be a case where one would 703 | # want to exec something on each and every step, right? 704 | for dir in \ 705 | "$SCRIPT_ROOT/each" \ 706 | "$SCRIPT_ROOT/$stage" \ 707 | "$JOB_SCRIPT_ROOT/each" \ 708 | "$JOB_SCRIPT_ROOT/$stage"; do 709 | 710 | [[ -d "$dir" ]] || continue 711 | is_dir_empty "$dir" && continue 712 | 713 | log "stage [$stage]: executing following scripts in [$dir]:" 714 | run-parts --test "${flags[@]}" "$dir" > >(tee -a "$LOG") 2> >(tee -a "$LOG" >&2) || err "run-parts dry-run for stage [$stage] in [$dir] failed w/ $?" 715 | 716 | start_timestamp="$(date +%s)" 717 | 718 | ERR_NOTIF="$(join -- "${ERR_NOTIF[@]}")" \ 719 | JOB_ID="${JOB_ID}-$stage" \ 720 | CURL_FLAGS="$(join -- "${CURL_FLAGS[@]}")" \ 721 | CONTAINERS="$(join -- "${CONTAINERS[@]}")" \ 722 | NODES_TO_BACK_UP="$(join -- "${NODES_TO_BACK_UP[@]}")" \ 723 | run-parts "${flags[@]}" "$dir" 724 | if [[ "$?" -ne 0 ]]; then 725 | msg="custom script execution for stage [$stage] in [$dir] failed" 726 | [[ "${SCRIPT_FAIL_FATAL:-true}" == true ]] && fail "${msg}; aborting" || err "${msg}; not aborting" 727 | fi 728 | 729 | t="$(( $(date +%s) - start_timestamp ))" 730 | log "stage [$stage]: executing [$dir] done in $(print_time "$t")" 731 | done 732 | } 733 | 734 | 735 | # 736 | # note this fun is exported 737 | contains() { 738 | local src i 739 | 740 | [[ "$#" -lt 2 ]] && { err "at least 2 args needed for $FUNCNAME"; return 2; } 741 | 742 | src="$1" 743 | shift 744 | 745 | for i in "$@"; do 746 | [[ "$i" == "$src" ]] && return 0 747 | done 748 | 749 | return 1 750 | } 751 | 752 | 753 | # note if SEPARATOR were guaranteed to be single char, then all this could be 754 | # replaced by "$(IFS=,; echo "${i[*]}")" 755 | join() { 756 | local opt OPTIND sep list i 757 | 758 | sep="$SEPARATOR" # default 759 | 760 | while getopts 's:' opt; do 761 | case "$opt" in 762 | s) sep="$OPTARG" 763 | ;; 764 | *) fail "$FUNCNAME called with unsupported flag(s) [$opt]" 765 | ;; 766 | esac 767 | done 768 | shift "$((OPTIND-1))" 769 | 770 | for i in "$@"; do 771 | [[ -z "$i" ]] && continue 772 | list+="${i}$sep" 773 | done 774 | 775 | echo "${list:0:$(( ${#list} - ${#sep} ))}" 776 | } 777 | 778 | 779 | # 780 | # note this fun is exported 781 | print_time() { 782 | local sec tot r 783 | 784 | sec="$1" 785 | 786 | tot=$((sec%60)) 787 | r="${tot}s" 788 | 789 | if [[ "$sec" -gt "$tot" ]]; then 790 | r="$((sec%3600/60))m:$r" 791 | let tot+=$((sec%3600)) 792 | fi 793 | 794 | if [[ "$sec" -gt "$tot" ]]; then 795 | r="$((sec%86400/3600))h:$r" 796 | let tot+=$((sec%86400)) 797 | fi 798 | 799 | if [[ "$sec" -gt "$tot" ]]; then 800 | r="$((sec/86400))d:$r" 801 | fi 802 | 803 | echo -n "$r" 804 | } 805 | 806 | 807 | # note this fun is exported 808 | is_digit() { 809 | [[ "$*" =~ ^[0-9]+$ ]] 810 | } 811 | 812 | 813 | mkdir -p "$LOG_ROOT" || { echo -e " ERROR: [mkdir -p $LOG_ROOT] failed w/ $?" >&2; exit 1; } 814 | [[ -f "$ENV_ROOT/common-env.conf" ]] && source "$ENV_ROOT/common-env.conf" 815 | 816 | if [[ "$DEBUG" == true ]]; then 817 | set -x 818 | printenv 819 | echo 820 | fi 821 | 822 | true # always exit common w/ good code 823 | -------------------------------------------------------------------------------- /scripts/setup.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # this is service bootstrap logic to be called from container entrypoint. 4 | # 5 | # - initialises crontab; 6 | # - sets ssh key, if available & adds our remote borg to know_hosts; 7 | # - configures msmtprc for mail notifications; 8 | 9 | readonly SELF="${0##*/}" 10 | JOB_ID="setup-$$" 11 | 12 | 13 | check_dependencies() { 14 | local i 15 | 16 | for i in curl docker mariadb mariadb-dump \ 17 | borg ssh-keygen ssh-keyscan tr \ 18 | sed find msmtp run-parts \ 19 | psql pg_dump pg_dumpall; do 20 | command -v "$i" >/dev/null || fail "[$i] not installed" 21 | done 22 | } 23 | 24 | 25 | setup_crontab() { 26 | #local cron_target 27 | #readonly cron_target='/var/spool/cron/crontabs/root' 28 | 29 | if [[ -f "$CRON_FILE" && -s "$CRON_FILE" ]]; then 30 | ## TODO: this won't work, as /config is mounted read-only: 31 | #grep -q '^BASH_ENV=' "$CRON_FILE" || sed -i '1s+^+BASH_ENV=/container.env\n+' "$CRON_FILE" 32 | #grep -q '^SHELL=' "$CRON_FILE" || sed -i '1s+^+SHELL=/bin/bash\n+' "$CRON_FILE" 33 | 34 | #[[ -f "$cron_target" ]] || fail "[$cron_target] does not exist; is cron installed?" 35 | #cp -- "$CRON_FILE" "$cron_target" 36 | 37 | # or, alterntaively, install via $crontab: 38 | /usr/bin/crontab "$CRON_FILE" || fail "crontab installation failed w/ [$?]" 39 | fi 40 | } 41 | 42 | 43 | install_ssh_key() { 44 | local ssh_key_target 45 | 46 | readonly ssh_key_target="$HOME/.ssh/id_rsa" 47 | 48 | [[ -d "$HOME/.ssh" ]] || fail "[~/.ssh] is not a dir; is ssh client installed?" 49 | if [[ -f "$SSH_KEY" && -s "$SSH_KEY" ]]; then 50 | cp -- "$SSH_KEY" "$ssh_key_target" || fail "ssh keyfile copy failed w/ $?" 51 | ssh-keygen -y -P "" -f "$ssh_key_target" &>/dev/null || fail "provided ssh key is password-protected - this is not supported" 52 | fi 53 | 54 | # sanitize .ssh perms: 55 | chmod -R u=rwX,g=,o= -- ~/.ssh 56 | 57 | #[[ -n "$REMOTE" ]] && add_remote_to_known_hosts_if_missing "$REMOTE" 58 | } 59 | 60 | 61 | setup_msmtp() { 62 | local target_conf 63 | 64 | target_conf='/etc/msmtprc' 65 | 66 | rm -f /usr/sbin/sendmail || fail "rm sendmail failed w/ $?" 67 | ln -s /usr/bin/msmtp /usr/sbin/sendmail || fail "linking sendmail failed w/ $?" 68 | 69 | if [[ -f "$MSMTPRC" && -s "$MSMTPRC" ]]; then 70 | cat -- "$MSMTPRC" > "$target_conf" 71 | else 72 | cat > "$target_conf" < "$target_conf" 131 | else 132 | cat > "$target_conf" < /env_vars.sh || { echo -e " ERROR: printenv failed" | tee -a "$LOG"; exit 1; } 149 | #env | sed -r "s/'/\\\'/gm" | sed -r "s/^([^=]+=)(.*)\$/\1'\2'/gm" \ > /etc/environment 150 | #declare -p | grep -Ev '\b(BASHOPTS|BASH_VERSINFO|EUID|PPID|SHELLOPTS|UID)=' > /container.env || { echo -e " ERROR: printenv failed" | tee -a "$LOG"; exit 1; } 151 | source /scripts_common.sh || { echo -e " ERROR: failed to import /scripts_common.sh" >&2; exit 1; } 152 | 153 | #chmod 600 /container.env || fail "chmod-ing /container.env failed w/ [$?]" 154 | 155 | check_dependencies 156 | validate_config_common -i 157 | setup_crontab 158 | install_ssh_key 159 | setup_msmtp 160 | setup_logrotate 161 | unset NO_SEND_MAIL 162 | 163 | exit 0 164 | 165 | --------------------------------------------------------------------------------