├── .gitignore ├── META.json ├── README.pod ├── bin ├── omnipitr-archive ├── omnipitr-backup-cleanup ├── omnipitr-backup-master ├── omnipitr-backup-slave ├── omnipitr-checksum ├── omnipitr-cleanup ├── omnipitr-monitor ├── omnipitr-restore ├── omnipitr-synch └── sanity-check.sh ├── doc ├── LICENSE ├── changes.pod ├── howto.pod ├── install.pod ├── internals.pod ├── intro.pod ├── omnipitr-archive.pod ├── omnipitr-backup-cleanup.pod ├── omnipitr-backup-master.pod ├── omnipitr-backup-slave.pod ├── omnipitr-cleanup.pod ├── omnipitr-monitor.pod ├── omnipitr-restore.pod ├── omnipitr-synch.pod └── todo.pod ├── lib └── OmniPITR │ ├── Log.pm │ ├── Pidfile.pm │ ├── Program.pm │ ├── Program │ ├── Archive.pm │ ├── Backup.pm │ ├── Backup │ │ ├── Cleanup.pm │ │ ├── Master.pm │ │ └── Slave.pm │ ├── Checksum.pm │ ├── Cleanup.pm │ ├── Monitor.pm │ ├── Monitor │ │ ├── Check.pm │ │ ├── Check │ │ │ ├── Archive_Queue.pm │ │ │ ├── Current_Archive_Time.pm │ │ │ ├── Current_Restore_Time.pm │ │ │ ├── Dump_State.pm │ │ │ ├── Errors.pm │ │ │ ├── Last_Archive_Age.pm │ │ │ ├── Last_Backup_Age.pm │ │ │ └── Last_Restore_Age.pm │ │ ├── Parser.pm │ │ └── Parser │ │ │ ├── Archive.pm │ │ │ ├── Backup.pm │ │ │ ├── Backup_Master.pm │ │ │ ├── Backup_Slave.pm │ │ │ └── Restore.pm │ ├── Restore.pm │ └── Synch.pm │ ├── Tools.pm │ └── Tools │ ├── CommandPiper.pm │ └── ParallelSystem.pm ├── t ├── 00-load.t ├── 10-perl-critic.t ├── 20-perltidy.t ├── perlcriticrc └── perltidyrc └── test ├── .gitignore ├── run.test.sh ├── tags └── test-lib ├── 01.metainfo ├── 02.make.master ├── 03.make.master.backup ├── 04.make_standalone ├── 05.make.normal.slave ├── 06.make.slave.backup ├── 07.make.sr.slave ├── 08.make.sr.slave.backup ├── 09.test.promotion ├── 10.make.master-sx.backup ├── 11.make.slave-sx.backup └── helper-dst-pipe.sh /.gitignore: -------------------------------------------------------------------------------- 1 | *.clog 2 | -------------------------------------------------------------------------------- /META.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "omnipitr", 3 | "abstract": "Advanced WAL File / Backup Management Tools", 4 | "description": "OmniPITR provides a set of tools for managing PITR, including archive_commands, restore_commands, and creating backups from either the master or slave server", 5 | "version": "2.0.0", 6 | "maintainer": "Robert Treat ", 7 | "license": "postgresql", 8 | "release_status": "stable", 9 | "prereqs": { 10 | "runtime": { 11 | "requires": { 12 | "PostgreSQL": "10.0.0" 13 | } 14 | } 15 | }, 16 | "resources":{ 17 | "repository":{ 18 | "url":"git://github.com/omniti-labs/omnipitr.git", 19 | "web":"https://github.com/omniti-labs/omnipitr", 20 | "type":"git" 21 | } 22 | }, 23 | "meta-spec": { 24 | "version": "1.0.0", 25 | "url": "http://pgxn.org/meta/spec.txt" 26 | }, 27 | "tags": [ 28 | "pitr","backups","replication" 29 | ], 30 | "provides": { 31 | "omnipitr": { 32 | "abstract": "Advanced WAL File / Backup Management Tools", 33 | "file": "bin/omnipitr-archive", 34 | "docfile": "doc/howto.pod", 35 | "version": "2.0.0" 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /README.pod: -------------------------------------------------------------------------------- 1 | =head1 OmniPITR 2 | 3 | =head2 OVERVIEW 4 | 5 | OmniPITR is a set of scripts to ease setting up WAL replication, and making hot backups from both Master and Slave systems. 6 | 7 | This set of scripts has been written to make the operation seamless, secure and as light on resources-usage as possible. 8 | 9 | To make it portable, it was written in Perl, but with assumption that it should work on any Perl installation - i.e. no dependencies on non-base Perl modules. 10 | 11 | =head2 INSTALLATION 12 | 13 | /* Simply fetch the directory L, and you're done. More details are in I. */ 14 | 15 | =head2 USAGE 16 | 17 | There is a set of scripts in bin/ directory. All named I are scripts meant for general usage (others are not really useful unless you'll encounter some problems). 18 | 19 | For every one of them you will have documentation in I.pod> - for example documentation for I script is in I. 20 | 21 | Quick list of programs: 22 | 23 | =over 24 | 25 | =item * omnipitr-archive 26 | 27 | This should be used on master server, as I command, setup in 28 | postgresql.conf. 29 | 30 | =item * omnipitr-restore 31 | 32 | This should be used on slave server/servers, as I command in 33 | recovery.conf file. 34 | 35 | =item * omnipitr-master-backup 36 | 37 | Used to make hot backup on master DB server. 38 | 39 | =item * omnipitr-slave-backup 40 | 41 | Used to make hot backup on slave DB server. 42 | 43 | =item * omnipitr-monitor 44 | 45 | General script for I/I type of systems - monitoring, graphing. Provides a way to check for replication lag, last backup timestamp, and other metrics. 46 | 47 | =item * omnipitr-cleanup 48 | 49 | Removes obsolete wal segments from wal archive, when using streaming 50 | replication 51 | 52 | =item * omnipitr-sync 53 | 54 | Send copy of PGDATA to remote locations - even multiple at the same time. 55 | 56 | =item * omnipitr-backup-cleanup 57 | 58 | This can be called from scheduler (crontab) to enforce retention period (7 days is default) for backups and related WAL archive files. 59 | 60 | =back 61 | 62 | In most of the cases you can simply call the program you want to use with I<--help> option to get brief overview of command line options. 63 | 64 | =head2 SUPPORT 65 | 66 | Currently there is simply mailing list that you can subscribe, and post your 67 | questions/problems to. 68 | 69 | Maling list page (includes option to subscribe, and view archives) is available 70 | here: http://lists.omniti.com/mailman/listinfo/pgtreats-users/ 71 | 72 | =head2 COPYRIGHT 73 | 74 | The OmniPITR project is Copyright (c) 2009-2013 OmniTI. All rights reserved. 75 | 76 | =head2 LICENCE INFORMATION 77 | 78 | https://github.com/omniti-labs/omnipitr/blob/master/doc/LICENSE 79 | -------------------------------------------------------------------------------- /bin/omnipitr-archive: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | use strict; 3 | 4 | use Cwd qw(abs_path); 5 | use FindBin; 6 | use lib abs_path( "$FindBin::Bin/../lib" ); 7 | 8 | use OmniPITR::Program::Archive; 9 | 10 | OmniPITR::Program::Archive->new()->run(); 11 | 12 | # vim: set ft=perl : 13 | -------------------------------------------------------------------------------- /bin/omnipitr-backup-cleanup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | use strict; 3 | 4 | use Cwd qw(abs_path); 5 | use FindBin; 6 | use lib abs_path( "$FindBin::Bin/../lib" ); 7 | 8 | use OmniPITR::Program::Backup::Cleanup; 9 | 10 | OmniPITR::Program::Backup::Cleanup->new()->run(); 11 | 12 | # vim: set ft=perl : 13 | -------------------------------------------------------------------------------- /bin/omnipitr-backup-master: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | use strict; 3 | 4 | use Cwd qw(abs_path); 5 | use FindBin; 6 | use lib abs_path( "$FindBin::Bin/../lib" ); 7 | 8 | use OmniPITR::Program::Backup::Master; 9 | 10 | OmniPITR::Program::Backup::Master->new()->run(); 11 | 12 | # vim: set ft=perl : 13 | -------------------------------------------------------------------------------- /bin/omnipitr-backup-slave: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | use strict; 3 | 4 | use Cwd qw(abs_path); 5 | use FindBin; 6 | use lib abs_path( "$FindBin::Bin/../lib" ); 7 | 8 | use OmniPITR::Program::Backup::Slave; 9 | 10 | OmniPITR::Program::Backup::Slave->new()->run(); 11 | 12 | # vim: set ft=perl : 13 | -------------------------------------------------------------------------------- /bin/omnipitr-checksum: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | use strict; 3 | 4 | use Cwd qw(abs_path); 5 | use FindBin; 6 | use lib abs_path( "$FindBin::Bin/../lib" ); 7 | 8 | use OmniPITR::Program::Checksum; 9 | 10 | OmniPITR::Program::Checksum->new()->run(); 11 | 12 | # vim: set ft=perl : 13 | -------------------------------------------------------------------------------- /bin/omnipitr-cleanup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | use strict; 3 | 4 | use Cwd qw(abs_path); 5 | use FindBin; 6 | use lib abs_path( "$FindBin::Bin/../lib" ); 7 | 8 | use OmniPITR::Program::Cleanup; 9 | 10 | OmniPITR::Program::Cleanup->new()->run(); 11 | 12 | # vim: set ft=perl : 13 | -------------------------------------------------------------------------------- /bin/omnipitr-monitor: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | use strict; 3 | 4 | use Cwd qw(abs_path); 5 | use FindBin; 6 | use lib abs_path( "$FindBin::Bin/../lib" ); 7 | 8 | use OmniPITR::Program::Monitor; 9 | 10 | OmniPITR::Program::Monitor->new()->run(); 11 | 12 | # vim: set ft=perl : 13 | -------------------------------------------------------------------------------- /bin/omnipitr-restore: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | use strict; 3 | 4 | use Cwd qw(abs_path); 5 | use FindBin; 6 | use lib abs_path( "$FindBin::Bin/../lib" ); 7 | 8 | use OmniPITR::Program::Restore; 9 | 10 | OmniPITR::Program::Restore->new()->run(); 11 | 12 | # vim: set ft=perl : 13 | -------------------------------------------------------------------------------- /bin/omnipitr-synch: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | use strict; 3 | 4 | use Cwd qw(abs_path); 5 | use FindBin; 6 | use lib abs_path( "$FindBin::Bin/../lib" ); 7 | 8 | use OmniPITR::Program::Synch; 9 | 10 | OmniPITR::Program::Synch->new()->run(); 11 | 12 | # vim: set ft=perl : 13 | -------------------------------------------------------------------------------- /bin/sanity-check.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | BIN_DIR=$( perl -le 'use Cwd qw(realpath); print realpath(shift)' "$( dirname $0 )" ) 4 | LIB_DIR=$( perl -le 'use Cwd qw(realpath); print realpath(shift)' "$( dirname $0 )/../lib" ) 5 | 6 | echo "Checking:" 7 | echo "- $BIN_DIR" 8 | echo "- $LIB_DIR" 9 | 10 | BIN_COUNT=0 11 | for x in "$BIN_DIR"/omnipitr-* 12 | do 13 | BIN_COUNT=$(( $BIN_COUNT + 1 )) 14 | done 15 | 16 | LIB_COUNT=$( find "$LIB_DIR" -name '*.pm' -print | wc -l ) 17 | 18 | echo "$BIN_COUNT programs, $LIB_COUNT libraries." 19 | 20 | declare -a WARNINGS ERRORS 21 | WARNINGS_COUNT=0 22 | ERRORS_COUNT=0 23 | 24 | # Base checks 25 | 26 | if [[ -z $( which ssh 2> /dev/null ) ]] 27 | then 28 | WARNINGS_COUNT=$(( 1 + $WARNINGS_COUNT )) 29 | WARNINGS[$WARNINGS_COUNT]="you don't have ssh program available" 30 | fi 31 | if [[ -z $( which rsync 2> /dev/null ) ]] 32 | then 33 | ERRORS_COUNT=$(( 1 + $ERRORS_COUNT )) 34 | ERRORS[$ERRORS_COUNT]="you don't have rsync program available" 35 | fi 36 | 37 | if [[ -z $( which perl 2> /dev/null ) ]] 38 | then 39 | ERRORS_COUNT=$(( 1 + $ERRORS_COUNT )) 40 | ERRORS[$ERRORS_COUNT]="you don't have Perl?!" 41 | else 42 | PERLVEROK=$( perl -e 'print $] >= 5.008 ? "ok" : "nok"' ) 43 | if [[ "$PERLVEROK" == "nok" ]] 44 | then 45 | WARNINGS_COUNT=$(( 1 + $WARNINGS_COUNT )) 46 | WARNINGS[$WARNINGS_COUNT]="your Perl is old (we support only 5.8 or newer. OmniPITR might work, but was not tested on your version of Perl)" 47 | fi 48 | fi 49 | 50 | # perl code checks 51 | 52 | for x in "$BIN_DIR"/omnipitr-* $( find "$LIB_DIR" -name '*.pm' -print ) 53 | do 54 | if ! perl -I"$LIB_DIR" -wc "$x" &>/dev/null 55 | then 56 | ERRORS_COUNT=$(( 1 + $ERRORS_COUNT )) 57 | ERRORS[$ERRORS_COUNT]="can't compile $x ?!" 58 | fi 59 | done 60 | 61 | # modules check 62 | 63 | for MODULE in $( egrep "^use[[:space:]]" "$BIN_DIR"/omnipitr-* $( find "$LIB_DIR" -name '*.pm' -print ) | perl -pe 's/^[^:]+:use\s+//; s/[;\s].*//' | egrep -v '^OmniPITR' | sort | uniq ) 64 | do 65 | if ! perl -I"$LIB_DIR" -e "use $MODULE" &>/dev/null 66 | then 67 | ERRORS_COUNT=$(( 1 + $ERRORS_COUNT )) 68 | ERRORS[$ERRORS_COUNT]="you don't have $MODULE Perl library (should be installed together with Perl)" 69 | fi 70 | done 71 | 72 | for MODULE in Time::HiRes 73 | do 74 | if ! perl -I"$LIB_DIR" -e "use $MODULE" &>/dev/null 75 | then 76 | WARNINGS_COUNT=$(( 1 + $WARNINGS_COUNT )) 77 | WARNINGS[$WARNINGS_COUNT]="you don't have $MODULE Perl library - it's optional, but it could help" 78 | fi 79 | done 80 | 81 | # versions check 82 | 83 | echo "Tar version" 84 | 85 | tar_version_ok="$( LC_ALL=C tar --version 2>/dev/null | head -n 1 | egrep '^tar \(GNU tar\) [0-9]*(\.[0-9]*)*$' | awk '{print $NF}' | awk -F. '$1>1 || ( $1 == 1 && $2>=20 ) {print "OK"}' )" 86 | if [[ "$tar_version_ok" != "OK" ]] 87 | then 88 | ERRORS_COUNT=$(( 1 + $ERRORS_COUNT )) 89 | ERRORS[$ERRORS_COUNT]="tar (in \$PATH) is either not gnu tar, or gnu tar earlier than required 1.20" 90 | fi 91 | 92 | # Report of status 93 | 94 | if [[ $WARNINGS_COUNT -gt 0 ]] 95 | then 96 | echo "Warnings:" 97 | for WARNING in "${WARNINGS[@]}" 98 | do 99 | echo "- $WARNING" 100 | done 101 | fi 102 | if [[ $ERRORS_COUNT -gt 0 ]] 103 | then 104 | echo "Errors:" 105 | for ERROR in "${ERRORS[@]}" 106 | do 107 | echo "- $ERROR" 108 | done 109 | fi 110 | if [[ "$WARNINGS_COUNT" -eq 0 && "$ERRORS_COUNT" -eq 0 ]] 111 | then 112 | echo "All checked, and looks ok." 113 | exit 0 114 | fi 115 | echo -n "All checked. " 116 | if [[ "$WARNINGS_COUNT" -eq 0 ]] 117 | then 118 | echo -n "No warnings. " 119 | elif [[ "$WARNINGS_COUNT" -eq 1 ]] 120 | then 121 | echo -n "1 warning. " 122 | else 123 | echo -n "${WARNINGS_COUNT} warnings. " 124 | fi 125 | if [[ "$ERRORS_COUNT" -eq 0 ]] 126 | then 127 | echo -n "No errors." 128 | elif [[ "$ERRORS_COUNT" -eq 1 ]] 129 | then 130 | echo -n "1 error." 131 | else 132 | echo -n "${ERRORS_COUNT} errors." 133 | fi 134 | 135 | echo 136 | exit 1 137 | -------------------------------------------------------------------------------- /doc/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009-2013, OmniTI Computer Consulting, Inc. 2 | 3 | Permission to use, copy, modify, and distribute this software and its 4 | documentation for any purpose, without fee, and without a written agreement 5 | is hereby granted, provided that the above copyright notice and this 6 | paragraph and the following two paragraphs appear in all copies. 7 | 8 | IN NO EVENT SHALL OmniTI Computer Consulting, Inc. BE LIABLE TO ANY PARTY 9 | FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, 10 | INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS 11 | DOCUMENTATION, EVEN IF OmniTI Computer Consulting, Inc. HAS BEEN ADVISED OF 12 | THE POSSIBILITY OF SUCH DAMAGE. 13 | 14 | OmniTI Computer Consulting, Inc. SPECIFICALLY DISCLAIMS ANY WARRANTIES, 15 | INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND 16 | FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN 17 | "AS IS" BASIS, AND OmniTI Computer Consulting, Inc. HAS NO OBLIGATIONS TO 18 | PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. 19 | -------------------------------------------------------------------------------- /doc/changes.pod: -------------------------------------------------------------------------------- 1 | =head1 OmniPITR 2 | 3 | =encoding utf8 4 | 5 | =head2 2019-06-25 6 | 7 | =over 8 | 9 | =item * Use numerical time zone offsets in backup meta files, instead of 10 | timezone codes, as it's more portable. Per report 11 | ( https://github.com/omniti-labs/omnipitr/issues/42 ) from Mike from New 12 | Zealand. 13 | 14 | =back 15 | 16 | =head2 2017-11-16 17 | 18 | =over 19 | 20 | =item * Release of version 2.0.0 21 | 22 | =item * Make OmniPITR work with PostgreSQL 10 23 | 24 | =item * This version of OmniPITR will *not* work properly with PostgreSQL 9.x 25 | or earlier. 26 | 27 | =back 28 | 29 | =head2 2015-04-26 30 | 31 | =over 32 | 33 | =item * Add missing documentation for --dry-run for omnipitr-backup-cleanup 34 | 35 | =back 36 | 37 | =head2 2015-07-22 38 | 39 | =over 40 | 41 | =item * Release of version 1.3.3 42 | 43 | =item * Add support for xz and lz4 compressions, thanks to Michael Pirogov 44 | 45 | =back 46 | 47 | =head2 2014-01-07 48 | 49 | =over 50 | 51 | =item * Fix for version checking for x.y.z versions of tar. Per report 52 | ( https://github.com/omniti-labs/omnipitr/issues/14 ) from David Johnston. 53 | 54 | =back 55 | 56 | =head2 2013-12-22 57 | 58 | =over 59 | 60 | =item * Release of version 1.3.2 61 | 62 | =item * Fix for handling --skip-xlog together with remote destinations (-dr) 63 | in omnipitr-backup-*. Bug reported 64 | (https://github.com/omniti-labs/omnipitr/issues/13) by Otto Bretz 65 | 66 | =back 67 | 68 | =head2 2013-12-17 69 | 70 | =over 71 | 72 | =item * Release of version 1.3.1 73 | 74 | =item * Added --truncate and --sleep to omnipitr-backup-cleanup 75 | 76 | =back 77 | 78 | =head2 2013-12-16 79 | 80 | =over 81 | 82 | =item * Release of version 1.3.0 83 | 84 | =item * Added --dry-run to omnipitr-backup-cleanup 85 | 86 | =item * Added --help/docs to omnipitr-backup-cleanup 87 | 88 | =back 89 | 90 | =head2 2013-12-13 91 | 92 | =over 93 | 94 | =item * Added omnipitr-backup-cleanup - program to remove old backups, and 95 | xlogs from walarchive, leaving consistent state (all backups can be restored 96 | using xlogs from walarchive). 97 | 98 | =back 99 | 100 | =head2 2013-12-12 101 | 102 | =over 103 | 104 | =item * Added timezone and hostname information to meta files 105 | 106 | =back 107 | 108 | =head2 2013-12-10 109 | 110 | =over 111 | 112 | =item * change retention of error messages in omnipitr-monitor cache to 3 days 113 | (from 30). This is to make sure that the cache file will not be too big. 114 | 115 | =back 116 | 117 | =head2 2013-11-29 118 | 119 | =over 120 | 121 | =item * *meta* files are generated for backups. These files contain 122 | information when backup has started, and what is the minimal xlog file 123 | required for recovery from it. 124 | 125 | =back 126 | 127 | =head2 2013-08-23 128 | 129 | =over 130 | 131 | =item * Release of version 1.2.0 132 | 133 | =item * --dst-pipe for omnipitr-backup-master, omnipitr-backup-slave and 134 | omnipitr-archive 135 | 136 | =item * support for using %r in omnipitr-restore for cleanup purposes 137 | 138 | =item * --skip-xlog option to disable creation of xlog tarball from backup 139 | processes 140 | 141 | =item * various bugfixes 142 | 143 | =back 144 | 145 | =head2 2013-04-01 146 | 147 | =over 148 | 149 | =item * Release of version 1.1.0, including all changes since release 1.0.0 150 | 151 | =back 152 | 153 | =head2 2012-12-11 154 | 155 | =over 156 | 157 | =item * Provide support for --help option 158 | 159 | =item * Add support for config file 160 | 161 | =item * Add --version option to all programs. 162 | 163 | =back 164 | 165 | =head2 2012-12-04 166 | 167 | =over 168 | 169 | =item * Make temp directories based on PID, to allow easy running (for example) 170 | 2 slaves on the same machine. 171 | 172 | =back 173 | 174 | =head2 2012-12-04 175 | 176 | =over 177 | 178 | =item * Add option to make omnipitr-restore able to remove files from walarchive 179 | before serving next xlog to PostgreSQL. This is important in cases where slave 180 | falls back a lot, and it's catching up - but it doesn't start to remove old, 181 | obsolete wal segments before it will fully catch up. Now, with new option 182 | (--remove-before) it will delete old wal segments during catchup phase too. 183 | 184 | =back 185 | 186 | =head2 2012-08-08 187 | 188 | =over 189 | 190 | =item * Make sanity-check.sh return sensible exit code - pull from François 191 | Beausoleil ( https://github.com/francois ) 192 | 193 | =back 194 | 195 | =head2 2012-08-08 196 | 197 | =over 198 | 199 | =item * Fix log line parsing in omnipitr-monitor - the bug in there prevented 200 | omnipitr-monitor from working when running on server with timezone "before" UTC 201 | (for example UTC-0400). 202 | 203 | =back 204 | 205 | =head2 2012-07-02 206 | 207 | =over 208 | 209 | =item * Release of version 1.0.0 210 | 211 | =item * Finally omnipitr-monitor is working 212 | 213 | =back 214 | 215 | =head2 2012-05-07 216 | 217 | =over 218 | 219 | =item * Release of version 0.7.0 220 | 221 | =item * Cherry-pick wal segments for omnipitr-backup-slave, instead of archiving 222 | whole WALARCHIVE 223 | 224 | =back 225 | 226 | =head2 2012-04-26 227 | 228 | =over 229 | 230 | =item * Release of version 0.6.0 231 | 232 | =item * Support parallelism of delivery to remote destinations, and unpacking of 233 | wal archives for omnipitr-backup-slave 234 | 235 | =back 236 | 237 | =head2 2012-03-30 238 | 239 | =over 240 | 241 | =item * Release of version 0.5.0 242 | 243 | =item * Direct destination support - details: http://www.depesz.com/2012/03/30/omnipitr-0-5-0/ 244 | 245 | =back 246 | 247 | =head2 2012-01-19 248 | 249 | =over 250 | 251 | =item * Release of version 0.4.0 252 | 253 | =item * Fix handling of symlinks to walarchive in omnipitr-backup-slave 254 | 255 | =back 256 | 257 | =head2 2012-01-17 258 | 259 | =over 260 | 261 | =item * New program omnipitr-cleanup to be used as archive_cleanup_command in 262 | recovery.conf - for streaming replication setups 263 | 264 | =back 265 | 266 | =head2 2012-01-11 267 | 268 | =over 269 | 270 | =item * Release version 0.3.1 271 | 272 | =item * Fixed bug with handling digests, that could cause omnipitr-backup-master 273 | or omnipitr-backup-slave to *not* make backup if bad digester was requested. 274 | 275 | =back 276 | 277 | =head2 2012-01-04 278 | 279 | =over 280 | 281 | =item * Release version 0.3.0 282 | 283 | =item * Added omnipitr-synch program, to send PostgreSQL data in sane way to 284 | remote machines (for example to setup new slave). 285 | 286 | =item * Switch to use environment based perl interpreter, instead of hardcoding 287 | path 288 | 289 | =back 290 | 291 | =head2 2011-12-14 292 | 293 | =over 294 | 295 | =item * Release version 0.2.1 296 | 297 | =item * Change handling of IO to use external tools (tee) to avoid deadlocks 298 | in case of slow compression 299 | 300 | =back 301 | 302 | =head2 2011-11-10 303 | 304 | =over 305 | 306 | =item * Release version 0.2.0 307 | 308 | =item * Added ability to create checksums of backups, with --digest option for 309 | omnipitr-backup-master and omnipitr-backup-slave programs 310 | 311 | =back 312 | 313 | =head2 2011-08-05 314 | 315 | =over 316 | 317 | =item * Release version 0.1.3 318 | 319 | =item * Fixed bug with handling Minimum Recovery Location in slave backup 320 | 321 | =back 322 | 323 | =head2 2011-06-30 324 | 325 | =over 326 | 327 | =item * Release version 0.1.2 328 | 329 | =item * Fixed bug with handling multiple local destinations for backups. 330 | 331 | =back 332 | 333 | =head2 2011-06-17 334 | 335 | =over 336 | 337 | =item * Release version 0.1.1 338 | 339 | =item * Changed short version of --psql-path from -pp to -sp - -pp was already 340 | taken by --pgcontrol-path. Bug reported by luoyi. 341 | 342 | =back 343 | 344 | =head2 2011-06-14 345 | 346 | =over 347 | 348 | =item * Release version 0.1.0 349 | 350 | =back 351 | 352 | =head2 2011-06-08 353 | 354 | =over 355 | 356 | =item * Fix slave backup on 9.0 systems (new option, -cm) 357 | 358 | =item * Add test system, not really configurable now, but works, and lets me 359 | easily verify that OmniPITR still works. 360 | 361 | =item * Add (finally) versions. So far only to libraries. Can be read with: 362 | 363 | perl -I/opt/omnipitr/lib -le 'use OmniPITR::Program; print $OmniPITR::Program::VERSION' 364 | 365 | =back 366 | 367 | =head2 2011-05-18 368 | 369 | =over 370 | 371 | =item * Allow omnipitr-backup-master and omnipitr-backup-slave to work when data 372 | dir path is given as symlink 373 | 374 | =back 375 | 376 | =head2 2011-03-24 377 | 378 | =over 379 | 380 | =item * Fix typo in generated .backup file - harmless as far as I know. 381 | 382 | =back 383 | 384 | =head2 2011-03-11 385 | 386 | =over 387 | 388 | =item * Fix handling of pidfile in backup-master and backup-slave programs. Bug 389 | spotted by Albert Czarnecki 390 | 391 | =back 392 | 393 | =head2 2011-02-10 394 | 395 | =over 396 | 397 | =item * Make the IMMEDIATE finish of recovery actually work. 398 | 399 | =item * Make omnipitr-backup-slave compatible with Streaming Replication in 400 | PostgreSQL 9.0 401 | 402 | =item * Add option (--streaming-replication) to make it possible to use 403 | omnipitr-restore in 9.0 Streaming Replication environment. 404 | 405 | =back 406 | 407 | =head2 2011-01-27 408 | 409 | =over 410 | 411 | =item * Silence warning if source is not defined on the command line 412 | 413 | =back 414 | 415 | =head2 2011-01-27 416 | 417 | =over 418 | 419 | =item * Silence warning if source is not defined on the command line 420 | 421 | =back 422 | 423 | =head2 2011-01-24 424 | 425 | =over 426 | 427 | =item * Add detection of tar version to sanity-check.sh 428 | 429 | =back 430 | 431 | =head2 2011-01-20 432 | 433 | =over 434 | 435 | =item * Add internals.pod with first part of documentation of "how it works, and 436 | why this way" 437 | 438 | =back 439 | 440 | =head2 2011-01-14 441 | 442 | =over 443 | 444 | =item * Locate any tablespaces in the data dir and include them in the backup 445 | for both masters and slaves. In the tar file these will be placed under the 446 | 'tablespaces' directory. 447 | 448 | =back 449 | 450 | =head2 2011-01-12 451 | 452 | =over 453 | 454 | =item * localize signal handler so that it does not get called outside of the 455 | function call 456 | 457 | =back 458 | 459 | =head2 2011-01-05 460 | 461 | =over 462 | 463 | =item * Add proper stub for make_xlog_archive in OmniPITR::Program::Backup, that 464 | enforces reimplementation in child classes. 465 | 466 | =back 467 | 468 | =head2 2010-09-28 469 | 470 | =over 471 | 472 | =item * Fix error with handling master backups where given data-dir contained 473 | trailing / character 474 | 475 | =back 476 | 477 | =head2 2010-09-10 478 | 479 | =over 480 | 481 | =item * Fix error handling for pg_controldata 482 | 483 | =back 484 | 485 | =head2 2010-09-08 486 | 487 | =over 488 | 489 | =item * Add option to handle various ways of dealing with problems with calls to 490 | pg_controldata (--error-pgcontroldata) to omnipitr-restore 491 | 492 | =back 493 | 494 | =head2 2010-09-07 495 | 496 | =over 497 | 498 | =item * Log whole response from pg_controldata in case there are problems with it 499 | 500 | =back 501 | 502 | =head2 2010-07-28 503 | 504 | =over 505 | 506 | =item * Add forgotten --nice-path to omnipitr-archive 507 | 508 | =item * Add option not to use nice for omnipitr-archive, and both 509 | omnipitr-backup-* programs 510 | 511 | =back 512 | 513 | =head2 2010-07-15 514 | 515 | =over 516 | 517 | =item * Added "IMPORTANT NOTICES" part to documents for omnipitr programs with 518 | additional info that might not be obvious always. 519 | 520 | =item * Added information about reuiring at least gnu tar 1.20 (earlier version 521 | didn't have --transform option which is required by omnipitr-backup-slave) 522 | 523 | =back 524 | 525 | =head2 2010-06-30 526 | 527 | =over 528 | 529 | =item * Refactoring done. 530 | 531 | =item * omnipitr-backup-slave in beta 532 | 533 | =back 534 | 535 | =head2 2010-06-29 536 | 537 | =over 538 | 539 | =item * First part of refactoring the code 540 | 541 | =back 542 | 543 | =head2 2010-06-28 544 | 545 | =over 546 | 547 | =item * Fix type in changelog (28 -> 23) 548 | 549 | =item * Fix call to tar with 2 --transform options. Some tar versions don't 550 | handle it well, and it's required for omnipitr-backup-slave 551 | 552 | =item * Exclude whole pg_log and pg_xlog directories if these are symlinks - no 553 | point in restoring them on slave system, and it's pretty common to be symlinked. 554 | 555 | =back 556 | 557 | =head2 2010-06-23 558 | 559 | =over 560 | 561 | =item * Fix description of -x option for omnipitr-backup-master 562 | 563 | =item * Add information that one should remove finish recovery trigger after 564 | successfull finish of wal replication. 565 | 566 | =item * alpha version of omnipitr-backup-slave 567 | 568 | =back 569 | 570 | =head2 2010-06-12 571 | 572 | =over 573 | 574 | =item * Fix ommission in workaround for PostgreSQL 8.2 and 8.3 - regarding order 575 | of xlog segments. 576 | 577 | =back 578 | 579 | =head2 2010-06-10 580 | 581 | =over 582 | 583 | =item * Fix bug caused by silencing rm - destination-backup directory wasn't 584 | removed after successful exit of backup. 585 | 586 | =back 587 | 588 | =head2 2010-06-07 589 | 590 | =over 591 | 592 | =item * Silence rmtree - disable confirmation message to be printed to screen - 593 | to make sure cron jobs with backup don't create irrelevant mails. 594 | 595 | =back 596 | 597 | =head2 2010-06-05 598 | 599 | =over 600 | 601 | =item * Added documentation about requirements 602 | 603 | =item * Added workaround for PostgreSQL 8.2 and 8.3, which do not wait for .backup 604 | "wal segment" to be archived before returning control from pg_stop_backup() 605 | call. 606 | 607 | =item * Added changelog 608 | 609 | =back 610 | 611 | =head2 COPYRIGHT 612 | 613 | The OmniPITR project is Copyright (c) 2009-2013 OmniTI. All rights reserved. 614 | -------------------------------------------------------------------------------- /doc/howto.pod: -------------------------------------------------------------------------------- 1 | =head1 OmniPITR - how to setup. 2 | 3 | =head2 OVERVIEW 4 | 5 | Following documentation walks you through installation, setup and running of 6 | master-slave replication of PostgreSQL database using WAL segment files (also 7 | known as PiTR Replication. 8 | 9 | It will also cover methods of making backups, both from master and slave 10 | servers, and methods for monitoring replication status. 11 | 12 | =head2 SETUP OVERVIEW 13 | 14 | Following document provides information about setting replication/backup 15 | scenario, on 3 machines, with following responsibilities: 16 | 17 | =over 18 | 19 | =item * master server - server (both hardware and software) that contains the working, 20 | writable PostgreSQL instance 21 | 22 | =item * slave server - server (both hardware and software) that contains secondary, 23 | warm or hot standby server. With PostgreSQL pre 9.0 it's not readable, but from 24 | 9.0 you can send read-only queries to it. 25 | 26 | =item * backup server - additional machine that is being used as storage for 27 | backup files - both full-database backups and/or backup of currently generated 28 | xlog files (WAL segments) 29 | 30 | =back 31 | 32 | Before setting replication it is required that you have PostgreSQL installed on 33 | both master and slave servers, and working PostgreSQL cluster on master. 34 | 35 | Paths should be the same on master and slave - at least to data directory. 36 | 37 | For the purpose of example, I assume following paths on each servers - you are 38 | in no way obliged to use the same paths - these are just explanations if 39 | something wouldn't be clear from examples later on: 40 | 41 | =head3 Master server paths 42 | 43 | =over 44 | 45 | =item * /home/postgres - home directory of postgres user - user that is used to 46 | run PostgreSQL 47 | 48 | =item * /home/postgres/data - data directory for PostgreSQL 49 | 50 | =item * /home/postgres/data/postgresql.conf - PostgreSQL configuration file for 51 | master 52 | 53 | =item * /var/tmp - temporary directory that can be freely used 54 | 55 | =back 56 | 57 | =head3 Slave server paths 58 | 59 | =over 60 | 61 | =item * /home/postgres - home directory of postgres user - user that is used to 62 | run PostgreSQL 63 | 64 | =item * /var/tmp - temporary directory that can be freely used 65 | 66 | =back 67 | 68 | =head3 Backup server paths 69 | 70 | =over 71 | 72 | =item * /var/backups - backups directory - all backups should arrive in here. 73 | 74 | =back 75 | 76 | Also I assume that PostgreSQL itself, and OmniPITR programs work from postgres 77 | system account. 78 | 79 | =head2 PREPARATION 80 | 81 | To make everything working we need a way to copy files from master to slave and 82 | backup, and from slave to backup. 83 | 84 | Since I is using RSync for all transfers, we can effectively use two 85 | ways of communication: 86 | 87 | =over 88 | 89 | =item * rsync over ssh 90 | 91 | =item * direct rsync 92 | 93 | =back 94 | 95 | First method is usually easier to setup - as you most likely have ssh daemon on 96 | all machines, so it is enough to install rsync program, and you're virtually 97 | good to go. 98 | 99 | Second method is a bit more complex, as you need additinally to setup rsync 100 | daemon, but it pays off in increased security, faster transfers and less 101 | resource-intensive work. The biggest drawback is that data being sent are not 102 | encrypted. Which might, or might not be an issue, depending on distance, and 103 | trust for connection - i.e. using encrypted vpn and then plain rsync is (in my 104 | opinion) favorable over rsync-over-ssh, due to extra security steps we can have 105 | when using rsync as daemon. 106 | 107 | In any way, we need to make some extra directories (all this directories should 108 | be writable by postgres user, preferably also owned by it): 109 | 110 | =over 111 | 112 | =item * On master server: 113 | 114 | =over 115 | 116 | =item * /home/postgres/omnipitr - this is where omnipitr will store it's 117 | internal datafiles (rather small) 118 | 119 | =item * /home/postgres/omnipitr/log - place for omnipitr log 120 | 121 | =item * /home/postgres/omnipitr/state - place for omnipitr state files 122 | 123 | =item * /var/tmp/omnipitr - place for temporary files (larger) created and used 124 | by omnipitr 125 | 126 | =back 127 | 128 | =item * On slave server: 129 | 130 | =over 131 | 132 | =item * /home/postgres/wal_archive - this is where master will send xlog 133 | segments to be used for replication 134 | 135 | =item * /home/postgres/omnipitr - this is where omnipitr will store it's 136 | internal datafiles (rather small) 137 | 138 | =item * /home/postgres/omnipitr/log - place for omnipitr log 139 | 140 | =item * /home/postgres/omnipitr/state - place for omnipitr state files 141 | 142 | =item * /var/tmp/omnipitr - place for temporary files (larger) created and used 143 | by omnipitr 144 | 145 | =back 146 | 147 | =item * On backup server: 148 | 149 | =over 150 | 151 | =item * /var/backups/database - top level directory for database backups 152 | 153 | =item * /var/backups/database/hot_backup - directory to put hot backup files in 154 | 155 | =item * /var/backups/database/xlog - directory to put xlog segments in 156 | 157 | =back 158 | 159 | =back 160 | 161 | And then we need to allow uploads to them. 162 | 163 | For this - you'd rather consult your sysadmins. For the sake of this document, I 164 | assume that the chosen method was direct rsync, and we have working following 165 | rsync paths: 166 | 167 | =over 168 | 169 | =item * rsync://slave/wal_archive/ - points to /home/postgres/wal_archive/, with 170 | write access for master, without password 171 | 172 | =item * rsync://backup/database/ - points to /var/backups/database, with write 173 | access for master and slave, without password 174 | 175 | =back 176 | 177 | =head2 ACTUAL INSTALLATION AND CONFIGURATION 178 | 179 | On both master and slave machines, please install omnipitr to /opt/omnipitr 180 | directory, for example using: 181 | 182 | $ git clone git://github.com/omniti-labs/omnipitr.git 183 | 184 | Now, on both machines, check if the installation is OK, that is run sanity 185 | check: 186 | 187 | $ /opt/omnipitr/bin/sanity-check.sh 188 | Checking: 189 | - /opt/omnipitr/bin 190 | - /opt/omnipitr/lib 191 | 5 programs, 9 libraries. 192 | All checked, and looks ok. 193 | 194 | if there are any errors/warnings - do whatever you can to fix them. 195 | 196 | Now. If this works well, we can move on to setting up all subsequent steps. 197 | 198 | =head3 Archival of XLOGs from master 199 | 200 | In /home/postgres/data/postgresql.conf find section "WRITE AHEAD LOG" -> 201 | "Archiving". Usually it contains lines like these: 202 | 203 | #archive_mode = off # allows archiving to be done 204 | #archive_command = '' # command to use to archive a logfile segment 205 | #archive_timeout = 0 # force a logfile segment switch after this 206 | 207 | You need to enable archive_mode: 208 | 209 | archive_mode = on 210 | 211 | (in older versions ( pre 8.2 ) there is no archive_mode. It's ok - just don't 212 | add it). 213 | 214 | Set archive_command to: 215 | 216 | archive_command = '/opt/omnipitr/bin/omnipitr-archive -l /home/postgres/omnipitr/log/omnipitr-^Y^m^d.log -s /home/postgres/omnipitr/state -dr gzip=rsync://slave/wal_archive/ -dr gzip=rsync://backup/database/xlog/ -db /var/tmp/omnipitr/dstbackup -t /var/tmp/omnipitr/ -v "%p"' 217 | 218 | and archive_timeout to: 219 | 220 | archive_timeout = 60 221 | 222 | This will make sure that in worst case you have 1 minute lag. 223 | 224 | Meaning of options: 225 | 226 | =over 227 | 228 | =item * -l /home/postgres/omnipitr/log/omnipitr-^Y^m^d.log : Path to logfile, 229 | will be automatically rotated on date change. 230 | 231 | =item * -s /home/postgres/omnipitr/state : Directory to keep state information, 232 | internal stuff, not really interesting, small files only, but it is required if 233 | we have more than 1 destination 234 | 235 | =item * -dr gzip=rsync://slave/wal_archive/ : sends gzip compressed wal segments 236 | to slave to appropriate path 237 | 238 | =item * -dr gzip=rsync://backup/database/xlog/ : sends the same gzip compressed 239 | wal segments to backup server for long term storage 240 | 241 | =item * -db /var/tmp/omnipitr/dstbackup : it is important that this path 242 | shouldn't exist - it's a directory, that (if it exists) will be used as 243 | additional local destination - for the purposes of omnipitr-backup-master program 244 | 245 | =item * -t /var/tmp/omnipitr/ : where to keep temporary files (when needed) 246 | 247 | =item * -v : log verbose information - mostly timings. 248 | 249 | =back 250 | 251 | Afterwards you need to restart PostgreSQL (or reload if you changed only 252 | archive_command and/or archive_timeout). 253 | 254 | Archiving should work nicely now, which you can see after couple of minutes 255 | (assuming you set archive_timeout to 60 seconds, like I showed above). 256 | 257 | You should start seeing compressed files showing on slave and backup servers, 258 | and appropriate information in logfile. 259 | 260 | =head3 Creation of hot backup on master 261 | 262 | This is actually pretty simple - assuming you have working archiving (described 263 | in previous section of this howto. 264 | 265 | You simply run this command: 266 | 267 | /opt/omnipitr/bin/omnipitr-backup-master -D /home/postgres/data -l /home/postgres/omnipitr/log/omnipitr-^Y^m^d.log -x /var/tmp/omnipitr/dstbackup -dr gzip=rsync://backup/database/hot_backup/ -t /var/tmp/omnipitr/ --pid-file /home/postgres/omnipitr/backup-master.pid -v 268 | 269 | Meaning of options: 270 | 271 | =over 272 | 273 | =item * -D /home/postgres/data : Where is data directory for PostgreSQL instance 274 | 275 | =item * -l /home/postgres/omnipitr/log/omnipitr-^Y^m^d.log : Where to log 276 | information - logfile will be daily rotated. Logfile can be shared between 277 | OmniPITR programs as each line contains identifier of program that wrote it 278 | 279 | =item * -x /var/tmp/omnipitr/dstbackup : Should be the same path as in I<-db> 280 | option to omnipitr-archive, and it shouldn't exist - omnipitr-backup-master will 281 | create it, and later on remove. 282 | 283 | =item * -dr gzip=rsync://backup/database/hot_backup/ : Will send gzip compressed 284 | backups to backup server to appropriate directory. 285 | 286 | =item * -t /var/tmp/omnipitr/ : Where to create temporary files 287 | 288 | =item * --pid-file /home/postgres/omnipitr/backup-master.pid : When running 289 | manually, it's not needed, but when the same command will be put in cronjob - 290 | it's nice to be sure that only 1 backup can run at the same time 291 | 292 | =item * -v : Log verbosely - mostly add timings. 293 | 294 | =back 295 | 296 | After running, on backup server you will get 2 files: 297 | 298 | =over 299 | 300 | =item * master-data-YYYY-MM-DD.tar.gz 301 | 302 | =item * master-xlog-YYYY-MM-DD.tar.gz 303 | 304 | =back 305 | 306 | which together form backup for master database. 307 | 308 | =head3 Starting up slave 309 | 310 | First you need to obtain master-data backup file from backup server. Xlog file 311 | is only used if you want to start the server as fully working r/w server and not 312 | replication slave. 313 | 314 | So, get the .tar.gz file to /home/postgres, and uncompress it. 315 | 316 | It will create /home/postgres/data directory which contains copy of PostgreSQL 317 | files from master. 318 | 319 | Now, edit /home/postgres/data/postgresql.conf (this, like any other file 320 | mentioned in this section - on slave), and change archive_command to: 321 | 322 | archive_command = '/bin/true' 323 | 324 | To make sure that slave will not try to archive xlog files. 325 | 326 | Afterwards, in /home/postgres/data directory make I file with 327 | single line: 328 | 329 | restore_command = ' /opt/omnipitr/bin/omnipitr-restore -l /var/tmp/omnipitr-^Y^m^d.log -s gzip=/home/postgres/wal_archive -f /home/postgres/omnipitr/finish.recovery -r -p /home/postgres/omnipitr/pause.removal -v -t /var/tmp/omnipitr/ -w 900 %f %p' 330 | 331 | Meaning of options: 332 | 333 | =over 334 | 335 | =item * -l /var/tmp/omnipitr-^Y^m^d.log : Where to log information - logfile 336 | will be daily rotated. Logfile can be shared between OmniPITR programs as each 337 | line contains identifier of program that wrote it 338 | 339 | =item * -s gzip=/home/postgres/wal_archive : Where to find xlog segments sent in 340 | from master, and how are they compressed 341 | 342 | =item * -f /home/postgres/omnipitr/finish.recovery : Trigger file - if it will 343 | exist - recovery will be stopped. 344 | 345 | =item * -r : remove no longer needed xlog segments - it's a bit more tricky than 346 | 'segments that have been already applied', but we can be sure it's safe 347 | operation - even in cases like stopping slave recovery for days. 348 | 349 | =item * -p /home/postgres/omnipitr/pause.removal : trigger file - if it exists - 350 | segments will no longer be removed when no longer needed - this is useful for 351 | omnipitr-backup-slave 352 | 353 | =item * -v : log verbosely 354 | 355 | =item * -t /var/tmp/omnipitr/ : where to create temporary files, if needed. 356 | 357 | =item * -w 900 : wait 15 minutes between xlog arrival and application - this is 358 | to be able to stop recovery in case of catastrophic query being run on master 359 | (think: I) 360 | 361 | =back 362 | 363 | Afterwards - just start PostgreSQL on slave, and check generated logs for 364 | information about progress, but everything should be working just fine now. 365 | 366 | =head2 COPYRIGHT 367 | 368 | The OmniPITR project is Copyright (c) 2009-2013 OmniTI. All rights reserved. 369 | -------------------------------------------------------------------------------- /doc/install.pod: -------------------------------------------------------------------------------- 1 | =head1 OmniPITR - Installation 2 | 3 | =head2 INSTALLATION 4 | 5 | You can get the OmniPITR project from L. Within this directory you will find set of directories: 6 | 7 | =over 8 | 9 | =item * bin/ 10 | 11 | This is where the scripts reside - usually this is the only place you need to 12 | look into. 13 | 14 | =item * doc/ 15 | 16 | Documentation. In Plain Old Documentation format, so you can view it practically 17 | everywhere Perl is with I command. 18 | 19 | =item * lib/ 20 | 21 | Here be dragons. Don't look. Don't touch. Be afraid. 22 | 23 | =back 24 | 25 | After fetching the omnipitr project/directory, put it in any place in your 26 | filesystem you'd want. Usually you will keep all of these together, but you can 27 | always remove the docs, put the scripts in some system bin/ directory, and put 28 | libs anyplace you want - just make sure that path to them is in I 29 | environment variable. 30 | 31 | In case you will not decide to split the distribution - i.e. you will have 32 | someplace in your system I directory, which will contain I and 33 | I directories (just like it is distributed) - you don't have to set 34 | I - scripts will automatically find their libs. 35 | 36 | =head2 REQUIREMENTS 37 | 38 | Standard Perl distribution should be enough. On some Linux systems (Debian, 39 | Ubuntu) core Perl modules have been packaged separately - in such case you need 40 | to install also perl-modules package. 41 | 42 | Additionally, you will need some programs installed: 43 | 44 | =over 45 | 46 | =item * nice 47 | 48 | =item * rsync 49 | 50 | =item * tar (it has to be GNU tar version 1.20 or newer) 51 | 52 | =back 53 | 54 | Additionally, you might want to use compression, in which case you need I, 55 | I or I compression programs. 56 | 57 | Please note that you can use I to get faster, but gzip-compatible, 58 | compression. To do so, specify all destinations/sources using I prefix, 59 | but add --gzip-path option with value pointing to I program. 60 | 61 | =head2 SANITY CHECK 62 | 63 | After installation it's good to run I script (from I) 64 | directory - it will check if all prerequisites are available. 65 | 66 | Output might look like this: 67 | 68 | postgres@server:~$ omnipitr/bin/sanity-check.sh 69 | Checking: 70 | - /var/lib/postgres/omnipitr/bin 71 | - /var/lib/postgres/omnipitr/lib 72 | 5 programs, 7 libraries. 73 | All checked, and looks ok. 74 | 75 | or like this: 76 | 77 | postgres@server:~$ omnipitr/bin/sanity-check.sh 78 | Checking: 79 | - /var/lib/postgres/omnipitr/bin 80 | - /var/lib/postgres/omnipitr/lib 81 | 5 programs, 7 libraries. 82 | Warnings: 83 | - you don't have ssh program available 84 | - you don't have rsync program available 85 | - your Perl is old (we support only 5.8 or newer. OmniPITR might work, but was not tested on your version of Perl) 86 | Errors: 87 | - you don't have any of programs used to transmit WAL files (ssh, rsync) 88 | - you don't have POSIX Perl library (should be installed together with Perl) 89 | All checked. 3 warnings, 2 errors. 90 | 91 | Depending on your particular usecase even the errors don't have to be fatal - 92 | for example, if your I/I binary is in directory that is not in 93 | I<$PATH> - it will not be found by sanity-check.sh, but you can always provide 94 | paths to them via I<--rsync-binary> or I<--ssh-binary> options to I 95 | scripts. 96 | 97 | After I returned C you should be 98 | good with starting using the scripts. 99 | 100 | =head2 INITIAL SETUP 101 | 102 | To start Wal replication you will need to: 103 | 104 | =over 105 | 106 | =item 1. Start archiving wal segments: I 107 | 108 | =item 2. Make initial hot backup on master: I 109 | 110 | =item 3. Transfer the backup to slave server(s) 111 | 112 | =item 4. Uncompress backup using normal tar commands. 113 | 114 | =item 5. Start PostgreSQL recovery from hot-backup: I 115 | 116 | =back 117 | 118 | Alternativaly steps 2-4 can be replaced by usage of I tool. 119 | 120 | Afterwards you can/should monitor replication parameters with 121 | I, and that would be all. Of course you can do hot-backups on 122 | slave (with I) at any moment - just remember that doing 123 | them on slave requires more disc space - details in 124 | I. 125 | 126 | =head2 HINTS 127 | 128 | It is technically possible, and suggested to use the same logfiles for all 129 | programs running on given server. 130 | 131 | That is - you can setup archival to log to "archive.log" and backup to log to 132 | "backup.log", but you can make them both log to "omnipitr.log" - it will not 133 | break anything, and makes it simpler to manage log files. 134 | 135 | Same thing applies to state directories - these can be the same between various 136 | programs from I package. 137 | 138 | When creating any files I obeys ulimit, but doesn't enforce any 139 | additional restrictions. So, if you'd like it to create files that are readable 140 | only to owner, simply run it like this: 141 | 142 | umask 0077; omnipitr-something .... 143 | 144 | =head2 COPYRIGHT 145 | 146 | The OmniPITR project is Copyright (c) 2009-2013 OmniTI. All rights reserved. 147 | 148 | -------------------------------------------------------------------------------- /doc/intro.pod: -------------------------------------------------------------------------------- 1 | =head1 OmniPITR 2 | 3 | =head2 OVERVIEW 4 | 5 | OmniPITR is a set of scripts to ease setting up WAL replication, and making hot backups from both Master and Slave systems. 6 | 7 | This set of scripts has been written to make the operation seamless, secure and as light on resources-usage as possible. 8 | 9 | To make it portable, it was written in Perl, but with assumption that it should work on any Perl installation - i.e. no dependencies on non-base Perl modules. 10 | 11 | =head2 INSTALLATION 12 | 13 | /* Simply fetch the directory L, and you're done. More details are in I. */ 14 | 15 | =head2 USAGE 16 | 17 | There is a set of scripts in bin/ directory. All named I are scripts meant for general usage (others are not really useful unless you'll encounter some problems). 18 | 19 | For every one of them you will have documentation in I.pod> - for example documentation for I script is in I. 20 | 21 | Quick list of programs: 22 | 23 | =over 24 | 25 | =item * omnipitr-archive 26 | 27 | This should be used on master server, as I command, setup in 28 | postgresql.conf. 29 | 30 | =item * omnipitr-restore 31 | 32 | This should be used on slave server/servers, as I command in 33 | recovery.conf file. 34 | 35 | =item * omnipitr-master-backup 36 | 37 | Used to make hot backup on master DB server. 38 | 39 | =item * omnipitr-slave-backup 40 | 41 | Used to make hot backup on slave DB server. 42 | 43 | =item * omnipitr-monitor 44 | 45 | General script for I/I type of systems - monitoring, graphing. Provides a way to check for replication lag, last backup timestamp, and other metrics. 46 | 47 | =item * omnipitr-cleanup 48 | 49 | Removes obsolete wal segments from wal archive, when using streaming 50 | replication 51 | 52 | =item * omnipitr-sync 53 | 54 | Send copy of PGDATA to remote locations - even multiple at the same time. 55 | 56 | =item * omnipitr-backup-cleanup 57 | 58 | This can be called from scheduler (crontab) to enforce retention period (7 days is default) for backups and related WAL archive files. 59 | 60 | =back 61 | 62 | In most of the cases you can simply call the program you want to use with I<--help> option to get brief overview of command line options. 63 | 64 | =head2 SUPPORT 65 | 66 | Currently there is simply mailing list that you can subscribe, and post your 67 | questions/problems to. 68 | 69 | Maling list page (includes option to subscribe, and view archives) is available 70 | here: http://lists.omniti.com/mailman/listinfo/pgtreats-users/ 71 | 72 | =head2 COPYRIGHT 73 | 74 | The OmniPITR project is Copyright (c) 2009-2013 OmniTI. All rights reserved. 75 | -------------------------------------------------------------------------------- /doc/omnipitr-archive.pod: -------------------------------------------------------------------------------- 1 | =head1 OmniPITR - omnipitr-archive 2 | 3 | =head2 USAGE 4 | 5 | /some/path/omnipitr/bin/omnipitr-archive [options] "%p" 6 | 7 | Options: 8 | 9 | =over 10 | 11 | =item --data-dir (-D) 12 | 13 | Where PostgreSQL datadir is located (path) 14 | 15 | =item --force-data-dir (-f) 16 | 17 | Forces usage of given (or default) data dir even if it doesn't look like 18 | PostgreSQL data directory. 19 | 20 | =item --dst-local (-dl) 21 | 22 | Where to copy the wal segment on current server (path) (you can provide many of 23 | these). 24 | 25 | You can also specify compression per-destination. Check L 26 | section of the doc. 27 | 28 | =item --dst-remote (-dr) 29 | 30 | Where to copy the wal segment on remote server. Supported ways to transport 31 | files are rsync and rsync over ssh. Please see L for more 32 | information (you can provide many of these) 33 | 34 | You can also specify compression per-destination. Check L 35 | section of the doc. 36 | 37 | =item --dst-backup (-db) 38 | 39 | Special destination, that cannot be compressed (existence of compression prefix 40 | will cause I to raise exception, has to be absolute path 41 | (starts with /), and can be non-existing. 42 | 43 | This is special directory that shouldn't usually exist, but if it exists - it 44 | will receive copy of every wal segment rotated. 45 | 46 | The sole purpose of this option is to make I working. If 47 | you do not intend to make hot backups on master - you can safely not use it. 48 | 49 | =item --dst-pipe (-dp) 50 | 51 | Specifies path to program that should receive xlog on stdin. 52 | 53 | The program will be called multiple times (for each xlog separately). Name of 54 | the xlog will be given as first, and only, argument to the program. 55 | 56 | If the file is to be compressed (using standard compression=/path syntax) 57 | - given file name will contain compression extension. 58 | 59 | =item --temp-dir (-t) 60 | 61 | Where to create temporary files (defaults to /tmp or I<$TMPDIR> environment 62 | variable location) 63 | 64 | =item --log (-l) 65 | 66 | Name of logfile (actually template, as it supports %% L 67 | markers. Unfortunately due to the %x usage by PostgreSQL, We cannot use %% 68 | macros directly. Instead - any occurence of ^ character in log dir will be first 69 | changed to %, and later on passed to strftime. 70 | 71 | Please note that on some systems (Solaris for example) default shell treats ^ as 72 | special character, which requires you to quote the log filename (if it contains 73 | ^ character). So you'd better write it as: 74 | 75 | --log '/var/log/omnipitr-^Y-^m-^d.log' 76 | 77 | =item --state-dir (-s) 78 | 79 | Name of directory to use as state-directory to handle errors when sending wal 80 | segments to many locations. 81 | 82 | =item --pid-file 83 | 84 | Name of file to use for pidfile. If it is specified, than only one copy of 85 | I (with this pidfile) can run at the same time. 86 | 87 | Trying to run second copy of I will result in an error. 88 | 89 | =item --parallel-jobs (-PJ) 90 | 91 | Number of parallel jobs that I can spawn to deliver archives 92 | to remote destinations. 93 | 94 | =item --verbose (-v) 95 | 96 | Log verbosely what is happening. 97 | 98 | =item --not-nice (-nn) 99 | 100 | Do not use nice for compressions. 101 | 102 | =item --nice-path (-np) 103 | 104 | Full path to nice program - in case you can't set proper PATH environment 105 | variable. 106 | 107 | =item --gzip-path (-gp) 108 | 109 | Full path to gzip program - in case you can't set proper PATH environment 110 | variable. 111 | 112 | =item --bzip2-path (-bp) 113 | 114 | Full path to bzip2 program - in case you can't set proper PATH environment 115 | variable. 116 | 117 | =item --lzma-path (-lp) 118 | 119 | Full path to lzma program - in case you can't set proper PATH environment 120 | variable. 121 | 122 | =item --lz4-path (-ll) 123 | 124 | Full path to lz4 program - in case you can't set proper PATH environment 125 | variable. 126 | 127 | =item --xz-path (-xz) 128 | 129 | Full path to xz program - in case you can't set proper PATH environment 130 | variable. 131 | 132 | =item --rsync-path (-rp) 133 | 134 | Full path to rsync program - in case you can't set proper PATH environment 135 | variable. 136 | 137 | =item --version (-V) 138 | 139 | Prints version of I, and exists. 140 | 141 | =item --help (-?) 142 | 143 | Prints this manual, and exists. 144 | 145 | =item --config-file (--config / --cfg) 146 | 147 | Loads options from config file. 148 | 149 | Format of the file is very simple - each line is treated as argument with 150 | optional value. 151 | 152 | Examples: 153 | 154 | --verbose 155 | --host 127.0.0.1 156 | -h=127.0.0.1 157 | --host=127.0.0.1 158 | 159 | It is important that you don't need to quote the values - value will always 160 | be up to the end of line (trailing spaces will be removed). So if you'd 161 | want, for example, to have magic-option set to "/mnt/badly named directory", 162 | you'd need to quote it when setting from command line: 163 | 164 | /some/omnipitr/program --magic-option="/mnt/badly named directory" 165 | 166 | but not in config: 167 | 168 | --magic-option=/mnt/badly named directory 169 | 170 | Empty lines, and comment lines (starting with #) are ignored. 171 | 172 | =back 173 | 174 | =head2 DESCRIPTION 175 | 176 | Call to I should be in I GUC in 177 | I. 178 | 179 | Which options should be given depends only on installation, but generally you 180 | will need at least: 181 | 182 | =over 183 | 184 | =item * --data-dir 185 | 186 | PostgreSQL "%p" passed file path relative to I, so it is required to 187 | know it. If not provided, it is assumed to be "." ( which is perfectly in line 188 | with PostgreSQL assumptions about archive_command ). 189 | 190 | =item * --log 191 | 192 | to make sure that information is logged someplace about archiving progress 193 | 194 | =item * one of --dst-local or --dst-remote 195 | 196 | to specify where to send the WAL segments to 197 | 198 | =back 199 | 200 | If you'll specify more than 1 destination, you will also need to specify 201 | I<--state-dir> 202 | 203 | Of course you can provide many --dst-local or many --dst-remote or many mix of 204 | these. 205 | 206 | Generally omnipitr-archive will try to deliver WAL segment to all destinations, 207 | and will fail if B of them will not accept new segment. 208 | 209 | Segments will be transferred to destinations in this order: 210 | 211 | =over 212 | 213 | =item 1. All B destinations, in order provided in command line 214 | 215 | =item 2. All B destinations, in order provided in command line 216 | 217 | =back 218 | 219 | In case any destination will fail, I will save state (which 220 | destinations it delivered the file to) and return error to PostgreSQL - which 221 | will cause PostgrerSQL to call I again for the same WAL 222 | segment after some time. 223 | 224 | State directory will be cleared after every successfull file send, so it should 225 | stay small in size (expect 1 file of under 500 bytes). 226 | 227 | When constructing command line to put in I PostgreSQL GUC, 228 | please remember that while providing C<"%p" "%f"> will work, I 229 | requires only "%p" 230 | 231 | =head3 Remote destination specification 232 | 233 | I delivers WAL segments to destination using rsync program. 234 | Both direct-rsync and rsync-over-ssh are supported (it's better to use direct 235 | rsync - it uses less resources due to lack of encryption. 236 | 237 | Destination url/location should be in a format that is usable by I 238 | program. 239 | 240 | For example you can use: 241 | 242 | =over 243 | 244 | =item * rsync://user@remote_host/module/path/ 245 | 246 | =item * host:/path/ 247 | 248 | =back 249 | 250 | To allow remote delivery you need to have rsync program. In case you're using 251 | rsync over ssh, I program has also to be available. 252 | 253 | In case your rsync/ssh programs are in custom directories simply set I<$PATH> 254 | environemnt variable before starting PostgreSQL. 255 | 256 | =head2 COMPRESSION 257 | 258 | Every destination can have specified compression. To use it you should prefix 259 | destination path/url with compression type followed by '=' sign. 260 | 261 | Allowed compression types: 262 | 263 | =over 264 | 265 | =item * gzip 266 | 267 | Compresses with gzip program, used file extension is .gz 268 | 269 | =item * bzip2 270 | 271 | Compresses with bzip2 program, used file extension is .bz2 272 | 273 | =item * lzma 274 | 275 | Compresses with lzma program, used file extension is .lzma 276 | 277 | =item * lz4 278 | 279 | Compresses with lz4 program, used file extension is .lz4 280 | 281 | =item * xz 282 | 283 | Compresses with xz program, used file extension is .xz 284 | 285 | =back 286 | 287 | All compressions are done I> to make the operation as unobtrusive as 288 | possible. 289 | 290 | All programs are passed I<--stdout> option to compress without modifying source file. 291 | 292 | If you want to pass any extra arguments to compression program, you can either: 293 | 294 | =over 295 | 296 | =item * make a wrapper 297 | 298 | Write a program/script that will be named in the same way your actual 299 | compression program is named, but adding some parameters to call 300 | 301 | =item * use environment variables 302 | 303 | All of supported compression programs use environment variables: 304 | 305 | =over 306 | 307 | =item * gzip - GZIP 308 | 309 | =item * bzip2 - BZIP2 310 | 311 | =item * lzma - XZ_OPT 312 | 313 | =back 314 | 315 | For details - please consult manual to your choosen compression tool. 316 | 317 | =back 318 | 319 | =head2 EXAMPLES 320 | 321 | =head3 Minimal setup, with copying file to local directory: 322 | 323 | archive_command='/.../omnipitr-archive -D /mnt/data/ -l /var/log/omnipitr/archive.log -dl /mnt/wal_archive/ "%p"' 324 | 325 | =head3 Minimal setup, with copying file to remote directory over rsync: 326 | 327 | archive_command='/.../omnipitr-archive -D /mnt/data/ -l /var/log/omnipitr/archive.log -dr rsync://slave/postgres/wal_archive/ "%p"' 328 | 329 | =head3 2 remote, compressed destinations, 1 local, with auto rotated logfile: 330 | 331 | archive_command='/.../omnipitr-archive -D /mnt/data/ -l /var/log/omnipitr/archive-^Y-^m-^d.log -dr gzip=rsync://slave/postgres/wal_archive/ -dr bzip2=backups@backupserver:/mnt/backups/wal_archive/ -dl /mnt/wal_archive/ -s /var/lib/postgres/.omnipitr/ "%p"' 332 | 333 | =head2 IMPORTANT NOTICES 334 | 335 | =over 336 | 337 | =item * You need to provide state dir in case you have > 1 destination. This is 338 | crucial, as if you're using "-dr ... -db" - your number of destinations changes 339 | from 1 to 2 (depending on whether -db directory exists) - i.e. not all runs of 340 | I will catch lack of state-dir as error! 341 | 342 | =item * It is generally good to provide 2 destination - one used by slave system 343 | to keep warm standby, and another to be handy in case (for example) backup files 344 | get corrupted. This second set of xlogs should be cleared with cronjob that 345 | removes too old files. As for what is too old - 2x your backup schedule. I.e. if 346 | you're doing daily hot backups - you should keep 2 days worth of backup xlogs. 347 | 348 | =back 349 | 350 | =head2 COPYRIGHT 351 | 352 | The OmniPITR project is Copyright (c) 2009-2013 OmniTI. All rights reserved. 353 | 354 | -------------------------------------------------------------------------------- /doc/omnipitr-backup-cleanup.pod: -------------------------------------------------------------------------------- 1 | =head1 OmniPITR - omnipitr-backup-cleanup 2 | 3 | =head2 USAGE 4 | 5 | /some/path/omnipitr/bin/omnipitr-backup-cleanup [options] 6 | 7 | Options: 8 | 9 | =over 10 | 11 | =item --archive (-a) 12 | 13 | Where archived wal segments are kept. 14 | 15 | Check L for more details. 16 | 17 | =item --backup-dir (-b) 18 | 19 | Location of backup directory. 20 | 21 | Check L for more details. 22 | 23 | =item --filename-template (-f) 24 | 25 | Template for naming output files. Should be the same as was used for making 26 | the backup by L or L. 27 | 28 | Defaults to the same value that these scripts use, so if you didn't change it 29 | for making backups, you don't need to specify it in here. 30 | 31 | =item --keep-days (-k) 32 | 33 | How long to keep backups. Defaults to 7 days. 34 | 35 | =item --truncate (-t) 36 | 37 | If the file is larger than --truncate, it will be removed in steps. Each step 38 | will remove --truncate bytes from the end of file, until the file will be 39 | smaller than --truncate. At this time the remaining part of file will be 40 | removed normally. 41 | 42 | If the value is 0 (default) - then regardless of size, file will be removed in 43 | one step. 44 | 45 | This is to prevent huge IO on certain filesystems (ext3 on Linux for example) 46 | when removing huge files. 47 | 48 | =item --sleep (-s) 49 | 50 | How many miliseconds to wait between truncate runs. 51 | 52 | When the truncating method is not used (--truncate 0, or file is too small) 53 | - there is no wait. But if truncating happens, after each truncate sleep of 54 | given number of miliseconds is used to make the removal take longer, but with 55 | lower I/O impact. 56 | 57 | =item --log (-l) 58 | 59 | Name of logfile (actually template, as it supports %% L 60 | markers. Unfortunately due to the %x usage by PostgreSQL, We cannot use %% 61 | macros directly. Instead - any occurence of ^ character in log dir will be first 62 | changed to %, and later on passed to strftime. 63 | 64 | Please note that on some systems (Solaris for example) default shell treats ^ as 65 | special character, which requires you to quote the log filename (if it contains 66 | ^ character). So you'd better write it as: 67 | 68 | --log '/var/log/omnipitr-^Y-^m-^d.log' 69 | 70 | =item --dry-run (-d) 71 | 72 | If this option is provided, instead of actually removing files 73 | I will only log information on what would be deleted. 74 | 75 | =item --verbose (-v) 76 | 77 | Log verbosely what is happening. 78 | 79 | =item --version (-V) 80 | 81 | Prints version of I, and exists. 82 | 83 | =item --help (-?) 84 | 85 | Prints this manual, and exists. 86 | 87 | =item --config-file (--config / --cfg) 88 | 89 | Loads options from config file. 90 | 91 | Format of the file is very simple - each line is treated as argument with 92 | optional value. 93 | 94 | Examples: 95 | 96 | --verbose 97 | --archive /mnt/wal_archive 98 | --backup-dir /mnt/backups 99 | --log /tmp/log 100 | 101 | It is important that you don't need to quote the values - value will always 102 | be up to the end of line (trailing spaces will be removed). So if you'd 103 | want, for example, to have magic-option set to "/mnt/badly named directory", 104 | you'd need to quote it when setting from command line: 105 | 106 | /some/omnipitr/program --magic-option="/mnt/badly named directory" 107 | 108 | but not in config: 109 | 110 | --magic-option=/mnt/badly named directory 111 | 112 | Empty lines, and comment lines (starting with #) are ignored. 113 | 114 | =back 115 | 116 | =head2 DESCRIPTION 117 | 118 | Call to I should (generally) be in some kind of 119 | scheduler - like crontab. 120 | 121 | Set of options that you prefer depends on your situation, but at the very 122 | least you need three options: 123 | 124 | =over 125 | 126 | =item * --backup-dir= - where are your backups. Please note that you have to 127 | have backups made with OmniPITR v.1.3.0 at least, as previous versions didn't 128 | have "meta" files. 129 | 130 | =item * --archive-dir= - where are xlog files. These should be xlogs generated 131 | by the same system that the backups are for. 132 | 133 | =item * --log= - where to store logs about removal. You can use "-" as log 134 | value, it will redirect logs to stdout. 135 | 136 | =back 137 | 138 | =head3 Storage dir specification 139 | 140 | Directory is, in simplest situation, just a path. Usually, though, you will 141 | have some kind of compression on one or both of the directories (backups, 142 | archives). 143 | 144 | In case of compression, you use the same syntax in every other omnipitr-* 145 | program - that is you prefix path with compression type and "=" sign. 146 | 147 | Allowed compression types: 148 | 149 | =over 150 | 151 | =item * gzip 152 | 153 | Used file extension is .gz 154 | 155 | =item * bzip2 156 | 157 | Used file extension is .bz2 158 | 159 | =item * lzma 160 | 161 | Used file extension is .lzma 162 | 163 | =item * lz4 164 | 165 | Used file extension is .lz4 166 | 167 | =item * xz 168 | 169 | Used file extension is .xz 170 | 171 | =back 172 | 173 | =head2 EXAMPLES 174 | 175 | =head3 Minimal setup: 176 | 177 | /path/to/omnipitr-backup-cleanup -a /mnt/wal_archive -b /mnt/backups -l - 178 | 179 | =head3 Compressed archive and backup dir, non default number of days to keep 180 | backups, and store logs 181 | 182 | /path/to/omnipitr-backup-cleanup -a gzip=/mnt/wal_archive -b gzip=/mnt/backups -l /var/log/omnipitr-backup-cleanup.log -v -k 3 183 | 184 | =head2 COPYRIGHT 185 | 186 | The OmniPITR project is Copyright (c) 2009-2013 OmniTI. All rights reserved. 187 | 188 | -------------------------------------------------------------------------------- /doc/omnipitr-cleanup.pod: -------------------------------------------------------------------------------- 1 | =head1 OmniPITR - omnipitr-cleanup 2 | 3 | =head2 USAGE 4 | 5 | /some/path/omnipitr/bin/omnipitr-cleanup [options] %r 6 | 7 | Options: 8 | 9 | =over 10 | 11 | =item --archive (-a) 12 | 13 | Where archived wal segments are kept. 14 | 15 | Check L for more details. 16 | 17 | =item --log (-l) 18 | 19 | Name of logfile (actually template, as it supports %% L 20 | markers. Unfortunately due to the %x usage by PostgreSQL, We cannot use %% 21 | macros directly. Instead - any occurence of ^ character in log dir will be first 22 | changed to %, and later on passed to strftime. 23 | 24 | Please note that on some systems (Solaris for example) default shell treats ^ as 25 | special character, which requires you to quote the log filename (if it contains 26 | ^ character). So you'd better write it as: 27 | 28 | --log '/var/log/omnipitr-^Y-^m-^d.log' 29 | 30 | =item --pid-file 31 | 32 | Name of file to use for pidfile. If it is specified, than only one copy of 33 | I (with this pidfile) can run at the same time. 34 | 35 | Trying to run second copy of I will result in an error. 36 | 37 | =item --verbose (-v) 38 | 39 | Log verbosely what is happening. 40 | 41 | =item --removal-pause-trigger (-p) 42 | 43 | Path to file, which, if exists, causes I to not do any removal 44 | - for example when running I. 45 | 46 | =item --version (-V) 47 | 48 | Prints version of I, and exists. 49 | 50 | =item --help (-?) 51 | 52 | Prints this manual, and exists. 53 | 54 | =item --config-file (--config / --cfg) 55 | 56 | Loads options from config file. 57 | 58 | Format of the file is very simple - each line is treated as argument with 59 | optional value. 60 | 61 | Examples: 62 | 63 | --verbose 64 | --host 127.0.0.1 65 | -h=127.0.0.1 66 | --host=127.0.0.1 67 | 68 | It is important that you don't need to quote the values - value will always 69 | be up to the end of line (trailing spaces will be removed). So if you'd 70 | want, for example, to have magic-option set to "/mnt/badly named directory", 71 | you'd need to quote it when setting from command line: 72 | 73 | /some/omnipitr/program --magic-option="/mnt/badly named directory" 74 | 75 | but not in config: 76 | 77 | --magic-option=/mnt/badly named directory 78 | 79 | Empty lines, and comment lines (starting with #) are ignored. 80 | 81 | =back 82 | 83 | =head2 DESCRIPTION 84 | 85 | Call to I should be in I variable in 86 | I. 87 | 88 | Which options should be given depends only on installation, but generally you 89 | will need: 90 | 91 | =over 92 | 93 | =item * --archive 94 | 95 | =item * --log 96 | 97 | =back 98 | 99 | And of course the %r at the end. 100 | 101 | This script is used only in cases of streaming replication, as in case of 102 | wal-file based replication L can remove obsolete wal files. 103 | 104 | =head3 Archive specification 105 | 106 | If the wal segments are compressed you have to prefix archive path with 107 | compression type followed by '=' sign. 108 | 109 | Allowed compression types: 110 | 111 | =over 112 | 113 | =item * gzip 114 | 115 | Used file extension is .gz 116 | 117 | =item * bzip2 118 | 119 | Used file extension is .bz2 120 | 121 | =item * lzma 122 | 123 | Used file extension is .lzma 124 | 125 | =item * lz4 126 | 127 | Used file extension is .lz4 128 | 129 | =item * xz 130 | 131 | Used file extension is .xz 132 | 133 | =back 134 | 135 | =head2 EXAMPLES 136 | 137 | =head3 Minimal setup: 138 | 139 | archive_cleanup_command='/.../omnipitr-cleanup -l /var/log/omnipitr/cleanup.log -a /mnt/wal_restore/ %r' 140 | 141 | =head3 Minimal setup, for gzip-compressed archives: 142 | 143 | archive_cleanup_command='/.../omnipitr-cleanup -l /var/log/omnipitr/cleanup.log -a gzip=/mnt/wal_restore/ %r' 144 | 145 | =head2 COPYRIGHT 146 | 147 | The OmniPITR project is Copyright (c) 2009-2013 OmniTI. All rights reserved. 148 | 149 | -------------------------------------------------------------------------------- /doc/omnipitr-monitor.pod: -------------------------------------------------------------------------------- 1 | =head1 OmniPITR - omnipitr-monitor 2 | 3 | =head2 USAGE 4 | 5 | /some/path/omnipitr/bin/omnipitr-monitor [options] 6 | 7 | Where: 8 | 9 | =over 10 | 11 | =item --log (-l) Points to logfile created by I tool that should be monitored. 12 | 13 | It can contain strftime-type marks - just like --log argument to other 14 | I programs.. Unfortunately due to the %x usage by PostgreSQL, We cannot use %% 15 | macros directly. Instead - any occurence of ^ character in log dir will be first 16 | changed to %, and later on passed to strftime. 17 | 18 | Please note that on some systems (Solaris for example) default shell treats ^ as 19 | special character, which requires you to quote the log filename (if it contains 20 | ^ character). So you'd better write it as: 21 | 22 | --log '/var/log/omnipitr-^Y-^m-^d.log' 23 | 24 | You can provide multiple --log options, in case your system is configured in 25 | such a way that every tool logs to its own logs. 26 | 27 | =item --check (-c) contains name of check that should be performed. 28 | 29 | For list of checks, please read L section. 30 | 31 | =item --state-dir (-s) is a name of directory that will be used to store state 32 | between calls to checks 33 | 34 | =item --verbose (-v) shows information about reading lines from log files. 35 | 36 | =item --psql-path (-pp) path to psql - used when getting data from 37 | database 38 | 39 | =item --temp-dir (-t) - Where to create temporary files (defaults to /tmp 40 | or I<$TMPDIR> environment variable location) 41 | 42 | =item options depend on which check being performed. 43 | 44 | Most checks don't have any options. List of options for check is supplied in 45 | L section. 46 | 47 | Any check can have it's own set of options, but there are four reserved options 48 | for database connection settings: 49 | 50 | =over 51 | 52 | =item --database (-d) 53 | 54 | Which database to connect to to issue required SQL queries. Defaults to 55 | template1. 56 | 57 | =item --host (-h) 58 | 59 | Which host to connect to when connecting to database to backup. Shouldn't really 60 | be changed in 99% of cases. Defaults to empty string - i.e. use UNIX sockets. 61 | 62 | =item --port (-p) 63 | 64 | Which port to connect to when connecting to database. Defaults to 5432. 65 | 66 | =item --username (-U) 67 | 68 | What username to use when connecting to database. Defaults to postgres. 69 | 70 | =item --version (-V) 71 | 72 | Prints version of I, and exists. 73 | 74 | =item --help (-?) 75 | 76 | Prints this manual, and exists. 77 | 78 | =back 79 | 80 | That is - if any check requires connection to database - details of connections 81 | will be expected to be passed using these options. 82 | 83 | And if the check doesn't need database access - it is guaranteed that these 84 | options (neither long nor short forms) will not be reused for other purposes. 85 | 86 | =back 87 | 88 | =head2 DESCRIPTION 89 | 90 | This script simply reads log files from other I tools, and reports 91 | values useful for monitoring in Nagios, Cacti or other tools. 92 | 93 | =head2 CHECKS 94 | 95 | =head3 last-archive-age 96 | 97 | When was the last WAL segment archived. Returns value in seconds being interval 98 | between "now" and the moment when last archive happened. 99 | 100 | Requires path to log from I 101 | 102 | =head3 current-archive-time 103 | 104 | This check returns 0 if there is currently no archive happening. If there is 105 | one, it will return number of seconds that passed since start of archiving. 106 | 107 | This might be used, for example, to raise alert if single archive takes more 108 | than x seconds. 109 | 110 | Requires path to log from I 111 | 112 | =head3 archive-queue 113 | 114 | Shows number of xlog files that are waiting to be archived. This, in normal 115 | circumstances, shouldn't be larger than 1, and usually should be 0. 116 | 117 | This check will need to connect to master database, so you might need to 118 | provide database connection parameters and/or psql path. 119 | 120 | Information about these options is in USAGE section at the beginning of this 121 | manual. 122 | 123 | =head3 last-restore-age 124 | 125 | When was the last WAL segment restored. Returns value in seconds being interval 126 | between "now" and the moment when last restore happened. 127 | 128 | Requires path to log from I 129 | 130 | =head3 errors 131 | 132 | Lists all errors that happened from last call to errors check in given logfile. 133 | 134 | Can have option: 135 | 136 | --from 137 | 138 | Where value of from is treated as: 139 | 140 | =over 141 | 142 | =item * Date time in format: %Y-%m-%d %H:%M:%S (for example: 2009-12-24 143 | 15:45:32) 144 | 145 | =item * interval, in seconds (for example: 300 - means check last 5 minutes) 146 | 147 | =back 148 | 149 | Works with logfiles of all types. 150 | 151 | =head3 last-backup-age 152 | 153 | Returns when last backup was finished, in seconds that passed since. 154 | 155 | Requires path to log from I or I 156 | 157 | =head2 EXAMPLES 158 | 159 | =head3 Getting age of last archive: 160 | 161 | .../omnipitr-monitor -l /var/log/omnipitr/archive-^Y-^m-^d.log -c last-archive-age -s /var/lib/omnipitr 162 | 163 | =head3 Getting age of last wal restore: 164 | 165 | .../omnipitr-monitor -l /var/log/omnipitr/restore-^Y-^m-^d.log -c last-restore-age -s /var/lib/omnipitr 166 | 167 | =head3 Getting errors for reporting: 168 | 169 | .../omnipitr-monitor -l /var/log/omnipitr/archive-^Y-^m-^d.log -c errors -s /var/lib/omnipitr 170 | 171 | =head3 Getting errors for reporting, but checking always only last 10 minutes of 172 | logs: 173 | 174 | .../omnipitr-monitor -l /var/log/omnipitr/archive-^Y-^m-^d.log -c errors -s /var/lib/omnipitr --from=600 175 | 176 | =head2 COPYRIGHT 177 | 178 | The OmniPITR project is Copyright (c) 2009-2013 OmniTI. All rights reserved. 179 | -------------------------------------------------------------------------------- /doc/omnipitr-synch.pod: -------------------------------------------------------------------------------- 1 | =head1 OmniPITR - omnipitr-synch 2 | 3 | =head2 USAGE 4 | 5 | /some/path/omnipitr/bin/omnipitr-synch [options] 6 | 7 | Options: 8 | 9 | =over 10 | 11 | =item --data-dir (-D) 12 | 13 | Where PostgreSQL datadir is located (path). If skipped, it will be taken from 14 | PostgreSQL settings. 15 | 16 | =item --database (-d) 17 | 18 | Which database to connect to to issue required SQL queries. Defaults to 19 | postgres. 20 | 21 | =item --host (-h) 22 | 23 | Which host to connect to when connecting to database to run pg_backup_* 24 | functions. Shouldn't really be changed in 99% of cases. Defaults to empty string 25 | - i.e. use UNIX sockets. 26 | 27 | =item --port (-p) 28 | 29 | Which port to connect to when connecting to database. Defaults to 5432. 30 | 31 | =item --username (-U) 32 | 33 | What username to use when connecting to database. Defaults to postgres. 34 | 35 | =item --output (-o) 36 | 37 | Where to copy DATADIR - syntax should be one of: 38 | 39 | =over 40 | 41 | =item * host:/absolute/path/to/new/place 42 | 43 | =item * user@host:/absolute/path/to/new/place 44 | 45 | =back 46 | 47 | If you need to use non-standard port number, you'll have to use ~/.ssh/config 48 | file to define it for given host - this is due to problem with passing ssh port 49 | number for rsync-over-ssh. 50 | 51 | You can have multiple --output options to send data to multiple new machines 52 | with single cost of preparing data on source. 53 | 54 | Please check L section for more details. 55 | 56 | =item --compress (-c) 57 | 58 | Whether to use compression - value of parameter is name of compression program 59 | (with path if necessary). 60 | 61 | It is also used (as a flag) when using --rsync option. 62 | 63 | =item --rsync (-r) 64 | 65 | If used, I will use rsync instead of tar to transfer data (it 66 | will still be rsync over ssh). The benefit of using rsync is that it can send 67 | just the differences so it might be faster. 68 | 69 | The drawback is that (depending on situation) it can be more taxing for source 70 | server hardware (IO/CPU). 71 | 72 | Rsync will be used with data compression (-z) if --compress was given. 73 | 74 | =item --map (-m) 75 | 76 | When transmitting tablespaces, you might want to change output path for 77 | tablespace files - this is used then. 78 | 79 | Please check L section for more details. 80 | 81 | =item --log (-l) 82 | 83 | Name of logfile (actually template, as it supports %% L 84 | markers. Unfortunately due to the %x usage by PostgreSQL, We cannot use %% 85 | macros directly. Instead - any occurence of ^ character in log dir will be first 86 | changed to %, and later on passed to strftime. 87 | 88 | Please note that on some systems (Solaris for example) default shell treats ^ as 89 | special character, which requires you to quote the log filename (if it contains 90 | ^ character). So you'd better write it as: 91 | 92 | --log '/var/log/omnipitr-^Y-^m-^d.log' 93 | 94 | =item --pid-file 95 | 96 | Name of file to use for pidfile. If it is specified, than only one copy of 97 | I (with this pidfile) can run at the same time. 98 | 99 | Trying to run second copy of I will result in an error. 100 | 101 | =item --verbose (-v) 102 | 103 | Log verbosely what is happening. 104 | 105 | =item --tee-path (-ep) 106 | 107 | Full path to tee program - in case you can't set proper PATH environment 108 | variable. 109 | 110 | =item --tar-path (-tp) 111 | 112 | Full path to tar program - in case you can't set proper PATH environment 113 | variable. 114 | 115 | =item --psql-path (-pp) 116 | 117 | Full path to psql program - in case you can't set proper PATH environment 118 | variable. 119 | 120 | =item --ssh-path (-sp) 121 | 122 | Full path to ssh program - in case you can't set proper PATH environment 123 | variable. 124 | 125 | =item --rsync-path (-rp) 126 | 127 | Full path to rsync program - in case you can't set proper PATH environment 128 | variable. 129 | 130 | =item --remote-tar-path (-rtp) 131 | 132 | Full path to tar program on output side. 133 | 134 | =item --remote-rsync-path (-rsp) 135 | 136 | Full path to rsync program on output side. 137 | 138 | =item --remote-compressor-path (-rcp) 139 | 140 | Full path to compression program that will be used to decompress data on remote 141 | machine (if local data will be compressed). Defaults to whatever was passed to 142 | --compress. 143 | 144 | =item --remote-rm-path (-rrp) 145 | 146 | Full path to rm program on output server - it will be used to clear output 147 | directories before uncompressing new data. 148 | 149 | =item --automatic (-a) 150 | 151 | Run without confirmations. Without this option, I will first 152 | gather data, apply output mappings, list all details, and then wait for use 153 | confirmation. 154 | 155 | =item --temp-dir (-t) 156 | 157 | Where to create temporary files (defaults to /tmp or I<$TMPDIR> environment 158 | variable location) 159 | 160 | =item --shell-path (-sh) 161 | 162 | Full path to shell to be used when calling compression/archiving/checksumming. 163 | 164 | It is important becaus the shell needs to support >( ... ) constructions. 165 | 166 | One of the shells that do support it is bash, and this is the default value for 167 | --shell-path. You can substitute different shell if you're sure it supports 168 | mentioned construction. 169 | 170 | =item --version (-V) 171 | 172 | Prints version of I, and exists. 173 | 174 | =item --help (-?) 175 | 176 | Prints this manual, and exists. 177 | 178 | =item --config-file (--config / --cfg) 179 | 180 | Loads options from config file. 181 | 182 | Format of the file is very simple - each line is treated as argument with 183 | optional value. 184 | 185 | Examples: 186 | 187 | --verbose 188 | --host 127.0.0.1 189 | -h=127.0.0.1 190 | --host=127.0.0.1 191 | 192 | It is important that you don't need to quote the values - value will always 193 | be up to the end of line (trailing spaces will be removed). So if you'd 194 | want, for example, to have magic-option set to "/mnt/badly named directory", 195 | you'd need to quote it when setting from command line: 196 | 197 | /some/omnipitr/program --magic-option="/mnt/badly named directory" 198 | 199 | but not in config: 200 | 201 | --magic-option=/mnt/badly named directory 202 | 203 | Empty lines, and comment lines (starting with #) are ignored. 204 | 205 | =back 206 | 207 | =head2 DESCRIPTION 208 | 209 | This program is meant to be ran by hand to setup new slave system for 210 | replication. 211 | 212 | It transfers PGDATA of PostgreSQL instance to new server, together with all 213 | necessary tablespaces, but skipping irrelevant files. 214 | 215 | The transfer can be made when running source instance, thanks to calls to 216 | pg_start_backup() and pg_stop_backup() PostgreSQL functions. 217 | 218 | Which options should be given depends only on installation, but generally you 219 | will need at least: 220 | 221 | =over 222 | 223 | =item * --output 224 | 225 | to specify where to send the data dir. 226 | 227 | I delivers files to destination using rsync program. Both 228 | direct-rsync and rsync-over-ssh are supported (it's better to use direct rsync 229 | - it uses less resources due to lack of encryption. 230 | 231 | To allow delivery you need to have ssh program. 232 | 233 | =item * --log 234 | 235 | to make sure that information is logged someplace about progress. Unlike other 236 | omnipitr-* programs, when you'll don't provide -l, I will output 237 | to STDOUT. This was done because unlike other omnipitr programs, this one is 238 | meant to be ran by hand, and not from cronjobs (although it's possible to do). 239 | 240 | =back 241 | 242 | =head2 OUTPUT 243 | 244 | If I detects additional tablespaces, they will be also 245 | sent to destination (--output) server. 246 | 247 | Full path to tablespaces will be te same as on source server, so for example, 248 | assuming you have tablespaces located in: 249 | 250 | =over 251 | 252 | =item * /ts1 253 | 254 | =item * /mnt/ssd/ts2 255 | 256 | =item * /var/ts3 257 | 258 | =back 259 | 260 | and PGDATA in /var/lib/pgsql/data, and you'll call I with: 261 | 262 | --output remote:/a/b/c 263 | 264 | Then: 265 | 266 | =over 267 | 268 | =item * content of /var/lib/pgsql/data (pgdata) will be delivered to remote:/a/b/c 269 | 270 | =item * tablespace from /ts1 will be delivered to remote:/ts1 271 | 272 | =item * tablespace from /mnt/ssd/ts2 will be delivered to remote:/mnt/ssd/ts2 273 | 274 | =item * tablespace from /var/ts3 will be delivered to remote:/var/ts3 275 | 276 | =back 277 | 278 | Since it might not always be desirable, I supports the notion 279 | of maps. These are used to change tablespace output paths (not data dir, just 280 | tablespace paths). 281 | 282 | --map option has following syntax: 283 | 284 | --map from:to 285 | 286 | for example: 287 | 288 | --map /ts1:remote:/x/y/z 289 | 290 | Above means that tablespace located in /ts1 directory locally will be delivered 291 | to directory /x/y/z on remote machine. 292 | 293 | Map syntax assumes the given paths are prefixes. So, for example adding: 294 | 295 | --map /:remote:/root/ --output remote:/a/b/c 296 | 297 | would (in our example situation described above): 298 | 299 | =over 300 | 301 | =item * deliver content of /var/lib/pgsql/data (pgdata) remote:/a/b/c 302 | 303 | =item * deliver tablespace from /ts1 to remote:/root/ts1 304 | 305 | =item * deliver tablespace from /mnt/ssd/ts2 to remote:/root/mnt/ssd/ts2 306 | 307 | =item * deliver tablespace from /var/ts3 to remote:/root/var/ts3 308 | 309 | =back 310 | 311 | If given tablespace is not matching any map rules, it will be delivered 312 | normally, like if no maps were provided. 313 | 314 | Please note and understand that changing paths to PostgreSQL instance is not 315 | supported - the mapping process is only meant as a way to simplify transfer of 316 | data in specific cases where dba needs this modification to fit the data on 317 | disk. 318 | 319 | Using it will not change PostgreSQL internal paths to use tablespaces in 320 | different locations. 321 | 322 | =head2 EXAMPLES 323 | 324 | =head3 Simplistic run, get data dir from Pg itself: 325 | 326 | /.../omnipitr-synch -o remote:/pgdata 327 | 328 | =head3 Automatic run, with tablespaces mapped to different delivery and logging 329 | 330 | /.../omnipitr-synch -D /mnt/data/ -l "/var/log/omnipitr/synch-^Y-^m-^d.log" -a -o remote:/pgdata -m /:remote:/tablespaces/ 331 | 332 | =head3 Automatic run, with 2 destinations and compression: 333 | 334 | /.../omnipitr-synch -a -o slave1:/pgdata -o slave2:/pgdata -c gzip 335 | 336 | =head2 IMPORTANT NOTICES 337 | 338 | =over 339 | 340 | =item * This program is dangerous - it will delete data on the destination 341 | server - you should be sure what you want to do not to delete important files. 342 | 343 | =back 344 | 345 | =head2 COPYRIGHT 346 | 347 | The OmniPITR project is Copyright (c) 2011-2012 OmniTI. All rights reserved. 348 | 349 | -------------------------------------------------------------------------------- /doc/todo.pod: -------------------------------------------------------------------------------- 1 | 2 | =head1 OmniPITR 3 | 4 | =head2 TODO 5 | 6 | Even while developing first version of the OmniPITR, we know there are things 7 | that could be improved. But first things first - we have to finish development 8 | of base functionality before starting work on new stuff. 9 | 10 | Here are the already known about missing features: 11 | 12 | =over 13 | 14 | =item * Add a way to specify "primary" destination for wal segments. 15 | 16 | This would be useful if you have WAL-slave in the same network, and additional 17 | slave, on a network link that can be down. Problems with non-primary 18 | destinations wouldn't stop replication to primary destinations. 19 | 20 | =item * Make it possible to provide remote source in omnipitr-restore and 21 | omnipitr-backup-slave (remote, via http/ftp) 22 | 23 | =item * Make it possible to provide multiple sources for omnipitr-restore 24 | 25 | This is for migration purposes - for example, when you're switching your 26 | walarchive from compressed to not-compressed. 27 | 28 | =item * Add support for using ZFS/LVM snapshops. 29 | 30 | This has the benefit of keeping required xlogs to a minimum for backups. 31 | 32 | =item * make omnipitr-backup-* scripts verify that they are being run as user 33 | running PostgreSQL 34 | 35 | =back 36 | 37 | =head2 DONE 38 | 39 | =over 40 | 41 | =item * [1.3.3] Add support for xz and lz4 compressions (similar to lzma) 42 | 43 | =item * [1.2.0] Support sending to cloud storage such as Amazon S3 - it's done 44 | via --dst-pipe option. 45 | 46 | =item * [1.2.0] make omnipitr-restore don't care about walarchive writability if -r 47 | option is not given 48 | 49 | =item * [1.2.0] Add option to skip archiving of xlogs to omnipitr-backup-* 50 | 51 | This is assuming the user is keeping their own xlog archive and will limit 52 | the ability to restore a standalone database. 53 | 54 | =item * [1.2.0] Ability to use %r in omnipitr-restore instead of relying on 55 | pg_controldata 56 | 57 | While definitely good thing, we will still keep code to use pg_controldata as %r 58 | is not available in 8.2. 59 | 60 | =item * [1.1.0] Provide support for --help option 61 | 62 | Well, it can be helpful to avoid having to type another command to get to docs 63 | 64 | =item * [1.1.0] Add support for config file 65 | 66 | This has the benefit over command line options that it lets you change the 67 | options without Pg restart (in omnipitr-restore case at least). 68 | 69 | =item * [1.1.0] Add --version option to all programs. 70 | 71 | =item * [1.1.0] Make temp directories inside given temp-dir, based on pid and/or random 72 | values - to allow easy running (for example) 2 slaves on the same machine. 73 | 74 | =item * [0.7.0] Have omnipitr-backup-* look at .backup file to determine exactly what 75 | xlogs it needs to keep for backup integrity. 76 | 77 | =item * [0.6.0] Deliver to multiple destinations in parallel 78 | 79 | When delivering wal segments to slave server, and to backup destination - we can 80 | reduce the time it takes, by delivering it in parallel. 81 | 82 | =item * [0.4.0] Add specialized program to serve as archive_cleanup_command - with 83 | support for compressed walarchive, and pausing removal 84 | 85 | =back 86 | 87 | =head2 COPYRIGHT 88 | 89 | The OmniPITR project is Copyright (c) 2009-2013 OmniTI. All rights reserved. 90 | -------------------------------------------------------------------------------- /lib/OmniPITR/Log.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Log; 2 | use strict; 3 | use warnings; 4 | use English qw( -no_match_vars ); 5 | use Carp; 6 | use File::Basename; 7 | use File::Path; 8 | use Data::Dumper; 9 | use POSIX qw(strftime floor); 10 | use IO::File; 11 | 12 | our $VERSION = '2.0.0'; 13 | 14 | BEGIN { 15 | eval { use Time::HiRes qw( time ); }; 16 | } 17 | 18 | =head1 new() 19 | 20 | Constructor for logger class. 21 | 22 | Takes one argument: template (using %*, strftime variables). 23 | 24 | This argument can also be reference to File Handle to force log output to given stream. This can be used for example like: 25 | 26 | my $logger = OmniPITR::Log->new( \*STDOUT ); 27 | 28 | =cut 29 | 30 | sub new { 31 | my $class = shift; 32 | my ( $filename_template ) = @_; 33 | croak( 'Logfile name template was not provided!' ) unless $filename_template; 34 | 35 | my $self = bless {}, $class; 36 | 37 | if ( ref $filename_template ) { 38 | 39 | # It's forced filehandle 40 | $self->{ 'forced_fh' } = $filename_template; 41 | } 42 | else { 43 | $self->{ 'template' } = $filename_template; 44 | } 45 | $self->{ 'program' } = basename( $PROGRAM_NAME ); 46 | $self->{ 'current_log_ts' } = 0; 47 | $self->{ 'current_log_fn' } = ''; 48 | 49 | return $self; 50 | } 51 | 52 | =head1 _log() 53 | 54 | Internal function, shouldn't be called from client code. 55 | 56 | Gets loglevel (assumed to be string), format, and list of values to 57 | fill-in in the format, using standard sprintf semantics. 58 | 59 | Each line (even in multiline log messages) is prefixed with 60 | metainformation (timestamp, pid, program name). 61 | 62 | In case reference is passed as one of args - it is dumped using 63 | Data::Dumper. 64 | 65 | Thanks to this this: 66 | 67 | $object->_log('loglevel', '%s', $object); 68 | 69 | Will print dump of $object and not stuff like 'HASH(0xdeadbeef)'. 70 | 71 | For client-code open methods check L, L and L. 72 | 73 | =cut 74 | 75 | sub _log { 76 | my $self = shift; 77 | my ( $level, $format, @args ) = @_; 78 | 79 | my $log_line_prefix = $self->_get_log_line_prefix(); 80 | my $fh = $self->_get_log_fh(); 81 | 82 | @args = map { ref $_ ? Dumper( $_ ) : $_ } @args; 83 | 84 | my $message = sprintf $format, @args; 85 | $message =~ s/\s*\z//; 86 | 87 | for my $line ( split /\r?\n/, $message ) { 88 | printf $fh '%s : %s : %s%s', $log_line_prefix, $level, $line, "\n"; 89 | } 90 | 91 | $fh->flush(); 92 | $fh->sync(); 93 | 94 | return; 95 | } 96 | 97 | =head1 log() 98 | 99 | Client code facing method, which calls internal L<_log()> method, giving 100 | 'LOG' as loglevel, and passing rest of arguments without modifications. 101 | 102 | Example: 103 | 104 | $logger->log( 'i = %u', $i ); 105 | 106 | =cut 107 | 108 | sub log { 109 | my $self = shift; 110 | return $self->_log( 'LOG', @_ ); 111 | } 112 | 113 | =head1 error() 114 | 115 | Client code facing method, which calls internal L<_log()> method, giving 116 | 'ERROR' as loglevel, and passing rest of arguments without 117 | modifications. 118 | 119 | Example: 120 | 121 | $logger->error( 'File creation failed: %s', $OS_ERROR ); 122 | 123 | =cut 124 | 125 | sub error { 126 | my $self = shift; 127 | return $self->_log( 'ERROR', @_ ); 128 | } 129 | 130 | =head1 fatal() 131 | 132 | Client code facing method, which calls internal L<_log()> method, giving 133 | 'FATAL' as loglevel, and passing rest of arguments without 134 | modifications. 135 | 136 | Additionally, after logging the message, it exits main program, setting 137 | error status 1. 138 | 139 | Example: 140 | 141 | $logger->fatal( 'Called from user with uid = %u, and not 0!', $user_uid ); 142 | 143 | =cut 144 | 145 | sub fatal { 146 | my $self = shift; 147 | $self->_log( 'FATAL', @_ ); 148 | exit( 1 ); 149 | } 150 | 151 | =head1 time_start() 152 | 153 | Starts timer. 154 | 155 | Should be used together with time_finish, for example like this: 156 | 157 | $logger->time_start( 'zipping' ); 158 | $zip->run(); 159 | $logger->time_finish( 'zipping' ); 160 | 161 | Arguments to time_start and time_finish should be the same to allow 162 | matching of events. 163 | 164 | =cut 165 | 166 | sub time_start { 167 | my $self = shift; 168 | my $comment = shift; 169 | $self->{ 'timers' }->{ $comment } = time(); 170 | return; 171 | } 172 | 173 | =head1 time_finish() 174 | 175 | Finished calculation of time for given block of code. 176 | 177 | Calling: 178 | 179 | $logger->time_finish( 'Compressing with gzip' ); 180 | 181 | Will log line like this: 182 | 183 | 2010-04-09 00:08:35.148118 +0200 : 19713 : omnipitr-archive : LOG : Timer [Compressing with gzip] took: 0.351s 184 | 185 | Assuming related time_start() was called 0.351s earlier. 186 | 187 | =cut 188 | 189 | sub time_finish { 190 | my $self = shift; 191 | my $comment = shift; 192 | my $start = delete $self->{ 'timers' }->{ $comment }; 193 | $self->log( 'Timer [%s] took: %.3fs', $comment, time() - ( $start || 0 ) ); 194 | return; 195 | } 196 | 197 | =head1 _get_log_line_prefix() 198 | 199 | Internal method generating line prefix, which is prepended to every 200 | logged line of text. 201 | 202 | Prefix contains ( " : " separated ): 203 | 204 | =over 205 | 206 | =item * Timestamp, with microsecond precision 207 | 208 | =item * Process ID (PID) of logging program 209 | 210 | =item * Name of program that logged the message 211 | 212 | =back 213 | 214 | =cut 215 | 216 | sub _get_log_line_prefix { 217 | my $self = shift; 218 | my $time = time(); 219 | my $date_time = strftime( '%Y-%m-%d %H:%M:%S', localtime $time ); 220 | my $microseconds = ( $time * 1_000_000 ) % 1_000_000; 221 | my $time_zone = strftime( '%z', localtime $time ); 222 | 223 | my $time_stamp = sprintf "%s.%06u %s", $date_time, $microseconds, $time_zone; 224 | return sprintf "%s : %u : %s", $time_stamp, $PROCESS_ID, $self->{ 'program' }; 225 | } 226 | 227 | =head1 _get_log_fh() 228 | 229 | Internal method handling logic to close and open logfiles when 230 | necessary, based of given logfile template, current time, and when 231 | previous logline was logged. 232 | 233 | At any given moment only 1 filehandle will be opened, and it will be 234 | closed, and reopened, when time changes in such way that it would 235 | require another filename. 236 | 237 | =cut 238 | 239 | sub _get_log_fh { 240 | my $self = shift; 241 | 242 | return $self->{ 'forced_fh' } if $self->{ 'forced_fh' }; 243 | 244 | my $time = floor( time() ); 245 | return $self->{ 'log_fh' } if $self->{ 'current_log_ts' } == $time; 246 | 247 | $self->{ 'current_log_ts' } = $time; 248 | my $filename = strftime( $self->{ 'template' }, localtime $time ); 249 | return $self->{ 'log_fh' } if $self->{ 'current_log_fn' } eq $filename; 250 | 251 | $self->{ 'current_log_fn' } = $filename; 252 | close delete $self->{ 'log_fh' } if exists $self->{ 'log_fh' }; 253 | 254 | my $dirname = dirname $filename; 255 | mkpath( $dirname ) unless -e $dirname; 256 | open my $fh, '>>', $filename or croak( "Cannot open $filename for writing: $OS_ERROR" ); 257 | 258 | $self->{ 'log_fh' } = $fh; 259 | return $fh; 260 | } 261 | 262 | 1; 263 | -------------------------------------------------------------------------------- /lib/OmniPITR/Pidfile.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Pidfile; 2 | use strict; 3 | use warnings; 4 | 5 | our $VERSION = '1.005'; 6 | use Fcntl qw( :flock ); 7 | use File::Basename qw( basename ); 8 | require File::Spec; 9 | 10 | sub new { 11 | my $class = shift; 12 | my %args = @_; 13 | my $self = bless \%args, $class; 14 | unless ( $self->{ pidfile } ) { 15 | my $basename = basename( $0 ); 16 | my $dir = -w "/var/run" ? "/var/run" : File::Spec->tmpdir(); 17 | die "Can't write to $dir\n" unless -w $dir; 18 | my $pidfile = "$dir/$basename.pid"; 19 | $self->_verbose( "pidfile: $pidfile\n" ); 20 | $self->{ pidfile } = $pidfile; 21 | } 22 | $self->_create_pidfile(); 23 | return $self; 24 | } 25 | 26 | sub DESTROY { 27 | my $self = shift; 28 | $self->_destroy_pidfile(); 29 | } 30 | 31 | sub pidfile { 32 | my $self = shift; 33 | return $self->{ pidfile }; 34 | } 35 | 36 | sub _verbose { 37 | my $self = shift; 38 | return unless $self->{ verbose }; 39 | print STDERR @_; 40 | } 41 | 42 | sub _get_pid { 43 | my $self = shift; 44 | my $pidfile = $self->{ pidfile }; 45 | $self->_verbose( "get pid from $pidfile\n" ); 46 | open( my $pid_fh, '<', $pidfile ) or die "can't read pid file $pidfile\n"; 47 | flock( $pid_fh, LOCK_SH ); 48 | my $pid = <$pid_fh>; 49 | chomp( $pid ); 50 | flock( $pid_fh, LOCK_UN ); 51 | close( $pid_fh ); 52 | $self->_verbose( "pid = $pid\n" ); 53 | return $pid; 54 | } 55 | 56 | sub _is_running { 57 | my $pid = shift; 58 | return kill( 0, $pid ); 59 | } 60 | 61 | sub _create_pidfile { 62 | my $self = shift; 63 | my $pidfile = $self->{ pidfile }; 64 | if ( -e $pidfile ) { 65 | $self->_verbose( "pidfile $pidfile exists\n" ); 66 | my $pid = $self->_get_pid(); 67 | $self->_verbose( "pid in pidfile $pidfile = $pid\n" ); 68 | if ( _is_running( $pid ) ) { 69 | if ( $self->{ silent } ) { 70 | exit; 71 | } 72 | else { 73 | die "$0 already running: $pid ($pidfile)\n"; 74 | } 75 | } 76 | else { 77 | $self->_verbose( "$pid has died - replacing pidfile\n" ); 78 | open( my $pid_fh, '>', $pidfile ) or die "Can't write to $pidfile\n"; 79 | print $pid_fh "$$\n"; 80 | close( $pid_fh ); 81 | } 82 | } 83 | else { 84 | $self->_verbose( "no pidfile $pidfile\n" ); 85 | open( my $pid_fh, '>', $pidfile ) or die "Can't write to $pidfile\n"; 86 | flock( $pid_fh, LOCK_EX ); 87 | print $pid_fh "$$\n"; 88 | flock( $pid_fh, LOCK_UN ); 89 | close( $pid_fh ); 90 | $self->_verbose( "pidfile $pidfile created\n" ); 91 | } 92 | $self->{ created } = 1; 93 | } 94 | 95 | sub _destroy_pidfile { 96 | my $self = shift; 97 | 98 | return unless $self->{ created }; 99 | my $pidfile = $self->{ pidfile }; 100 | $self->_verbose( "destroy $pidfile\n" ); 101 | unless ( $pidfile and -e $pidfile ) { 102 | die "pidfile $pidfile doesn't exist\n"; 103 | } 104 | my $pid = $self->_get_pid(); 105 | $self->_verbose( "pid in $pidfile = $pid\n" ); 106 | if ( $pid == $$ ) { 107 | $self->_verbose( "remove pidfile: $pidfile\n" ); 108 | unlink( $pidfile ) if $pidfile and -e $pidfile; 109 | } 110 | else { 111 | $self->_verbose( "$pidfile not my pidfile - maybe my parents?\n" ); 112 | my $ppid = getppid(); 113 | $self->_verbose( "parent pid = $ppid\n" ); 114 | if ( $ppid != $pid ) { 115 | die "pid $pid in $pidfile is not mine ($$) - I am $0 - or my parents ($ppid)\n"; 116 | } 117 | } 118 | } 119 | 120 | #------------------------------------------------------------------------------ 121 | # 122 | # Start of POD 123 | # 124 | #------------------------------------------------------------------------------ 125 | 126 | =head1 NAME 127 | 128 | OmniPITR::Pidfile - a simple OO Perl module for maintaining a process id file for 129 | the curent process 130 | 131 | =head1 SYNOPSIS 132 | 133 | my $pp = OmniPITR::Pidfile->new( pidfile => "/path/to/your/pidfile" ); 134 | # if the pidfile already exists, die here 135 | $pidfile = $pp->pidfile(); 136 | undef $pp; 137 | # unlink $pidfile here 138 | 139 | my $pp = OmniPITR::Pidfile->new(); 140 | # creates pidfile in default location - /var/run or File::Spec->tmpdir ... 141 | my $pidfile = $pp=>pidfile(); 142 | # tells you where this pidfile is ... 143 | 144 | my $pp = OmniPITR::Pidfile->new( silent => 1 ); 145 | # if the pidfile already exists, exit silently here 146 | ... 147 | undef $pp; 148 | 149 | =head1 DISCLAIMER 150 | 151 | This code has been taken directly from Proc::Pidfile distribution by Ave Wrigley. 152 | 153 | The only change in it is B using Proc::ProcessTable module, but 154 | instead relying on kill(0, $pid) functionality. 155 | 156 | =head1 DESCRIPTION 157 | 158 | OmniPITR::Pidfile is a very simple OO interface which manages a pidfile for the 159 | current process. You can pass the path to a pidfile to use as an argument to 160 | the constructor, or you can let OmniPITR::Pidfile choose one (basically, 161 | "/var/run/$basename", if you can write to /var/run, otherwise 162 | "/$tmpdir/$basename"). 163 | 164 | Pidfiles created by OmniPITR::Pidfile are automatically removed on destruction of 165 | the object. At destruction, the module checks the process id in the pidfile 166 | against its own, and against its parents (in case it is a spawned child of the 167 | process that originally created the OmniPITR::Pidfile object), and barfs if it 168 | doesn't match either. 169 | 170 | If you pass a "silent" parameter to the constructor, then it will still check 171 | for the existence of a pidfile, but will exit silently if one is found. This is 172 | useful for, for example, cron jobs, where you don't want to create a new 173 | process if one is already running, but you don't necessarily want to be 174 | informed of this by cron. 175 | 176 | =head1 SEE ALSO 177 | 178 | Proc::PID::File, Proc::Pidfile 179 | 180 | =head1 AUTHOR 181 | 182 | Ave Wrigley 183 | 184 | =head1 COPYRIGHT 185 | 186 | Copyright (c) 2003 Ave Wrigley. All rights reserved. This program is free 187 | software; you can redistribute it and/or modify it under the same terms as Perl 188 | itself. 189 | 190 | =cut 191 | 192 | #------------------------------------------------------------------------------ 193 | # 194 | # End of POD 195 | # 196 | #------------------------------------------------------------------------------ 197 | 198 | #------------------------------------------------------------------------------ 199 | # 200 | # True ... 201 | # 202 | #------------------------------------------------------------------------------ 203 | 204 | 1; 205 | 206 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Checksum.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Checksum; 2 | use strict; 3 | use warnings; 4 | 5 | our $VERSION = '2.0.0'; 6 | use base qw( OmniPITR::Program ); 7 | 8 | use Carp; 9 | use File::Spec; 10 | use Digest; 11 | use File::Basename; 12 | use English qw( -no_match_vars ); 13 | use Getopt::Long qw( :config no_ignore_case ); 14 | 15 | =head1 run() 16 | 17 | Main function, called by actual script in bin/, wraps all work done by script with the sole exception of reading and validating command line arguments. 18 | 19 | These tasks (reading and validating arguments) are in this module, but they are called from L 20 | 21 | Name of called method should be self explanatory, and if you need further information - simply check doc for the method you have questions about. 22 | 23 | =cut 24 | 25 | sub run { 26 | my $self = shift; 27 | $self->{ 'digest' }->addfile( \*STDIN ); 28 | printf "%s *%s\n", $self->{ 'digest' }->hexdigest, $self->{ 'filename' }; 29 | return; 30 | } 31 | 32 | =head1 read_args 33 | 34 | Function which does all the parsing of command line argument. 35 | 36 | =cut 37 | 38 | sub read_args { 39 | my $self = shift; 40 | 41 | my @argv_copy = @ARGV; 42 | 43 | my $config = {}; 44 | 45 | my $status = GetOptions( $config, qw( digest|d=s filename|f=s help|h|? version|V list|l ) ); 46 | if ( !$status ) { 47 | $self->show_help_and_die(); 48 | } 49 | 50 | $self->show_help_and_die() if $config->{ 'help' }; 51 | $self->show_available_digests() if $config->{ 'list' }; 52 | 53 | if ( $config->{ 'version' } ) { 54 | 55 | # The $self->VERSION below returns value of $VERSION variable in class of $self. 56 | printf '%s ver. %s%s', basename( $PROGRAM_NAME ), $self->VERSION, "\n"; 57 | exit; 58 | } 59 | 60 | $self->{ 'digest' } = $config->{ 'digest' } || 'MD5'; 61 | $self->{ 'filename' } = $config->{ 'filename' } || ''; 62 | 63 | # Restore original @ARGV 64 | @ARGV = @argv_copy; 65 | 66 | } 67 | 68 | sub show_available_digests { 69 | my $self = shift; 70 | 71 | my %found = (); 72 | for my $path ( @INC ) { 73 | my $digest_dir = File::Spec->catdir( $path, 'Digest' ); 74 | next unless -d $digest_dir; 75 | opendir my $dir, $digest_dir or next; 76 | my @content = readdir $dir; 77 | closedir $dir; 78 | 79 | for my $item ( @content ) { 80 | my $full_path = File::Spec->catfile( $digest_dir, $item ); 81 | next unless -f $full_path; 82 | next unless $item =~ m{\A(.*[A-Z].*)\.pm\z}; 83 | my $module = $1; 84 | $found{ $module } = 1; 85 | } 86 | } 87 | print "Available digests:\n"; 88 | printf "- %s\n", $_ for sort keys %found; 89 | exit( 0 ); 90 | } 91 | 92 | =head1 validate_args() 93 | 94 | Does all necessary validation of given command line arguments. 95 | 96 | =cut 97 | 98 | sub validate_args { 99 | my $self = shift; 100 | 101 | eval { 102 | my $digest = Digest->new( $self->{ 'digest' } ); 103 | $self->{ 'digest' } = $digest; 104 | }; 105 | if ( $EVAL_ERROR ) { 106 | printf STDERR "Cannot initialize digester %s. Try %s --list\n", $self->{ 'digest' }, $PROGRAM_NAME; 107 | exit 1; 108 | } 109 | return; 110 | } 111 | 112 | 1; 113 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Cleanup.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Cleanup; 2 | use strict; 3 | use warnings; 4 | 5 | our $VERSION = '2.0.0'; 6 | use base qw( OmniPITR::Program ); 7 | 8 | use Carp; 9 | use OmniPITR::Tools qw( :all ); 10 | use English qw( -no_match_vars ); 11 | use File::Spec; 12 | use Getopt::Long qw( :config no_ignore_case ); 13 | use Cwd; 14 | 15 | =head1 run() 16 | 17 | Main function, called by actual script in bin/, wraps all work done by 18 | script with the sole exception of reading and validating command line 19 | arguments. 20 | 21 | These tasks (reading and validating arguments) are in this module, but 22 | they are called from L 23 | 24 | Name of called method should be self explanatory, and if you need 25 | further information - simply check doc for the method you have questions 26 | about. 27 | 28 | =cut 29 | 30 | sub run { 31 | my $self = shift; 32 | 33 | if ( $self->{ 'removal-pause-trigger' } && -e $self->{ 'removal-pause-trigger' } ) { 34 | $self->log->log( 'Pause trigger exists (%s), NOT removing any old segments.', $self->{ 'removal-pause-trigger' } ); 35 | return; 36 | } 37 | 38 | my @to_be_removed = $self->get_list_of_segments_to_remove(); 39 | 40 | return if 0 == scalar @to_be_removed; 41 | 42 | my $count = unlink map { File::Spec->catfile( $self->{ 'archive' }->{ 'path' }, $_ ) } @to_be_removed; 43 | if ( $self->{ 'verbose' } ) { 44 | if ( $count == scalar @to_be_removed ) { 45 | $self->log->log( 'Segment %s removed.', $_ ) for @to_be_removed; 46 | } 47 | else { 48 | $self->log->log( 'Segment %s removed.', $_ ) for grep { !-e File::Spec->catfile( $self->{ 'archive' }->{ 'path' }, $_ ) } @to_be_removed; 49 | } 50 | } 51 | $self->log->log( '%d segments removed.', $count ); 52 | 53 | return; 54 | } 55 | 56 | =head1 get_list_of_segments_to_remove() 57 | 58 | Scans archive directory, and returns names of all files, which are 59 | "older" than last required segment (given as argument on command line) 60 | 61 | Older - is defined as alphabetically smaller than required segment. 62 | 63 | =cut 64 | 65 | sub get_list_of_segments_to_remove { 66 | my $self = shift; 67 | my $last_important = $self->{ 'segment' }; 68 | 69 | my $extension = undef; 70 | $extension = ext_for_compression( $self->{ 'archive' }->{ 'compression' } ) if $self->{ 'archive' }->{ 'compression' }; 71 | my $dir; 72 | 73 | unless ( opendir( $dir, $self->{ 'archive' }->{ 'path' } ) ) { 74 | $self->log->fatal( 'Cannot open archive directory (%s) for reading: %s', $self->{ 'archive' }->{ 'path' }, $OS_ERROR ); 75 | } 76 | my @content = readdir $dir; 77 | closedir $dir; 78 | 79 | my @too_old = (); 80 | for my $file ( @content ) { 81 | my $copy = $file; 82 | $file =~ s/\Q$extension\E\z// if $extension; 83 | next unless $file =~ m{\A[a-fA-F0-9]{24}(?:\.[a-fA-F0-9]{8}\.backup)?\z}; 84 | next unless $file lt $last_important; 85 | push @too_old, $copy; 86 | } 87 | if ( 0 == scalar @too_old ) { 88 | $self->log->log( 'No files to be removed.' ) if $self->verbose; 89 | return; 90 | } 91 | 92 | my @sorted = sort @too_old; 93 | 94 | $self->log->log( '%u segments too old, to be removed.', scalar @too_old ) if $self->verbose; 95 | $self->log->log( 'First segment to be removed: %s. Last one: %s', $too_old[ 0 ], $too_old[ -1 ] ) if $self->verbose; 96 | 97 | return @sorted; 98 | } 99 | 100 | =head1 read_args_specification 101 | 102 | Defines which options are legal for this program. 103 | 104 | =cut 105 | 106 | sub read_args_specification { 107 | my $self = shift; 108 | 109 | return { 110 | 'log' => { 'type' => 's', 'aliases' => [ 'l' ] }, 111 | 'pid-file' => { 'type' => 's' }, 112 | 'verbose' => { 'aliases' => [ 'v' ] }, 113 | 'archive' => { 'type' => 's', 'aliases' => [ 'a' ], }, 114 | 'removal-pause-trigger' => { 'type' => 's', 'aliases' => [ 'p' ], }, 115 | }; 116 | } 117 | 118 | =head1 read_args_normalization 119 | 120 | Function called back from OmniPITR::Program::read_args(), with parsed args as hashref. 121 | 122 | Is responsible for putting arguments to correct places, initializing logs, and so on. 123 | 124 | =cut 125 | 126 | sub read_args_normalization { 127 | my $self = shift; 128 | my $args = shift; 129 | 130 | for my $key ( keys %{ $args } ) { 131 | next if $key =~ m{ \A (?: archive | log ) \z }x; # Skip those, not needed in $self 132 | $self->{ $key } = $args->{ $key }; 133 | } 134 | 135 | $self->log->fatal( 'Archive path not provided!' ) unless $args->{ 'archive' }; 136 | 137 | if ( $args->{ 'archive' } =~ s/\A(gzip|bzip2|lzma|lz4|xz)=// ) { 138 | $self->{ 'archive' }->{ 'compression' } = $1; 139 | } 140 | $self->{ 'archive' }->{ 'path' } = $args->{ 'archive' }; 141 | 142 | # These could theoretically go into validation, but we need to check if we can get anything to put in segment key in $self 143 | $self->log->fatal( 'WAL segment name has not been given' ) if 0 == scalar @{ $args->{ '-arguments' } }; 144 | $self->log->fatal( 'Too many arguments given.' ) if 1 < scalar @{ $args->{ '-arguments' } }; 145 | 146 | $self->{ 'segment' } = $args->{ '-arguments' }->[ 0 ]; 147 | 148 | $self->log->log( 'Called with parameters: %s', join( ' ', @ARGV ) ) if $self->verbose; 149 | 150 | return; 151 | } 152 | 153 | =head1 validate_args() 154 | 155 | Does all necessary validation of given command line arguments. 156 | 157 | One exception is for compression programs paths - technically, it could 158 | be validated in here, but benefit would be pretty limited, and code to 159 | do so relatively complex, as compression program path might, but doesn't 160 | have to be actual file path - it might be just program name (without 161 | path), which is the default. 162 | 163 | =cut 164 | 165 | sub validate_args { 166 | my $self = shift; 167 | 168 | $self->log->fatal( 'Given segment name is not valid (%s)', $self->{ 'segment' } ) unless $self->{ 'segment' } =~ m{\A(?:[a-fA-F0-9]{24}(?:\.[a-fA-F0-9]{8}\.backup|\.partial)?|[a-fA-F0-9]{8}\.history)\z}; 169 | 170 | $self->log->fatal( 'Given archive (%s) is not a directory', $self->{ 'archive' }->{ 'path' } ) unless -d $self->{ 'archive' }->{ 'path' }; 171 | $self->log->fatal( 'Given archive (%s) is not readable', $self->{ 'archive' }->{ 'path' } ) unless -r $self->{ 'archive' }->{ 'path' }; 172 | $self->log->fatal( 'Given archive (%s) is not writable', $self->{ 'archive' }->{ 'path' } ) unless -w $self->{ 'archive' }->{ 'path' }; 173 | 174 | return; 175 | } 176 | 177 | 1; 178 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Monitor.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Monitor; 2 | use strict; 3 | use warnings; 4 | 5 | our $VERSION = '2.0.0'; 6 | use base qw( OmniPITR::Program ); 7 | 8 | use Carp; 9 | use English qw( -no_match_vars ); 10 | use Getopt::Long qw( :config no_ignore_case pass_through ); 11 | use Storable qw( fd_retrieve store_fd ); 12 | use File::Basename; 13 | use File::Spec; 14 | use Fcntl qw( :flock :seek ); 15 | use POSIX qw( strftime ); 16 | use Time::Local; 17 | 18 | =head1 run() 19 | 20 | Main function, called by actual script in bin/, wraps all work done by 21 | script with the sole exception of reading and validating command line 22 | arguments. 23 | 24 | These tasks (reading and validating arguments) are in this module, but they 25 | are called from L 26 | 27 | Name of called method should be self explanatory, and if you need further 28 | information - simply check doc for the method you have questions about. 29 | 30 | =cut 31 | 32 | sub run { 33 | my $self = shift; 34 | $self->load_state(); 35 | if ( $self->read_logs() ) { 36 | $self->clean_old_state(); 37 | $self->save_state(); 38 | } 39 | 40 | my $check_state_dir = File::Spec->catfile( $self->{ 'state-dir' }, 'Check-' . $self->{ 'check' } ); 41 | if ( !-d $check_state_dir ) { 42 | $self->log->fatal( 'Cannot create state dir for check (%s) : %s', $check_state_dir, $OS_ERROR ) unless mkdir $check_state_dir; 43 | } 44 | 45 | my $O = $self->{ 'check_object' }; 46 | $O->setup( 47 | 'state-dir' => $check_state_dir, 48 | 'log' => $self->{ 'log' }, 49 | 'psql' => sub { return $self->psql( @_ ) }, 50 | ); 51 | 52 | $O->get_args(); 53 | $O->run_check( $self->{ 'state' } ); 54 | return; 55 | } 56 | 57 | =head1 read_logs() 58 | 59 | Wraps all work related to finding actual log files, reading and parsing 60 | them, and extracting information to "state". 61 | 62 | =cut 63 | 64 | sub read_logs { 65 | my $self = shift; 66 | 67 | $self->get_list_of_log_files(); 68 | 69 | # SHORTCUT 70 | my $F = $self->state( 'files' ); 71 | 72 | # SHORTCUT 73 | 74 | my @sorted_files = sort { $F->{ $a }->{ 'start_epoch' } <=> $F->{ $b }->{ 'start_epoch' } } @{ $self->{ 'log_files' } }; 75 | 76 | my $any_changes = undef; 77 | 78 | for my $filename ( @sorted_files ) { 79 | 80 | # Shortcut 81 | my $D = $F->{ $filename }; 82 | 83 | # Shortcut 84 | 85 | my $size = ( stat( $filename ) )[ 7 ]; 86 | next if ( $D->{ 'seek' } ) && ( $D->{ 'seek' } >= $size ); 87 | 88 | my $i = 0; 89 | open my $fh, '<', $filename or $self->log->fatal( 'Cannot open %s to read: %s', $filename, $OS_ERROR ); 90 | seek( $fh, $D->{ 'seek' }, SEEK_SET ) if defined $D->{ 'seek' }; 91 | while ( my $line = <$fh> ) { 92 | 93 | # We might read file that is being written to, so we should disregard any line that is partially written. 94 | last unless $line =~ s{\r?\n\z}{}; 95 | 96 | $self->parse_line( $line ); 97 | 98 | $D->{ 'seek' } = tell( $fh ); 99 | $i++; 100 | } 101 | close $fh; 102 | $self->log->log( 'Read %d lines from %s', $i, $filename ) if $self->{ 'verbose' }; 103 | $any_changes = 1 if $i; 104 | } 105 | 106 | return $any_changes; 107 | } 108 | 109 | =head1 parse_line() 110 | 111 | Given line from logs, parses it to atoms, and stores important information to state. 112 | 113 | =cut 114 | 115 | sub parse_line { 116 | my $self = shift; 117 | my $line = shift; 118 | 119 | my $epoch = $self->extract_epoch( $line ); 120 | $self->log->fatal( 'Cannot parse line: %s', $line ) unless $line =~ s/\A(.{26}) [+-]\d+ : (\d+) : omnipitr-(\S+) : //; 121 | my $timestamp = $1; 122 | my $pid = $2; 123 | my $program_name = $3; 124 | 125 | my $data = { 126 | 'epoch' => $epoch, 127 | 'timestamp' => $timestamp, 128 | 'pid' => $pid, 129 | 'line' => $line, 130 | }; 131 | 132 | if ( $line =~ m{^(ERROR|FATAL) : } ) { 133 | push @{ $self->{ 'state' }->{ 'errors' }->{ $1 } }, $data; 134 | } 135 | 136 | my $P = $self->{ 'parser' }->{ $program_name }; 137 | if ( !defined $P ) { 138 | my $ignore; 139 | ( $P, $ignore ) = $self->load_dynamic_object( 'OmniPITR::Program::Monitor::Parser', $program_name ); 140 | if ( defined $P ) { 141 | $self->{ 'parser' }->{ $program_name } = $P; 142 | $P->setup( 143 | 'state' => $self->{ 'state' }, 144 | 'log' => $self->{ 'log' }, 145 | ); 146 | } 147 | else { 148 | $self->{ 'parser' }->{ $program_name } = ''; 149 | } 150 | } 151 | 152 | $P->handle_line( $data ) if ref $P; 153 | 154 | return; 155 | } 156 | 157 | =head1 clean_old_state() 158 | 159 | Calls ->clean_state() on all parser objects (that were used in current iteration). 160 | 161 | This is to remove from state old data, that is of no use currently. 162 | 163 | =cut 164 | 165 | sub clean_old_state { 166 | my $self = shift; 167 | 168 | for my $P ( values %{ $self->{ 'parser' } } ) { 169 | next unless ref $P; 170 | next unless $P->can( 'clean_state' ); 171 | $P->clean_state(); 172 | } 173 | 174 | my $cutoff = time() - 3 * 24 * 60 * 60; # 3 days ago 175 | for my $type ( qw( ERROR FATAL ) ) { 176 | next unless defined $self->{ 'state' }->{ 'errors' }->{ $type }; 177 | $self->{ 'state' }->{ 'errors' }->{ $type } = [ grep { $_->{ 'epoch' } >= $cutoff } @{ $self->{ 'state' }->{ 'errors' }->{ $type } } ]; 178 | } 179 | 180 | return; 181 | } 182 | 183 | =head1 get_list_of_log_files() 184 | 185 | Scans given log paths, and collects list of files that are log files. 186 | 187 | List of all log files is stored in $self->{'log_files'}, being arrayref. 188 | 189 | =cut 190 | 191 | sub get_list_of_log_files { 192 | my $self = shift; 193 | 194 | # SHORTCUT 195 | my $F = $self->state( 'files' ); 196 | $F = $self->state( 'files', {} ) unless defined $F; 197 | 198 | # SHORTCUT 199 | 200 | $self->{ 'log_files' } = []; 201 | my @scan_for_timestamps = (); 202 | my %exists_file = (); 203 | 204 | for my $template ( @{ $self->{ 'log-paths' } } ) { 205 | my $glob = $template; 206 | $glob =~ s/\%./*/g; 207 | for my $filename ( glob( $glob ) ) { 208 | $exists_file{ $filename } = 1; 209 | if ( $F->{ $filename } ) { 210 | push @{ $self->{ 'log_files' } }, $filename; 211 | } 212 | else { 213 | push @scan_for_timestamps, [ $filename, $template ]; 214 | } 215 | } 216 | } 217 | 218 | for my $filename ( keys %{ $F } ) { 219 | delete $F->{ $filename } if !$exists_file{ $filename }; 220 | } 221 | 222 | for my $file ( @scan_for_timestamps ) { 223 | my ( $filename, $template ) = @{ $file }; 224 | 225 | my $fh; 226 | next unless open $fh, '<', $filename; 227 | 228 | my $data; 229 | my $length = sysread( $fh, $data, 27 ); 230 | close $fh; 231 | 232 | next if 27 != $length; 233 | 234 | my $epoch = $self->extract_epoch( $data ); 235 | next unless defined $epoch; 236 | 237 | my $reconstructed_filename = strftime( $template, localtime( $epoch ) ); 238 | next unless $reconstructed_filename eq $filename; 239 | 240 | $F->{ $filename }->{ 'start_epoch' } = $epoch; 241 | push @{ $self->{ 'log_files' } }, $filename; 242 | } 243 | return; 244 | } 245 | 246 | =head1 extract_epoch() 247 | 248 | Given line from logs, it returns epoch value of leading timestamp. 249 | 250 | If the line cannot be parsed, or the value is not sensible time - undef is returned. 251 | 252 | Returned epoch can (and usually will) contain fractional part - subsecond data with precision of up to microsecond (0.000001s). 253 | 254 | =cut 255 | 256 | sub extract_epoch { 257 | my $self = shift; 258 | my $line = shift; 259 | return if 27 > length $line; 260 | return unless my @elements = $line =~ m{\A(\d\d\d\d)-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d{6}) }; 261 | my $subsecond = pop @elements; 262 | $elements[ 1 ]--; # Time::Local expects months in range 0-11, and not 1-12. 263 | my $epoch; 264 | eval { $epoch = timelocal( reverse @elements ); }; 265 | return if $EVAL_ERROR; 266 | return $epoch + $subsecond; 267 | } 268 | 269 | =head1 state() 270 | 271 | Helper function, accessor, to state hash. 272 | 273 | Has 1, or two arguments. In case of one argument - returns value, from state, for given key. 274 | 275 | If it has two arguments, then - if 2nd argument is undef - it removes the key from state, and returns. 276 | 277 | If the 2nd argument is defined, it sets value for given key to given value, and returns it. 278 | 279 | =cut 280 | 281 | sub state { 282 | my $self = shift; 283 | my $key = shift; 284 | if ( 0 == scalar @_ ) { 285 | return $self->{ 'state' }->{ $key }; 286 | } 287 | my $value = shift; 288 | if ( defined $value ) { 289 | return $self->{ 'state' }->{ $key } = $value; 290 | } 291 | delete $self->{ 'state' }->{ $key }; 292 | return; 293 | } 294 | 295 | =head1 load_state() 296 | 297 | Handler reading of state file. 298 | 299 | It is important to note that it uses locking, so it will not conflict with 300 | state writing from another run of omnipitr-monitor. 301 | 302 | =cut 303 | 304 | sub load_state { 305 | my $self = shift; 306 | $self->{ 'state' } = {}; 307 | 308 | $self->{ 'state-file' } = File::Spec->catfile( $self->{ 'state-dir' }, 'omnipitr-monitor.state' ); 309 | return unless -f $self->{ 'state-file' }; 310 | 311 | open my $fh, '<', $self->{ 'state-file' } 312 | or $self->log->fatal( "Cannot open state file (%s) for reading: %s", $self->{ 'state-file' }, $OS_ERROR ); 313 | 314 | # Make sure the file is not written to, at the moment. 315 | flock( $fh, LOCK_SH ); 316 | 317 | $self->{ 'state' } = fd_retrieve( $fh ); 318 | close $fh; 319 | 320 | return; 321 | } 322 | 323 | =head1 save_state() 324 | 325 | Saves state in safe way (with proper locking). 326 | 327 | =cut 328 | 329 | sub save_state { 330 | my $self = shift; 331 | 332 | $self->{ 'state-file' } = File::Spec->catfile( $self->{ 'state-dir' }, 'omnipitr-monitor.state' ); 333 | 334 | my $fh; 335 | if ( -f $self->{ 'state-file' } ) { 336 | open $fh, '+<', $self->{ 'state-file' } 337 | or $self->log->fatal( "Cannot open state file (%s) for writing: %s", $self->{ 'state-file' }, $OS_ERROR ); 338 | } 339 | else { 340 | open $fh, '>', $self->{ 'state-file' } 341 | or $self->log->fatal( "Cannot open state file (%s) for writing: %s", $self->{ 'state-file' }, $OS_ERROR ); 342 | } 343 | 344 | # Make sure the file is not written to, at the moment. 345 | flock( $fh, LOCK_EX ); 346 | 347 | store_fd( $self->{ 'state' }, $fh ); 348 | 349 | # In case current state was smaller than previously written 350 | truncate( $fh, tell( $fh ) ); 351 | 352 | close $fh; 353 | 354 | return; 355 | } 356 | 357 | =head1 read_args() 358 | 359 | Function which handles reading of base arguments ( i.e. without options specific to checks ). 360 | 361 | =cut 362 | 363 | sub read_args { 364 | my $self = shift; 365 | 366 | my %args = ( 367 | 'temp-dir' => $ENV{ 'TMPDIR' } || '/tmp', 368 | 'psql-path' => 'psql', 369 | ); 370 | 371 | $self->show_help_and_die( 'Error while reading command line arguments. Please check documentation in doc/omnipitr-archive.pod' ) 372 | unless GetOptions( 373 | \%args, 374 | 'log|l=s@', 375 | 'check|c=s', 376 | 'state-dir|s=s', 377 | 'verbose|v', 378 | 'database|d=s', 379 | 'host|h=s', 380 | 'port|p=i', 381 | 'username|U=s', 382 | 'temp-dir|t=s', 383 | 'psql-path|pp=s', 384 | 'help|?', 385 | 'version|V', 386 | ); 387 | 388 | if ( $args{ 'version' } ) { 389 | printf '%s ver. %s%s', basename( $PROGRAM_NAME ), $VERSION, "\n"; 390 | exit; 391 | } 392 | 393 | $self->show_help_and_die() if $args{ 'help' }; 394 | 395 | for my $key ( qw( check state-dir verbose database host port username temp-dir psql-path ) ) { 396 | next unless defined $args{ $key }; 397 | $self->{ $key } = $args{ $key }; 398 | } 399 | 400 | $self->{ 'log-paths' } = $args{ 'log' } if defined $args{ 'log' }; 401 | 402 | $self->{ 'log' } = OmniPITR::Log->new( \*STDOUT ); 403 | 404 | return; 405 | } 406 | 407 | =head1 validate_args() 408 | 409 | Does all necessary validation of given command line arguments. 410 | 411 | =cut 412 | 413 | sub validate_args { 414 | my $self = shift; 415 | 416 | $self->log->fatal( '--state-dir has not been provided!' ) unless defined $self->{ 'state-dir' }; 417 | $self->log->fatal( "Given --state-dir (%s) does not exist", $self->{ 'state-dir' } ) unless -e $self->{ 'state-dir' }; 418 | $self->log->fatal( "Given --state-dir (%s) is not a directory", $self->{ 'state-dir' } ) unless -d $self->{ 'state-dir' }; 419 | $self->log->fatal( "Given --state-dir (%s) is not writable", $self->{ 'state-dir' } ) unless -w $self->{ 'state-dir' }; 420 | 421 | $self->log->fatal( '--check has not been provided!' ) unless defined $self->{ 'check' }; 422 | $self->log->fatal( 'Invalid value for --check: %s', $self->{ 'check' } ) 423 | unless $self->{ 'check' } =~ m{ \A [a-zA-Z0-9]+ (?: [_-][a-zA-Z0-9]+ )* \z }x; 424 | 425 | $self->log->fatal( '--log has not been provided!' ) unless defined $self->{ 'log-paths' }; 426 | 427 | for my $path ( @{ $self->{ 'log-paths' } } ) { 428 | $path =~ s/\^/\%/g; 429 | } 430 | 431 | ( $self->{ 'check_object' }, $self->{ 'check' } ) = $self->load_dynamic_object( 'OmniPITR::Program::Monitor::Check', $self->{ 'check' } ); 432 | $self->log->fatal( 'Check code cannot be loaded.' ) unless $self->{ 'check_object' }; 433 | 434 | return; 435 | } 436 | 437 | =head1 load_dynamic_object() 438 | 439 | Loads class which name is based on arguments. 440 | 441 | If loading will succeed, creates new object of this class and returns it. 442 | 443 | If it fails - ends program with logged message. 444 | 445 | =cut 446 | 447 | sub load_dynamic_object { 448 | my $self = shift; 449 | my $prefix = shift; 450 | my $dynamic_part = shift; 451 | 452 | $prefix =~ s/:+\z//; 453 | 454 | $dynamic_part =~ s/[^a-zA-Z0-9]/_/g; 455 | $dynamic_part =~ s/([a-zA-Z0-9]+)/\u\L$1/g; 456 | 457 | my $full_class_name = $prefix . '::' . $dynamic_part; 458 | 459 | my $class_filename = $full_class_name . '.pm'; 460 | $class_filename =~ s{::}{/}g; 461 | 462 | my $object; 463 | 464 | eval { 465 | require $class_filename; 466 | $object = $full_class_name->new(); 467 | }; 468 | if ( $EVAL_ERROR ) { 469 | $self->log->error( 'Cannot load class %s: %s', $full_class_name, $EVAL_ERROR ) if $self->{ 'verbose' }; 470 | return ( undef, undef ); 471 | } 472 | 473 | return $object, $dynamic_part; 474 | } 475 | 476 | 1; 477 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Monitor/Check.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Monitor::Check; 2 | use strict; 3 | use warnings; 4 | use Carp; 5 | use English qw( -no_match_vars ); 6 | 7 | our $VERSION = '2.0.0'; 8 | 9 | =head1 NAME 10 | 11 | OmniPITR::Program::Monitor::Check - base for omnipitr-monitor checks 12 | 13 | =head1 SYNOPSIS 14 | 15 | package OmniPITR::Program::Monitor::Check::Whatever; 16 | use base qw( OmniPITR::Program::Monitor::Check ); 17 | sub setup { ... } 18 | sub get_args { ... } 19 | sub run_check { ... } 20 | 21 | =head1 DESCRIPTION 22 | 23 | This is base class that we expect all check classes inherit from. 24 | 25 | While not technically requirement, it might make writing check classes simpler. 26 | 27 | =head1 CONTROL FLOW 28 | 29 | When omnipitr-monitor creates check object, it doesn't pass any arguments (yet). 30 | 31 | Afterwards, it calls ->setup() function, passing (as hash): 32 | 33 | =over 34 | 35 | =item * state-dir - directory where check can store it's own data, in subdirectory named like last element of check package name 36 | 37 | =item * log - log object 38 | 39 | =item * psql - coderef which will run given query via psql, and return whole output as scalar 40 | 41 | =back 42 | 43 | Afterwards, omnipitr-monitor will run "get_args" method (if it's defined), to get all necessary options from command line - options specifically for this check. 44 | 45 | Finally run_check() method will be called, with one argument - being full copy of omnipitr-monitor internal state. 46 | 47 | =head1 METHODS 48 | 49 | =head2 new() 50 | 51 | Object constructor. No logic in here. Just makes simple hashref based object. 52 | 53 | =cut 54 | 55 | sub new { 56 | my $class = shift; 57 | my $self = bless {}, $class; 58 | return $self; 59 | } 60 | 61 | =head2 setup() 62 | 63 | Sets check for work - receives state-dir and log object from omnipitr-monitor. 64 | 65 | =cut 66 | 67 | sub setup { 68 | my $self = shift; 69 | my %args = @_; 70 | for my $key ( qw( log state-dir psql ) ) { 71 | croak( "$key not given in call to ->setup()." ) unless defined $args{ $key }; 72 | $self->{ $key } = $args{ $key }; 73 | } 74 | return; 75 | } 76 | 77 | =head2 get_args() 78 | 79 | This method should be overriden in check class if the check has some options get from command line. 80 | 81 | =cut 82 | 83 | sub get_args { 84 | my $self = shift; 85 | return; 86 | } 87 | 88 | =head1 log() 89 | 90 | Shortcut to make code a bit nicer. 91 | 92 | Returns logger object. 93 | 94 | =cut 95 | 96 | sub log { return shift->{ 'log' }; } 97 | 98 | =head1 psql() 99 | 100 | Runs given query via psql. 101 | 102 | =cut 103 | 104 | sub psql { 105 | my $self = shift; 106 | return $self->{ 'psql' }->( @_ ); 107 | } 108 | 109 | 1; 110 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Monitor/Check/Archive_Queue.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Monitor::Check::Archive_Queue; 2 | 3 | use strict; 4 | use warnings; 5 | use Carp; 6 | use English qw( -no_match_vars ); 7 | 8 | our $VERSION = '2.0.0'; 9 | use base qw( OmniPITR::Program::Monitor::Check ); 10 | 11 | use Data::Dumper; 12 | 13 | sub run_check { 14 | my $self = shift; 15 | my $state = shift; 16 | 17 | my $x = $self->psql( 'select pg_walfile_name(pg_current_wal_lsn())' ); 18 | $x =~ s/\s*\z//; 19 | my ( $timeline, $current_xlog ) = $self->split_xlog_filename( $x ); 20 | 21 | $current_xlog--; # Decrease because if we are currently in xlog 12, then the last archived can be at most 11. 22 | 23 | my $last_archive = undef; 24 | my $last_archived_xlog = undef; 25 | 26 | my $S = $state->{ 'Archive' }; 27 | while ( my ( $xlog, $X ) = each %{ $S->{ $timeline } } ) { 28 | next unless defined $X->[ 1 ]; 29 | if ( ( !defined $last_archive ) 30 | || ( $last_archive < $X->[ 1 ] ) ) 31 | { 32 | $last_archive = $X->[ 1 ]; 33 | $last_archived_xlog = $xlog; 34 | } 35 | } 36 | $last_archived_xlog =~ s/(..)\z//; 37 | my $lower = hex( $1 ); 38 | my $upper = hex( $last_archived_xlog ); 39 | 40 | print $current_xlog - ( 255 * $upper + $lower ), "\n"; 41 | 42 | return; 43 | } 44 | 45 | =head1 split_xlog_filename() 46 | 47 | Splits given xlog filename (24 hex characters) into a pair of timeline and xlog offset. 48 | 49 | Timeline is trimmed of leading 0s, and xlog offset to converted to decimal. 50 | 51 | =cut 52 | 53 | sub split_xlog_filename { 54 | my $self = shift; 55 | my $xlog_name = shift; 56 | 57 | my ( $timeline, @elements ) = unpack( '(A8)3', $xlog_name ); 58 | $timeline =~ s/^0*//; 59 | 60 | $elements[ 0 ] =~ s/^0*//; 61 | $elements[ 1 ] =~ s/^0{6}//; 62 | my $upper = hex( $elements[ 0 ] ); 63 | my $lower = hex( $elements[ 1 ] ); 64 | return ( $timeline, $upper * 255 + $lower ); 65 | } 66 | 67 | 1; 68 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Monitor/Check/Current_Archive_Time.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Monitor::Check::Current_Archive_Time; 2 | 3 | use strict; 4 | use warnings; 5 | use Carp; 6 | use English qw( -no_match_vars ); 7 | 8 | our $VERSION = '2.0.0'; 9 | use base qw( OmniPITR::Program::Monitor::Check ); 10 | 11 | use Time::HiRes qw( time ); 12 | 13 | sub run_check { 14 | my $self = shift; 15 | my $state = shift; 16 | 17 | my $S = $state->{ 'Archive' }; 18 | for my $T ( values %{ $S } ) { 19 | for my $X ( values %{ $T } ) { 20 | next if defined $X->[ 1 ]; 21 | printf '%f%s', time() - $X->[ 0 ], "\n"; 22 | return; 23 | } 24 | } 25 | 26 | print "0\n"; 27 | return; 28 | } 29 | 30 | 1; 31 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Monitor/Check/Current_Restore_Time.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Monitor::Check::Current_Restore_Time; 2 | 3 | use strict; 4 | use warnings; 5 | use Carp; 6 | use English qw( -no_match_vars ); 7 | 8 | our $VERSION = '2.0.0'; 9 | use base qw( OmniPITR::Program::Monitor::Check ); 10 | 11 | use Time::HiRes qw( time ); 12 | 13 | sub run_check { 14 | my $self = shift; 15 | my $state = shift; 16 | 17 | my $S = $state->{ 'Restore' }; 18 | for my $T ( values %{ $S } ) { 19 | for my $X ( values %{ $T } ) { 20 | next if defined $X->[ 1 ]; 21 | printf '%f%s', time() - $X->[ 0 ], "\n"; 22 | return; 23 | } 24 | } 25 | 26 | print "0\n"; 27 | return; 28 | } 29 | 30 | 1; 31 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Monitor/Check/Dump_State.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Monitor::Check::Dump_State; 2 | 3 | use strict; 4 | use warnings; 5 | use Carp; 6 | use English qw( -no_match_vars ); 7 | 8 | our $VERSION = '2.0.0'; 9 | use base qw( OmniPITR::Program::Monitor::Check ); 10 | 11 | use Data::Dumper; 12 | 13 | sub run_check { 14 | my $self = shift; 15 | my $state = shift; 16 | my $d = Data::Dumper->new( [ $state ], [ 'state' ] ); 17 | $d->Sortkeys( 1 ); 18 | $d->Indent( 1 ); 19 | $d->{ 'xpad' } = ' '; 20 | print $d->Dump(); 21 | return; 22 | } 23 | 24 | 1; 25 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Monitor/Check/Errors.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Monitor::Check::Errors; 2 | 3 | use strict; 4 | use warnings; 5 | use Carp; 6 | use English qw( -no_match_vars ); 7 | 8 | our $VERSION = '2.0.0'; 9 | use base qw( OmniPITR::Program::Monitor::Check ); 10 | 11 | use Getopt::Long qw( :config no_ignore_case ); 12 | use Time::Local; 13 | use Data::Dumper; 14 | use File::Spec; 15 | 16 | sub run_check { 17 | my $self = shift; 18 | my $state = shift; 19 | $self->load_from_timestamp() if $self->{ 'state-based-from' }; 20 | my @all_errors; 21 | for my $type ( qw( ERROR FATAL ) ) { 22 | next unless $state->{ 'errors' }->{ $type }; 23 | push @all_errors, @{ $state->{ 'errors' }->{ $type } }; 24 | } 25 | return if 0 == scalar @all_errors; 26 | my @sorted_to_print = sort { $a->{ 'epoch' } <=> $b->{ 'epoch' } } grep { $_->{ 'epoch' } > $self->{ 'from' } } @all_errors; 27 | return if 0 == scalar @sorted_to_print; 28 | for my $line ( @sorted_to_print ) { 29 | printf '%s : %s : %s%s', @{ $line }{ qw( timestamp pid line ) }, "\n"; 30 | } 31 | my $final_ts = $sorted_to_print[ -1 ]->{ 'epoch' }; 32 | $self->save_from_timestamp( $final_ts ) if $self->{ 'state-based-from' }; 33 | return; 34 | } 35 | 36 | sub get_args { 37 | my $self = shift; 38 | my $from = undef; 39 | GetOptions( 'from=s' => \$from ); 40 | if ( !defined $from ) { 41 | $self->{ 'state-based-from' } = 1; 42 | } 43 | elsif ( my @elements = $from =~ m{\A(\d\d\d\d)-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)\z} ) { 44 | $elements[ 1 ]--; # Time::Local expects months in range 0-11, and not 1-12. 45 | my $epoch; 46 | eval { $epoch = timelocal( reverse @elements ); }; 47 | $self->log->fatal( 'Given date (%s) is not valid', $from ) if $EVAL_ERROR; 48 | $self->{ 'from' } = $epoch; 49 | } 50 | elsif ( $from =~ m{\A\d+\z} ) { 51 | $self->{ 'from' } = time() - $from; 52 | } 53 | else { 54 | $self->log->fatal( 'Bad format of given --from, should be YYYY-MM-DD HH:MI:SS, or just an integer' ); 55 | } 56 | return; 57 | } 58 | 59 | sub load_from_timestamp { 60 | my $self = shift; 61 | if ( open my $fh, '<', File::Spec->catfile( $self->{ 'state-dir' }, 'from-timestamp' ) ) { 62 | my $timestamp = <$fh>; 63 | close $fh; 64 | chomp $timestamp; 65 | $self->{ 'from' } = $timestamp if $timestamp =~ m{\A\d+(?:\.\d+)?\z}; 66 | } 67 | $self->{ 'from' } ||= 0; 68 | return; 69 | } 70 | 71 | sub save_from_timestamp { 72 | my $self = shift; 73 | my $final_ts = shift; 74 | if ( open my $fh, '>', File::Spec->catfile( $self->{ 'state-dir' }, 'from-timestamp' ) ) { 75 | printf $fh '%.6f', $final_ts; 76 | close $fh; 77 | } 78 | return; 79 | } 80 | 81 | 1; 82 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Monitor/Check/Last_Archive_Age.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Monitor::Check::Last_Archive_Age; 2 | 3 | use strict; 4 | use warnings; 5 | use Carp; 6 | use English qw( -no_match_vars ); 7 | 8 | our $VERSION = '2.0.0'; 9 | use base qw( OmniPITR::Program::Monitor::Check ); 10 | 11 | use Time::HiRes qw( time ); 12 | 13 | sub run_check { 14 | my $self = shift; 15 | my $state = shift; 16 | 17 | my $last_archive = undef; 18 | 19 | my $S = $state->{ 'Archive' }; 20 | for my $T ( values %{ $S } ) { 21 | for my $X ( values %{ $T } ) { 22 | next unless defined $X->[ 1 ]; 23 | if ( ( !defined $last_archive ) 24 | || ( $last_archive < $X->[ 1 ] ) ) 25 | { 26 | $last_archive = $X->[ 1 ]; 27 | } 28 | } 29 | } 30 | 31 | if ( defined $last_archive ) { 32 | printf '%f%s', time() - $last_archive, "\n"; 33 | } 34 | else { 35 | print "0\n"; 36 | } 37 | return; 38 | } 39 | 40 | 1; 41 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Monitor/Check/Last_Backup_Age.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Monitor::Check::Last_Backup_Age; 2 | 3 | use strict; 4 | use warnings; 5 | use Carp; 6 | use English qw( -no_match_vars ); 7 | 8 | our $VERSION = '2.0.0'; 9 | use base qw( OmniPITR::Program::Monitor::Check ); 10 | 11 | use Time::HiRes qw( time ); 12 | 13 | sub run_check { 14 | my $self = shift; 15 | my $state = shift; 16 | 17 | my $last_backup = undef; 18 | 19 | for my $backup_type ( qw( Backup_Master Backup_Slave ) ) { 20 | next unless $state->{ $backup_type }; 21 | my $S = $state->{ $backup_type }; 22 | for my $backup ( reverse @{ $S } ) { 23 | next unless $backup->[ 2 ]; 24 | if ( ( !defined $last_backup ) || ( $last_backup < $backup->[ 2 ] ) ) { 25 | $last_backup = $backup->[ 2 ]; 26 | } 27 | last; 28 | } 29 | } 30 | 31 | if ( defined $last_backup ) { 32 | printf '%f%s', time() - $last_backup, "\n"; 33 | } 34 | else { 35 | print "0\n"; 36 | } 37 | return; 38 | } 39 | 40 | 1; 41 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Monitor/Check/Last_Restore_Age.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Monitor::Check::Last_Restore_Age; 2 | 3 | use strict; 4 | use warnings; 5 | use Carp; 6 | use English qw( -no_match_vars ); 7 | 8 | our $VERSION = '2.0.0'; 9 | use base qw( OmniPITR::Program::Monitor::Check ); 10 | 11 | use Time::HiRes qw( time ); 12 | 13 | sub run_check { 14 | my $self = shift; 15 | my $state = shift; 16 | 17 | my $last_restore = undef; 18 | 19 | my $S = $state->{ 'Restore' }; 20 | for my $T ( values %{ $S } ) { 21 | for my $X ( values %{ $T } ) { 22 | next unless defined $X->[ 1 ]; 23 | if ( ( !defined $last_restore ) 24 | || ( $last_restore < $X->[ 1 ] ) ) 25 | { 26 | $last_restore = $X->[ 1 ]; 27 | } 28 | } 29 | } 30 | if ( defined $last_restore ) { 31 | printf '%f%s', time() - $last_restore, "\n"; 32 | } 33 | else { 34 | print "0\n"; 35 | } 36 | return; 37 | } 38 | 39 | 1; 40 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Monitor/Parser.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Monitor::Parser; 2 | use strict; 3 | use warnings; 4 | use Carp; 5 | use Data::Dumper; 6 | use English qw( -no_match_vars ); 7 | 8 | our $VERSION = '2.0.0'; 9 | 10 | =head1 NAME 11 | 12 | OmniPITR::Program::Monitor::Parser - base for omnipitr-monitor parsers 13 | 14 | =head1 SYNOPSIS 15 | 16 | package OmniPITR::Program::Monitor::Parser::Whatever; 17 | use base qw( OmniPITR::Program::Monitor::Parser ); 18 | sub handle_line { ... } 19 | 20 | =head1 DESCRIPTION 21 | 22 | This is base class for parsers of lines from particular omnipitr programs. 23 | 24 | =head1 CONTROL FLOW 25 | 26 | When omnipitr-monitor creates parser object, it doesn't pass any arguments (yet). 27 | 28 | Afterwards, it calls ->setup() function, passing (as hash): 29 | 30 | =over 31 | 32 | =item * state - hashref with current state - all modifications will be stored by omnipitr-monitor 33 | 34 | =item * log - log object 35 | 36 | =back 37 | 38 | For each line from given program, ->handle_line() method will be called, with single argument, being hashref with keys: 39 | 40 | =over 41 | 42 | =item * timestamp - timestamp, as it was written in the log line 43 | 44 | =item * epoch - same timestamp, but converted to epoch format 45 | 46 | =item * line - data logged by actual program, with all prefixes removed 47 | 48 | =item * pid - process id of the process that logged given line 49 | 50 | =back 51 | 52 | =head1 METHODS 53 | 54 | =head2 new() 55 | 56 | Object constructor. No logic in here. Just makes simple hashref based object. 57 | 58 | =cut 59 | 60 | sub new { 61 | my $class = shift; 62 | my $self = bless {}, $class; 63 | $class =~ s/.*:://; 64 | $self->{ 'class' } = $class; 65 | return $self; 66 | } 67 | 68 | =head2 setup() 69 | 70 | Sets check for work - receives state-dir and log object from omnipitr-monitor. 71 | 72 | =cut 73 | 74 | sub setup { 75 | my $self = shift; 76 | my %args = @_; 77 | for my $key ( qw( log state ) ) { 78 | croak( "$key not given in call to ->setup()." ) unless defined $args{ $key }; 79 | $self->{ $key } = $args{ $key }; 80 | } 81 | $self->{ 'state' }->{ $self->{ 'class' } } = $self->empty_state() unless defined $self->{ 'state' }->{ $self->{ 'class' } }; 82 | return; 83 | } 84 | 85 | =head1 empty_state 86 | 87 | This method should be overwritten in parsers that assume that their state is something else then hashref. 88 | 89 | =cut 90 | 91 | sub empty_state { 92 | return {}; 93 | } 94 | 95 | =head1 log() 96 | 97 | Shortcut to make code a bit nicer. 98 | 99 | Returns logger object. 100 | 101 | =cut 102 | 103 | sub log { return shift->{ 'log' }; } 104 | 105 | =head1 state() 106 | 107 | Helper function, accessor, to state hash. Or, to be exact, to subhash in state that relates to current parser. 108 | 109 | Has 1, or two arguments. In case of one argument - returns value, from state, for given key. 110 | 111 | If it has two arguments, then - if 2nd argument is undef - it removes the key from state, and returns. 112 | 113 | If the 2nd argument is defined, it sets value for given key to given value, and returns it. 114 | 115 | =cut 116 | 117 | sub state { 118 | my $self = shift; 119 | my $S = $self->{ 'state' }->{ $self->{ 'class' } }; 120 | 121 | return $S if 0 == scalar @_; 122 | 123 | my $key = shift; 124 | return $S->{ $key } if 0 == scalar @_; 125 | 126 | my $value = shift; 127 | return $S->{ $key } = $value if defined $value; 128 | 129 | delete $S->{ $key }; 130 | return; 131 | } 132 | 133 | =head1 split_xlog_filename() 134 | 135 | Splits given xlog filename (24 hex characters) into a pair of timeline and xlog offset. 136 | 137 | Both are trimmed of leading 0s to save space on state saving. 138 | 139 | =cut 140 | 141 | sub split_xlog_filename { 142 | my $self = shift; 143 | my $xlog_name = shift; 144 | 145 | my ( $timeline, @elements ) = unpack( '(A8)3', $xlog_name ); 146 | $timeline =~ s/^0*//; 147 | 148 | $elements[ 0 ] =~ s/^0*//; 149 | $elements[ 1 ] =~ s/^0{6}//; 150 | my $xlog_offset = join '', @elements; 151 | 152 | return ( $timeline, $xlog_offset ); 153 | } 154 | 155 | 1; 156 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Monitor/Parser/Archive.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Monitor::Parser::Archive; 2 | 3 | use strict; 4 | use warnings; 5 | use Carp; 6 | use English qw( -no_match_vars ); 7 | 8 | our $VERSION = '2.0.0'; 9 | use base qw( OmniPITR::Program::Monitor::Parser ); 10 | 11 | =head1 Parser/Archvie state data structure 12 | 13 | Within state->{'Archive'} data is kept using following structure: 14 | 15 | state->{'Archive'}->{ Timeline }->{ Offset } = [ DATA ] 16 | 17 | Where 18 | 19 | =over 20 | 21 | =item * Timeline - leading-zero-trimmed timeline of wal segment 22 | 23 | =item * Offset - offset of wal segment 24 | 25 | =item * DATA - data about archiving this segment 26 | 27 | =back 28 | 29 | For example, data about segment 0000000100008930000000E0 will be in 30 | 31 | state->{Archive}->{1}->{8930E0} 32 | 33 | and for 000000010000012300000005 in 34 | 35 | state->{Archive}->{1}->{12305} 36 | 37 | Please note additional 0 before 5 in last example - it's due to fact that we keep always 2 last characters from wal segment name. 38 | 39 | DATA is arrayref which contains: 40 | 41 | =over 42 | 43 | =item * [0] - epoch of when omnipitr-archive was called, for the first time, for given wal segment 44 | 45 | =item * [1] - epoch of when omnipitr-archive last time delivered the segment 46 | 47 | =back 48 | 49 | =cut 50 | 51 | sub handle_line { 52 | my $self = shift; 53 | my $D = shift; 54 | my $S = $self->state(); 55 | 56 | if ( $D->{ 'line' } =~ m{\ALOG : Called with parameters: .* pg_wal/([a-f0-9]{24})\z}i ) { 57 | my ( $timeline, $xlog_offset ) = $self->split_xlog_filename( $1 ); 58 | $S->{ $timeline }->{ $xlog_offset }->[ 0 ] ||= $D->{ 'epoch' }; 59 | return; 60 | } 61 | 62 | if ( $D->{ 'line' } =~ m{\ALOG : Segment .*/([a-f0-9]{24}) successfully sent to all destinations\.\z}i ) { 63 | my ( $timeline, $xlog_offset ) = $self->split_xlog_filename( $1 ); 64 | $S->{ $timeline }->{ $xlog_offset }->[ 1 ] = $D->{ 'epoch' }; 65 | return; 66 | } 67 | 68 | return; 69 | } 70 | 71 | sub clean_state { 72 | my $self = shift; 73 | my $S = $self->state(); 74 | 75 | my $cutoff = time() - 7 * 24 * 60 * 60; # week ago. 76 | 77 | my @timelines = keys %{ $S }; 78 | 79 | for my $t ( @timelines ) { 80 | my @offsets = keys %{ $S->{ $t } }; 81 | for my $o ( @offsets ) { 82 | next unless $S->{ $t }->{ $o }->[ 1 ]; 83 | next if $S->{ $t }->{ $o }->[ 1 ] >= $cutoff; 84 | delete $S->{ $t }->{ $o }; 85 | } 86 | delete $S->{ $t } if 0 == scalar keys %{ $S->{ $t } }; 87 | } 88 | 89 | return; 90 | } 91 | 92 | 1; 93 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Monitor/Parser/Backup.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Monitor::Parser::Backup; 2 | 3 | use strict; 4 | use warnings; 5 | use Carp; 6 | use English qw( -no_match_vars ); 7 | 8 | our $VERSION = '2.0.0'; 9 | use base qw( OmniPITR::Program::Monitor::Parser ); 10 | 11 | =head1 Backup_Slave/Backup_Master *base* state data structure 12 | 13 | Logs parsing for Backup Slave and Backup Master are virtually the same, so the logic has been moved to shared parent - Backup.pm. 14 | 15 | But the data is stored separately in state->{'Backup_Slave'} and state->{'Backup_Master'} depending on where it came from. 16 | 17 | In all examples below, I write state->{'Backup*'}, and it relates to both places in state. 18 | 19 | Within state->{'Backup*'} data is kept using following structure: 20 | 21 | state->{'Backup*'}->[ n ] = { DATA } 22 | 23 | Where 24 | 25 | =over 26 | 27 | =item * n is just a number, irrelevant. The only important fact is that higher n means that backup started later. 28 | 29 | =item * DATA - data about archiving this segment 30 | 31 | =back 32 | 33 | DATA is arrayref which contains: 34 | 35 | =over 36 | 37 | =item * [0] - epoch of when omnipitr-backup was called 38 | 39 | =item * [1] - process number for omnipitr-backup-* program (pid) 40 | 41 | =item * [2] - epoch when backup was fully done. 42 | 43 | =back 44 | 45 | =cut 46 | 47 | sub handle_line { 48 | my $self = shift; 49 | my $D = shift; 50 | my $S = $self->state(); 51 | 52 | if ( $D->{ 'line' } =~ m{\ALOG : Called with parameters: } ) { 53 | push @{ $S }, [ $D->{ 'epoch' }, $D->{ 'pid' } ]; 54 | return; 55 | } 56 | 57 | if ( $D->{ 'line' } =~ m{\ALOG : All done\.\s*\z} ) { 58 | for my $backup ( @{ $S } ) { 59 | next if $D->{ 'pid' } != $backup->[ 1 ]; 60 | next if defined $backup->[ 2 ]; 61 | $backup->[ 2 ] = $D->{ 'epoch' }; 62 | } 63 | return; 64 | } 65 | 66 | return; 67 | } 68 | 69 | sub empty_state { 70 | return []; 71 | } 72 | 73 | 1; 74 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Monitor/Parser/Backup_Master.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Monitor::Parser::Backup_Master; 2 | 3 | use strict; 4 | use warnings; 5 | use Carp; 6 | use English qw( -no_match_vars ); 7 | 8 | our $VERSION = '2.0.0'; 9 | use base qw( OmniPITR::Program::Monitor::Parser::Backup ); 10 | 11 | 1; 12 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Monitor/Parser/Backup_Slave.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Monitor::Parser::Backup_Slave; 2 | 3 | use strict; 4 | use warnings; 5 | use Carp; 6 | use English qw( -no_match_vars ); 7 | 8 | our $VERSION = '2.0.0'; 9 | use base qw( OmniPITR::Program::Monitor::Parser::Backup ); 10 | 11 | 1; 12 | -------------------------------------------------------------------------------- /lib/OmniPITR/Program/Monitor/Parser/Restore.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Program::Monitor::Parser::Restore; 2 | 3 | use strict; 4 | use warnings; 5 | use Carp; 6 | use English qw( -no_match_vars ); 7 | 8 | our $VERSION = '2.0.0'; 9 | use base qw( OmniPITR::Program::Monitor::Parser ); 10 | 11 | =head1 Parser/Restore state data structure 12 | 13 | Within state->{'Restore'} data is kept using following structure: 14 | 15 | state->{'Restore'}->{ Timeline }->{ Offset } = [ DATA ] 16 | 17 | Where 18 | 19 | =over 20 | 21 | =item * Timeline - leading-zero-trimmed timeline of wal segment 22 | 23 | =item * Offset - offset of wal segment 24 | 25 | =item * DATA - data about archiving this segment 26 | 27 | =back 28 | 29 | For example, data about segment 0000000100008930000000E0 will be in 30 | 31 | state->{Restore}->{1}->{8930E0} 32 | 33 | and for 000000010000012300000005 in 34 | 35 | state->{Restore}->{1}->{12305} 36 | 37 | Please note additional 0 before 5 in last example - it's due to fact that we keep always 2 last characters from wal segment name. 38 | 39 | DATA is arrayref which contains: 40 | 41 | =over 42 | 43 | =item * [0] - epoch of when omnipitr-restore was called, for the first time, for given wal segment 44 | 45 | =item * [1] - epoch of when omnipitr-restore last time delivered the segment 46 | 47 | =back 48 | 49 | =cut 50 | 51 | sub handle_line { 52 | my $self = shift; 53 | my $D = shift; 54 | my $S = $self->state(); 55 | 56 | if ( $D->{ 'line' } =~ m{\ALOG : Called with parameters: .* ([a-f0-9]{24}) pg_wal/RECOVERYXLOG\s*\z}i ) { 57 | my ( $timeline, $xlog_offset ) = $self->split_xlog_filename( $1 ); 58 | $S->{ $timeline }->{ $xlog_offset }->[ 0 ] ||= $D->{ 'epoch' }; 59 | return; 60 | } 61 | 62 | if ( $D->{ 'line' } =~ m{\ALOG : Segment ([a-f0-9]{24}) restored\s*\z}i ) { 63 | my ( $timeline, $xlog_offset ) = $self->split_xlog_filename( $1 ); 64 | $S->{ $timeline }->{ $xlog_offset }->[ 1 ] = $D->{ 'epoch' }; 65 | return; 66 | } 67 | 68 | return; 69 | } 70 | 71 | sub clean_state { 72 | my $self = shift; 73 | my $S = $self->state(); 74 | 75 | my $cutoff = time() - 7 * 24 * 60 * 60; # week ago. 76 | 77 | my @timelines = keys %{ $S }; 78 | 79 | for my $t ( @timelines ) { 80 | my @offsets = keys %{ $S->{ $t } }; 81 | for my $o ( @offsets ) { 82 | next unless $S->{ $t }->{ $o }->[ 1 ]; 83 | next if $S->{ $t }->{ $o }->[ 1 ] >= $cutoff; 84 | delete $S->{ $t }->{ $o }; 85 | } 86 | delete $S->{ $t } if 0 == scalar keys %{ $S->{ $t } }; 87 | } 88 | 89 | return; 90 | } 91 | 92 | 1; 93 | -------------------------------------------------------------------------------- /lib/OmniPITR/Tools.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Tools; 2 | use strict; 3 | use warnings; 4 | use English qw( -no_match_vars ); 5 | use Carp; 6 | use Digest::MD5; 7 | use File::Temp qw( tempfile ); 8 | use base qw( Exporter ); 9 | 10 | our $VERSION = '2.0.0'; 11 | our @EXPORT_OK = qw( file_md5sum run_command ext_for_compression ); 12 | our %EXPORT_TAGS = ( 'all' => \@EXPORT_OK ); 13 | 14 | =head1 ext_for_compression() 15 | 16 | Simple function returning file extension for given compression schema. 17 | 18 | It could be easily handled with hash, but I decided to use function, to 19 | have explicit die/croak in case of bad compression type. 20 | 21 | =cut 22 | 23 | sub ext_for_compression { 24 | my $compression = lc shift; 25 | return '.gz' if $compression eq 'gzip'; 26 | return '.bz2' if $compression eq 'bzip2'; 27 | return '.lzma' if $compression eq 'lzma'; 28 | return '.lz4' if $compression eq 'lz4'; 29 | return '.xz' if $compression eq 'xz'; 30 | croak 'Unknown compression type: ' . $compression; 31 | } 32 | 33 | =head1 file_md5sum() 34 | 35 | Wrapper around Digest::MD5 to calculate md5 sum of file. 36 | 37 | Returned checksum is hex encoded - like output of I program. 38 | 39 | =cut 40 | 41 | sub file_md5sum { 42 | my $filename = shift; 43 | 44 | my $ctx = Digest::MD5->new; 45 | 46 | open my $fh, '<', $filename or croak( sprintf( 'Cannot open file for md5summing %s : %s', $filename, $OS_ERROR ) ); 47 | $ctx->addfile( $fh ); 48 | my $md5 = $ctx->hexdigest(); 49 | close $fh; 50 | 51 | return $md5; 52 | } 53 | 54 | =head1 run_command() 55 | 56 | Runs given program, adding proper escapes of values, and gets stdout and 57 | stderr of it. 58 | 59 | Returns hashref which contains: 60 | 61 | =over 62 | 63 | =item * stderr - stringified stderr output from program 64 | 65 | =item * stdout - stringified stdout output from program 66 | 67 | =item * status - return value of system() call 68 | 69 | =item * error_code - undef in case there was no error, or stringified 70 | error information 71 | 72 | =back 73 | 74 | =cut 75 | 76 | sub run_command { 77 | my ( $temp_dir, @cmd ) = @_; 78 | 79 | my $real_command = join( ' ', map { quotemeta } @cmd ); 80 | 81 | my ( $stdout_fh, $stdout_filename ) = tempfile( 'stdout.XXXXXX', 'DIR' => $temp_dir ); 82 | my ( $stderr_fh, $stderr_filename ) = tempfile( 'stderr.XXXXXX', 'DIR' => $temp_dir ); 83 | 84 | $real_command .= sprintf ' 2>%s >%s', quotemeta $stderr_filename, quotemeta $stdout_filename; 85 | 86 | my $reply = {}; 87 | $reply->{ 'status' } = system $real_command; 88 | local $/ = undef; 89 | $reply->{ 'stdout' } = <$stdout_fh>; 90 | $reply->{ 'stderr' } = <$stderr_fh>; 91 | 92 | close $stdout_fh; 93 | close $stderr_fh; 94 | 95 | unlink( $stdout_filename, $stderr_filename ); 96 | 97 | if ( $CHILD_ERROR == -1 ) { 98 | $reply->{ 'error_code' } = $OS_ERROR; 99 | } 100 | elsif ( $CHILD_ERROR & 127 ) { 101 | $reply->{ 'error_code' } = sprintf "child died with signal %d, %s coredump\n", ( $CHILD_ERROR & 127 ), ( $CHILD_ERROR & 128 ) ? 'with' : 'without'; 102 | } 103 | else { 104 | $reply->{ 'error_code' } = $CHILD_ERROR >> 8; 105 | } 106 | 107 | return $reply; 108 | } 109 | 110 | 1; 111 | -------------------------------------------------------------------------------- /lib/OmniPITR/Tools/CommandPiper.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Tools::CommandPiper; 2 | use strict; 3 | use warnings; 4 | use English qw( -no_match_vars ); 5 | use Data::Dumper; 6 | use File::Spec; 7 | use File::Temp qw( tempdir ); 8 | 9 | =head1 NAME 10 | 11 | OmniPITR::Tools::CommandPiper - Class for building complex pipe-based shell commands 12 | 13 | =cut 14 | 15 | our $VERSION = '2.0.0'; 16 | 17 | =head1 SYNOPSIS 18 | 19 | General usage is: 20 | 21 | my $run = OmniPITR::Tools::CommandPiper->new( 'ls', '-l' ); 22 | $run->add_stdout_file( '/tmp/z.txt' ); 23 | $run->add_stdout_file( '/tmp/y.txt' ); 24 | my $checksummer = $run->add_stdout_program( 'md5sum', '-' ); 25 | $checksummer->add_stdout_file( '/tmp/checksum.txt' ); 26 | 27 | system( $run->command() ); 28 | 29 | Will run: 30 | 31 | mkfifo /tmp/CommandPiper-26195-oCZ7Sw/fifo-0 32 | md5sum - < /tmp/CommandPiper-26195-oCZ7Sw/fifo-0 > /tmp/checksum.txt & 33 | ls -l | tee /tmp/z.txt /tmp/y.txt > /tmp/CommandPiper-26195-oCZ7Sw/fifo-0 34 | wait 35 | rm /tmp/CommandPiper-26195-oCZ7Sw/fifo-0 36 | 37 | While it might look like overkill for something that could be achieved by: 38 | 39 | ls -l | tee /tmp/z.txt /tmp/y.txt | md5sum - > /tmp/checksum.txt 40 | 41 | the way it works is beneficial for cases with multiple different redirections. 42 | 43 | For example - it works great for taking single backup, compressing it with 44 | two different tools, saving it to multiple places, including delivering it 45 | via ssh tunnel to file on remote server. All while taking checksums, and 46 | also delivering them to said locations. 47 | 48 | =head1 DESCRIPTION 49 | 50 | It is important to note that to make the final shell command (script) work, it should be run within shell that has access to: 51 | 52 | =over 53 | 54 | =item * mkfifo 55 | 56 | =item * rm 57 | 58 | =item * tee 59 | 60 | =back 61 | 62 | commands. These are standard on all known to me Unix-alike systems, so it 63 | shouldn't be a problem. 64 | 65 | =head1 MODULE VARIABLES 66 | 67 | =head2 $fifo_dir 68 | 69 | Temporary directory used to store all the fifos. It takes virtually no disk 70 | space, so it can be created anywhere. 71 | 72 | Thanks to L logic, the directory will be removed when the 73 | program will end. 74 | 75 | =cut 76 | 77 | our $fifo_dir = tempdir( 'CommandPiper-' . $$ . '-XXXXXX', 'CLEANUP' => 1, 'TMPDIR' => 1 ); 78 | 79 | =head1 METHODS 80 | 81 | =head2 new() 82 | 83 | Object constructor. 84 | 85 | Given options are treated as program that is run to generate stdout. 86 | 87 | =cut 88 | 89 | sub new { 90 | my $class = shift; 91 | my $self = bless {}, $class; 92 | $self->{ 'tee' } = 'tee'; 93 | $self->{ 'write_mode' } = '>'; 94 | $self->{ 'cmd' } = [ @_ ]; 95 | $self->{ 'stdout_files' } = []; 96 | $self->{ 'stdout_programs' } = []; 97 | $self->{ 'stderr_files' } = []; 98 | $self->{ 'stderr_programs' } = []; 99 | $self->{ 'fifos' } = []; 100 | return $self; 101 | } 102 | 103 | =head2 set_write_mode() 104 | 105 | Sets whether writes of data should overwrite, or append (> vs. >>) 106 | 107 | Accepted values: 108 | 109 | =over 110 | 111 | =item * overwrite 112 | 113 | =item * append 114 | 115 | =back 116 | 117 | Any other would switch back to default, which is overwrite. 118 | 119 | =cut 120 | 121 | sub set_write_mode { 122 | my $self = shift; 123 | my $want = shift; 124 | $self->{ 'write_mode' } = $want eq 'append' ? '>>' : '>'; 125 | return; 126 | } 127 | 128 | =head2 set_tee_path() 129 | 130 | Sets path to tee program, when using tee is required. 131 | 132 | $program->set_tee_path( '/opt/gnu/bin/tee' ); 133 | 134 | Value of tee path will be automatically copied to all newly created stdout and stderr programs. 135 | 136 | =cut 137 | 138 | sub set_tee_path { 139 | my $self = shift; 140 | $self->{ 'tee' } = shift; 141 | return; 142 | } 143 | 144 | =head2 add_stdout_file() 145 | 146 | Adds another file destination for stdout from current program. 147 | 148 | =cut 149 | 150 | sub add_stdout_file { 151 | my $self = shift; 152 | my $stdout = shift; 153 | push @{ $self->{ 'stdout_files' } }, $stdout; 154 | return; 155 | } 156 | 157 | =head2 add_stdout_program() 158 | 159 | Add another program that should receive stdout from current program, as its (the new program) stdin. 160 | 161 | =cut 162 | 163 | sub add_stdout_program { 164 | my $self = shift; 165 | push @{ $self->{ 'stdout_programs' } }, $self->new_subprogram( @_ ); 166 | return $self->{ 'stdout_programs' }->[ -1 ]; 167 | } 168 | 169 | =head2 add_stderr_file() 170 | 171 | Add another program that should receive stdout from current program, as its (the new program) stdin. 172 | 173 | =cut 174 | 175 | sub add_stderr_file { 176 | my $self = shift; 177 | my $stderr = shift; 178 | push @{ $self->{ 'stderr_files' } }, $stderr; 179 | return; 180 | } 181 | 182 | =head2 add_stderr_program() 183 | 184 | Add another program that should receive stderr from current program, as its (the new program) stdin. 185 | 186 | =cut 187 | 188 | sub add_stderr_program { 189 | my $self = shift; 190 | push @{ $self->{ 'stderr_programs' } }, $self->new_subprogram( @_ ); 191 | return $self->{ 'stderr_programs' }->[ -1 ]; 192 | } 193 | 194 | =head2 new_subprogram() 195 | 196 | Helper function to create sub-programs, inheriting settings 197 | 198 | =cut 199 | 200 | sub new_subprogram { 201 | my $self = shift; 202 | my $sub_program = OmniPITR::Tools::CommandPiper->new( @_ ); 203 | for my $key ( qw( tee write_mode ) ) { 204 | $sub_program->{ $key } = $self->{ $key }; 205 | } 206 | return $sub_program; 207 | } 208 | 209 | =head2 command() 210 | 211 | Returns stringified command that should be ran via "system" that does all the described redirections. 212 | 213 | Alternatively, the command can be written to text file, and run with 214 | 215 | bash /name/of/the/file 216 | 217 | =cut 218 | 219 | sub command { 220 | my $self = shift; 221 | 222 | # Get list of all fifos that are necessary to create, so we can run "mkfifo" on it. 223 | my @fifos = $self->get_all_fifos( 0 ); 224 | 225 | my $fifo_preamble = scalar( @fifos ) ? 'mkfifo ' . join( " ", map { quotemeta( $_->[ 0 ] ) } @fifos ) . "\n" : ''; 226 | 227 | # This loop actually writes (well, appends to the $fifo_preamble variable) fifo'ed commands, like: 228 | # md5sum - < /tmp/CommandPiper-26195-oCZ7Sw/fifo-0 > /tmp/checksum.txt & 229 | for my $fifo ( @fifos ) { 230 | $fifo_preamble .= sprintf "%s &\n", $fifo->[ 1 ]->get_command_with_stdin( $fifo->[ 0 ] ); 231 | } 232 | 233 | # we need to remove the fifos afterwards. 234 | my $fifo_cleanup = scalar( @fifos ) ? 'rm ' . join( " ", map { quotemeta( $_->[ 0 ] ) } @fifos ) . "\n" : ''; 235 | 236 | return $fifo_preamble . $self->get_command_with_stdin() . "\nwait\n" . $fifo_cleanup; 237 | } 238 | 239 | =head2 base_program() 240 | 241 | Helper functions which returns current program, with its arguments, properly 242 | escaped, so that it can be included in shell script/command. 243 | 244 | =cut 245 | 246 | sub base_program { 247 | my $self = shift; 248 | return join( ' ', map { quotemeta $_ } @{ $self->{ 'cmd' } } ); 249 | } 250 | 251 | =head2 get_command_with_stdin() 252 | 253 | This is the most important part of the code. 254 | 255 | In here, there is single line generated that runs current program adding all 256 | necessary stdout and stderr redirections. 257 | 258 | Optional $fifo argument is treated as place where current program should 259 | read it's stdin from. If it's absent, it will read stdin from normal STDIN, 260 | but if it is there, generated command will contain: 261 | 262 | ... < $fifo 263 | 264 | for stdin redirection. 265 | 266 | This redirection is for fifo-reading commands. 267 | 268 | =cut 269 | 270 | sub get_command_with_stdin { 271 | my $self = shift; 272 | my $fifo = shift; 273 | 274 | # start is always the command itself 275 | my $command = $self->base_program(); 276 | 277 | # Now, we add stdin redirection, if it's needed 278 | $command .= ' < ' . quotemeta( $fifo ) if defined $fifo; 279 | 280 | # If these are stderr files, add stderr redirection "2> ..." 281 | if ( 0 < scalar @{ $self->{ 'stderr_files' } } ) { 282 | 283 | # Due to how stderr redirection works, we can't really handle more 284 | # than one stderr file. This is handled in L, which, 285 | # changes multiple stderr files, into single stderr file, which is 286 | # fifo to "tee" which outputs to all the files. 287 | # The croak() is just a sanity check. 288 | croak( "This should never happen. Too many stderr files?!" ) if 1 < scalar @{ $self->{ 'stderr_files' } }; 289 | 290 | # Actually add stderr redirect. 291 | $command .= sprintf ' 2%s %s', $self->{ 'write_mode' }, quotemeta( $self->{ 'stderr_files' }->[ 0 ] ); 292 | } 293 | 294 | # If there are no files to capture stdout - we're done - no sense in 295 | # walking through further logic 296 | return $command if 0 == scalar @{ $self->{ 'stdout_files' } }; 297 | 298 | # If there is just one stdout file, then just add redirect like 299 | # /some/command > file 300 | # or >> file, in case of appending. 301 | if ( 1 == scalar @{ $self->{ 'stdout_files' } } ) { 302 | return sprintf( '%s %s %s', $command, $self->{ 'write_mode' }, quotemeta( $self->{ 'stdout_files' }->[ 0 ] ) ); 303 | } 304 | 305 | # If there are many stdout files, then we need tee. 306 | # This needs to take the final file off the list, so we can: 307 | # ... | tee file1 file2 > file3 308 | # as opposed to: 309 | # ... | tee file1 file2 file3 310 | # since the latter would also output the content to normal STDOUT. 311 | my $final_file = pop @{ $self->{ 'stdout_files' } }; 312 | 313 | # The tee run itself - basically "tee file1 file2 ... file(N-1) > fileN" 314 | my $tee = sprintf '%s%s %s %s %s', 315 | quotemeta( $self->{ 'tee' } ), 316 | $self->{ 'write_mode' } eq '>' ? '' : ' -a', 317 | join( ' ', map { quotemeta( $_ ) } @{ $self->{ 'stdout_files' } } ), 318 | $self->{ 'write_mode' }, 319 | quotemeta( $final_file ); 320 | 321 | return "$command | $tee"; 322 | } 323 | 324 | =head2 get_all_fifos() 325 | 326 | To generate output script we need first to generate all fifos. 327 | 328 | Since the command itself is built from tree-like data structure, we need to 329 | parse it depth-first, and find all cases where fifo is needed, and add it to 330 | list of fifos to be created. 331 | Actual lines to generate "mkfifo .." and "... < fifo" commands are in 332 | L method. 333 | 334 | All stdout and stderr programs (i.e. programs that should receive given 335 | command stdout or stderr) are turned into files (fifos), so after running 336 | get_all_fifos, no command in the whole tree should have any 337 | "stdout_programs" or "stderr_programs". Instead, they will have more 338 | "std*_files", and fifos will be created. 339 | 340 | While processing all the commands down the tree, this method also checks if 341 | given command doesn't have multiple stderr_files. 342 | 343 | Normally you can output stdout to multiple files with: 344 | 345 | /some/command | tee file1 file2 > file3 346 | 347 | but there is no easy (and readable) way to do it with stderr. 348 | 349 | So, if there are many stderr files, new fifo is created which does: 350 | 351 | tee file1 file2 > file3 352 | 353 | and then current command is changed to have only this single fifo as it's 354 | stderr_file. 355 | 356 | =cut 357 | 358 | sub get_all_fifos { 359 | my $self = shift; 360 | 361 | # The id itself is irrelevant, but we need to keep names of generated 362 | # fifos unique. 363 | # So they are stored in temp directory ( $fifo_dir, module variable ), 364 | # and named as: "fifo-n", where n is simply monotonically increasing 365 | # integer. 366 | # $fifo_id is simply information how many fifos have been already 367 | # created across whole command tree. 368 | my $fifo_id = shift; 369 | 370 | # Will contain information about all fifos in current command and all of 371 | # its subcommands (stdout/stderr programs) 372 | my @fifos = (); 373 | 374 | # Recursively call get_all_fifos() for all stdout_programs and 375 | # stderr_programs, to fill the @fifo in correct order (depth first). 376 | for my $sub ( @{ $self->{ 'stdout_programs' } }, @{ $self->{ 'stderr_programs' } } ) { 377 | push @fifos, $sub->get_all_fifos( $fifo_id + scalar @fifos ); 378 | } 379 | 380 | # For every stdout program, make new fifo, attach this program to 381 | # created fifo, and push fifo as stdout_file. 382 | # Entry in stdout_programs gets removed. 383 | while ( my $sub = shift @{ $self->{ 'stdout_programs' } } ) { 384 | my $fifo_name = $self->get_fifo_name( $fifo_id + scalar @fifos ); 385 | push @fifos, [ $fifo_name, $sub ]; 386 | push @{ $self->{ 'stdout_files' } }, $fifo_name; 387 | } 388 | 389 | # Same logic as above, but this time working on stderr. 390 | # This should probably be moved to single loop, but the inner part of 391 | # the while() {} is so small that I don't really care at the moment. 392 | while ( my $sub = shift @{ $self->{ 'stderr_programs' } } ) { 393 | my $fifo_name = $self->get_fifo_name( $fifo_id + scalar @fifos ); 394 | push @fifos, [ $fifo_name, $sub ]; 395 | push @{ $self->{ 'stderr_files' } }, $fifo_name; 396 | } 397 | 398 | # As described in the method documentation - when there are many 399 | # stderr_files, we need to add fifo with tee to multiply the stream. 400 | # This is done here. 401 | if ( 1 < scalar @{ $self->{ 'stderr_files' } } ) { 402 | 403 | # tee should get all, but last one, arguments, as the last one will 404 | # be provided by ">" or ">>" redirect. 405 | my $final_stderr_file = pop @{ $self->{ 'stderr_files' } }; 406 | 407 | # We're creating subprogram for the tee and all (but last one) files 408 | my $stderr_tee = $self->new_subprogram( $self->{ 'tee' }, @{ $self->{ 'stderr_files' } } ); 409 | 410 | # The last file gets attached to tee as stdout file, so in final 411 | # script is will be added as "> file" or ">> file" 412 | $stderr_tee->add_stdout_file( $final_stderr_file ); 413 | 414 | # Generate fifo for the tee command 415 | my $fifo_name = $self->get_fifo_name( $fifo_id + scalar @fifos ); 416 | push @fifos, [ $fifo_name, $stderr_tee ]; 417 | 418 | # Change stderr_files so that there will be just one of them, 419 | # pointing to fifo for tee. 420 | $self->{ 'stderr_files' } = [ $fifo_name ]; 421 | } 422 | 423 | # At the moment @fifos contains information about all fifos that are 424 | # needed by subprograms and by current program too. 425 | return @fifos; 426 | } 427 | 428 | =head2 get_fifo_name() 429 | 430 | Each fifo needs unique name. Method for generation is simple - we're using 431 | predefined $fifo_dir, in which the fifo will be named "fifo-$id". 432 | 433 | This is very simple, but I wanted to keep it separately so that it could be 434 | changed easily in future. 435 | 436 | =cut 437 | 438 | sub get_fifo_name { 439 | my $self = shift; 440 | my $id = shift; 441 | return File::Spec->catfile( $fifo_dir, "fifo-" . $id ); 442 | } 443 | 444 | 1; 445 | -------------------------------------------------------------------------------- /lib/OmniPITR/Tools/ParallelSystem.pm: -------------------------------------------------------------------------------- 1 | package OmniPITR::Tools::ParallelSystem; 2 | use strict; 3 | use warnings; 4 | use Carp qw( croak ); 5 | use File::Temp qw( tempfile ); 6 | use Time::HiRes; 7 | use POSIX qw( :sys_wait_h ); 8 | use English qw( -no_match_vars ); 9 | 10 | =head1 NAME 11 | 12 | OmniPITR::Tools::ParallelSystem - Class for running multiple shell commands in parallel. 13 | 14 | =cut 15 | 16 | our $VERSION = '2.0.0'; 17 | 18 | =head1 SYNOPSIS 19 | 20 | General usage is: 21 | 22 | my $run = OmniPITR::Tools::ParallelSystem( 'max_jobs' => 2, ... ); 23 | $run->add_command( 'command' => [ 'ls', '-l', '/' ] ); 24 | $run->add_command( 'command' => [ 'ls', '-l', '/root' ] ); 25 | $run->run(); 26 | my $results = $run->results(); 27 | 28 | =head1 DESCRIPTION 29 | 30 | ParallelSystem strives to make the task of running in parallel simple, and effective. 31 | 32 | It lets you define any number of commands, set max number of concurrent workers, set startup/finish hooks, and run the whole thing. 33 | 34 | Final ->run() is blocking, and it (temporarily) sets CHLD signal handler to its own code, but it is reset to original value afterwards. 35 | 36 | =head1 INTERNALS 37 | 38 | =head2 new() 39 | 40 | Object constructor. Takes one obligatory argument, and two optional: 41 | 42 | =over 43 | 44 | =item * max_jobs - obligatory integer, >= 1, defines how many workers to run at a time 45 | 46 | =item * on_start - coderef (anonymous sub for example) that will be called, every time new worker is spawned. There will be one argument to the code, and it will be job descriptor. More information 47 | about job descriptors in docs for L method. 48 | 49 | =item * on_finish - coderef (anonymous sub for example) that will be called, every time worker finishes. There will be one argument to the code, and it will be job descriptor. More information 50 | about job descriptors in docs for L method. 51 | 52 | =back 53 | 54 | If there are problems with arguments (max_jobs not given, or bad, or hooks given, but not CODE refs - exception will be raised using Carp::croak(). 55 | 56 | Arguments are passed as hash - both hash and hashref are accepted, so you can both: 57 | 58 | my $run = OmniPITR::Tools::ParallelSystem->new( 59 | 'max_jobs' => 2, 60 | 'on_finish' => sub { call_logging( shift ) }, 61 | ); 62 | 63 | and 64 | 65 | my $run = OmniPITR::Tools::ParallelSystem->new( 66 | { 67 | 'max_jobs' => 2, 68 | 'on_finish' => sub { call_logging( shift ) }, 69 | } 70 | ); 71 | 72 | =cut 73 | 74 | sub new { 75 | my $class = shift; 76 | my $args = ref( $_[ 0 ] ) ? $_[ 0 ] : { @ARG }; 77 | my $self = { 'commands' => [], }; 78 | croak( 'max_jobs not provided' ) unless defined $args->{ 'max_jobs' }; 79 | croak( 'max_jobs is not integer' ) unless $args->{ 'max_jobs' } =~ m{\A\d+\z}; 80 | croak( 'max_jobs is not >= 1!' ) unless $args->{ 'max_jobs' } >= 1; 81 | $self->{ 'max_jobs' } = $args->{ 'max_jobs' }; 82 | for my $hook ( qw( on_start on_finish ) ) { 83 | next unless defined $args->{ $hook }; 84 | croak( "Hook for $hook provided, but is not a code?!" ) unless 'CODE' eq ref( $args->{ $hook } ); 85 | $self->{ $hook } = $args->{ $hook }; 86 | } 87 | return bless $self, $class; 88 | } 89 | 90 | =head2 results() 91 | 92 | Returns arrayref with all job descriptors (check L method docs for details), after all the jobs have been ran. 93 | 94 | =cut 95 | 96 | sub results { 97 | return shift->{ 'results' }; 98 | } 99 | 100 | =head2 add_command() 101 | 102 | Adds new command to queue of things to be run. 103 | 104 | Given argument (both hash and hashref are accepted) is treated as job descriptor. 105 | 106 | To make the whole thing run, the only key needed is "command" - which should be arrayref of command and its arguments. The command cannot require any parsing - it will be passed directly to exec() 107 | syscall. 108 | 109 | There can be more keys in the job descriptor, and all of them will be stored, and passed back in on_start/on_finish hooks, and will be present in ->results() data. 110 | 111 | But, there will be several keys added by OmniPITR::Tools::ParallelSystem itself: 112 | 113 | =over 114 | 115 | =item * Available in all 3 places: on_start and on_finish hooks, and final ->results(): 116 | 117 | =over 118 | 119 | =item * started - exact time when the worker has started. Time is as epoch time with microsecond precision. 120 | 121 | =item * pid - pid of worker process - it will be available in 122 | 123 | =item * stderr - name of temporary file that contains stderr output (in on_start hook), or stderr output from command (in other cases) 124 | 125 | =item * stdout - name of temporary file that contains stdout output (in on_start hook), or stdout output from command (in other cases) 126 | 127 | =back 128 | 129 | =item * Additional information available in all 2 places: on_finish hooks and final ->results(): 130 | 131 | =over 132 | 133 | =item * ended - exact time when the worker has ended - it will be available in on_finish hook, and in results. Time is as epoch time with microsecond precision. 134 | 135 | =item * status - numerical status of worker exit. Rules for understanding the value are in perldoc perlvar - as "CHILD_ERROR" - a.k.a. $? 136 | 137 | =back 138 | 139 | =back 140 | 141 | If application provides more keys to add_command, all of them will be preserverd, and passed back to app in hook calls, and in results output. 142 | 143 | =cut 144 | 145 | sub add_command { 146 | my $self = shift; 147 | my $args = ref( $_[ 0 ] ) ? $_[ 0 ] : { @ARG }; 148 | croak( "No 'command' in given args to add_command?!" ) unless defined $args->{ 'command' }; 149 | push @{ $self->{ 'commands' } }, $args; 150 | return; 151 | } 152 | 153 | =head2 add_commands() 154 | 155 | Simple wrapper to simplify adding multiple commands. 156 | 157 | Input should be array (or arrayref) of hashrefs, each hashref should be valid job descriptor, as described in L docs. 158 | 159 | =cut 160 | 161 | sub add_commands { 162 | my $self = shift; 163 | my $args = 'ARRAY' eq ref( $_[ 0 ] ) ? $_[ 0 ] : \@ARG; 164 | $self->add_command( $_ ) for @{ $args }; 165 | return; 166 | } 167 | 168 | =head2 run() 169 | 170 | Main loop responsible of running commands, and handling end of workers. 171 | 172 | =cut 173 | 174 | sub run { 175 | my $self = shift; 176 | 177 | $self->{ 'previous_chld_handler' } = $SIG{ 'CHLD' }; 178 | 179 | $SIG{ 'CHLD' } = sub { 180 | local ( $OS_ERROR, $CHILD_ERROR ); 181 | my $pid; 182 | while ( ( $pid = waitpid( -1, WNOHANG ) ) > 0 ) { 183 | $self->{ 'finished_workers' }->{ $pid } = { 184 | 'ended' => Time::HiRes::time(), 185 | 'status' => $CHILD_ERROR, 186 | }; 187 | } 188 | }; 189 | 190 | $self->{ 'workers' } = {}; 191 | $self->{ 'finished_workers' } = {}; 192 | 193 | while ( 1 ) { 194 | last if ( 0 == scalar keys %{ $self->{ 'workers' } } ) and ( 0 == scalar keys %{ $self->{ 'finished_workers' } } ) and ( 0 == scalar @{ $self->{ 'commands' } } ); 195 | next if $self->start_new_worker(); 196 | next if $self->handle_finished_workers(); 197 | sleep 1; # this will be cancelled by signal, so the sleep time doesn't matter much. 198 | } 199 | 200 | # The no warnings/use warnings "dance" is a workaround for stupid warnings in perl 5.8 201 | no warnings; 202 | $SIG{ 'CHLD' } = $self->{ 'previous_chld_handler' }; 203 | use warnings; 204 | 205 | # The no warnings/use warnings "dance" is a workaround for stupid warnings in perl 5.8 206 | 207 | delete $self->{ 'previous_chld_handler' }; 208 | return; 209 | } 210 | 211 | =head2 start_new_worker() 212 | 213 | Internal method that does actual starting of new worker (if it can be started, and if there is actual work for it to do). 214 | 215 | Calls on_start hook if needed. 216 | 217 | =cut 218 | 219 | sub start_new_worker { 220 | my $self = shift; 221 | return if scalar( keys %{ $self->{ 'workers' } } ) >= $self->{ 'max_jobs' }; 222 | return if 0 == scalar @{ $self->{ 'commands' } }; 223 | 224 | my $new_command = shift @{ $self->{ 'commands' } }; 225 | 226 | my ( $stdout_fh, $stdout_filename ) = tempfile(); 227 | my ( $stderr_fh, $stderr_filename ) = tempfile(); 228 | 229 | $new_command->{ 'stdout' } = $stdout_filename; 230 | $new_command->{ 'stderr' } = $stderr_filename; 231 | $new_command->{ 'started' } = Time::HiRes::time(); 232 | 233 | my $child_pid = fork(); 234 | 235 | if ( $child_pid ) { 236 | 237 | # it's master 238 | $new_command->{ 'pid' } = $child_pid; 239 | $self->{ 'workers' }->{ $child_pid } = $new_command; 240 | close $stdout_fh; 241 | close $stderr_fh; 242 | if ( $self->{ 'on_start' } ) { 243 | $self->{ 'on_start' }->( $new_command ); 244 | } 245 | return 1; 246 | } 247 | 248 | # worker 249 | 250 | open( STDOUT, '>&', $stdout_fh ); 251 | open( STDERR, '>&', $stderr_fh ); 252 | if ( $new_command->{ 'destination_type' } eq 'pipe' ) { 253 | open my $fh, '<', $new_command->{ 'local_file' } or die 'Cannot read from: ' . $new_command->{ 'local_file' } . ': ' . $OS_ERROR; 254 | open( STDIN, '<&', $fh ); 255 | } 256 | 257 | unless ( exec( @{ $new_command->{ 'command' } } ) ) { 258 | 259 | my $err_msg = $OS_ERROR; 260 | my $str_command = join( ' ', @{ $new_command->{ 'command' } } ); 261 | printf $stderr_fh "Couldn't run: %s : %s\n", $str_command, $err_msg; 262 | 263 | # Current process can't call exit(), as doing so would call destructors on 264 | # all objects, which in turn would remove temporary directory, which 265 | # would remove the stderr-catch file, that is used to communicate to 266 | # main process. 267 | POSIX::_exit(1); 268 | } 269 | } 270 | 271 | =head2 handle_finished_workers() 272 | 273 | Internal method which does necessary work when worker finishes. Reads stdout/stderr files, unlinks temp files, calls on_finish hook. 274 | 275 | =cut 276 | 277 | sub handle_finished_workers { 278 | my $self = shift; 279 | return if 0 == scalar keys %{ $self->{ 'finished_workers' } }; 280 | 281 | my @pids = keys %{ $self->{ 'finished_workers' } }; 282 | 283 | for my $pid ( @pids ) { 284 | my $data = delete $self->{ 'finished_workers' }->{ $pid }; 285 | 286 | # sanity check - this shouldn't ever happen. 287 | next unless $self->{ 'workers' }->{ $pid }; 288 | 289 | my $full_data = delete $self->{ 'workers' }->{ $pid }; 290 | $full_data->{ 'ended' } = $data->{ 'ended' }; 291 | $full_data->{ 'status' } = $data->{ 'status' }; 292 | 293 | for my $file_type ( qw( stdout stderr ) ) { 294 | my $filename = $full_data->{ $file_type }; 295 | if ( open my $fh, '<', $filename ) { 296 | local $/; 297 | $full_data->{ $file_type } = <$fh>; 298 | close $fh; 299 | unlink $filename; 300 | } 301 | } 302 | push @{ $self->{ 'results' } }, $full_data; 303 | 304 | if ( $self->{ 'on_finish' } ) { 305 | $self->{ 'on_finish' }->( $full_data ); 306 | } 307 | } 308 | return 1; 309 | } 310 | 311 | 1; 312 | -------------------------------------------------------------------------------- /t/00-load.t: -------------------------------------------------------------------------------- 1 | #!perl -T 2 | 3 | use Test::More tests => 29; 4 | 5 | BEGIN { 6 | use_ok( 'OmniPITR::Log' ) || print "Bail out on OmniPITR::Log\n"; 7 | use_ok( 'OmniPITR::Pidfile' ) || print "Bail out on OmniPITR::Pidfile\n"; 8 | use_ok( 'OmniPITR::Program' ) || print "Bail out on OmniPITR::Program\n"; 9 | use_ok( 'OmniPITR::Program::Archive' ) || print "Bail out on OmniPITR::Program::Archive\n"; 10 | use_ok( 'OmniPITR::Program::Backup' ) || print "Bail out on OmniPITR::Program::Backup\n"; 11 | use_ok( 'OmniPITR::Program::Backup::Master' ) || print "Bail out on OmniPITR::Program::Backup::Master\n"; 12 | use_ok( 'OmniPITR::Program::Backup::Slave' ) || print "Bail out on OmniPITR::Program::Backup::Slave\n"; 13 | use_ok( 'OmniPITR::Program::Cleanup' ) || print "Bail out on OmniPITR::Program::Cleanup\n"; 14 | use_ok( 'OmniPITR::Program::Monitor' ) || print "Bail out on OmniPITR::Program::Monitor\n"; 15 | use_ok( 'OmniPITR::Program::Monitor::Check' ) || print "Bail out on OmniPITR::Program::Monitor::Check\n"; 16 | use_ok( 'OmniPITR::Program::Monitor::Check::Archive_Queue' ) || print "Bail out on OmniPITR::Program::Monitor::Check::Archive_Queue\n"; 17 | use_ok( 'OmniPITR::Program::Monitor::Check::Current_Archive_Time' ) || print "Bail out on OmniPITR::Program::Monitor::Check::Current_Archive_Time\n"; 18 | use_ok( 'OmniPITR::Program::Monitor::Check::Current_Restore_Time' ) || print "Bail out on OmniPITR::Program::Monitor::Check::Current_Restore_Time\n"; 19 | use_ok( 'OmniPITR::Program::Monitor::Check::Dump_State' ) || print "Bail out on OmniPITR::Program::Monitor::Check::Dump_State\n"; 20 | use_ok( 'OmniPITR::Program::Monitor::Check::Errors' ) || print "Bail out on OmniPITR::Program::Monitor::Check::Errors\n"; 21 | use_ok( 'OmniPITR::Program::Monitor::Check::Last_Archive_Age' ) || print "Bail out on OmniPITR::Program::Monitor::Check::Last_Archive_Age\n"; 22 | use_ok( 'OmniPITR::Program::Monitor::Check::Last_Backup_Age' ) || print "Bail out on OmniPITR::Program::Monitor::Check::Last_Backup_Age\n"; 23 | use_ok( 'OmniPITR::Program::Monitor::Check::Last_Restore_Age' ) || print "Bail out on OmniPITR::Program::Monitor::Check::Last_Restore_Age\n"; 24 | use_ok( 'OmniPITR::Program::Monitor::Parser' ) || print "Bail out on OmniPITR::Program::Monitor::Parser\n"; 25 | use_ok( 'OmniPITR::Program::Monitor::Parser::Archive' ) || print "Bail out on OmniPITR::Program::Monitor::Parser::Archive\n"; 26 | use_ok( 'OmniPITR::Program::Monitor::Parser::Backup' ) || print "Bail out on OmniPITR::Program::Monitor::Parser::Backup\n"; 27 | use_ok( 'OmniPITR::Program::Monitor::Parser::Backup_Master' ) || print "Bail out on OmniPITR::Program::Monitor::Parser::Backup_Master\n"; 28 | use_ok( 'OmniPITR::Program::Monitor::Parser::Backup_Slave' ) || print "Bail out on OmniPITR::Program::Monitor::Parser::Backup_Slave\n"; 29 | use_ok( 'OmniPITR::Program::Monitor::Parser::Restore' ) || print "Bail out on OmniPITR::Program::Monitor::Parser::Restore\n"; 30 | use_ok( 'OmniPITR::Program::Restore' ) || print "Bail out on OmniPITR::Program::Restore\n"; 31 | use_ok( 'OmniPITR::Program::Synch' ) || print "Bail out on OmniPITR::Program::Synch\n"; 32 | use_ok( 'OmniPITR::Tools' ) || print "Bail out on OmniPITR::Tools\n"; 33 | use_ok( 'OmniPITR::Tools::CommandPiper' ) || print "Bail out on OmniPITR::Tools::CommandPiper\n"; 34 | use_ok( 'OmniPITR::Tools::ParallelSystem' ) || print "Bail out on OmniPITR::Tools::ParallelSystem\n"; 35 | } 36 | 37 | diag( "Testing OmniPITR $OmniPITR::Program::VERSION, Perl $], $^X" ); 38 | -------------------------------------------------------------------------------- /t/10-perl-critic.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use strict; 3 | use warnings; 4 | use File::Spec; 5 | use Test::More; 6 | use English qw(-no_match_vars); 7 | use File::Find; 8 | 9 | eval { require Test::Perl::Critic; }; 10 | 11 | if ( $EVAL_ERROR ) { 12 | my $msg = 'Test::Perl::Critic required to criticise code'; 13 | plan( skip_all => $msg ); 14 | } 15 | 16 | my $rcfile = File::Spec->catfile( 't', 'perlcriticrc' ); 17 | Test::Perl::Critic->import( -profile => $rcfile ); 18 | 19 | my @files = (); 20 | find( 21 | sub { 22 | return unless -f; 23 | return unless /\.pm\z/; 24 | return if $File::Find::name eq 'lib/Pg/SQL/Parser/SQL.pm'; 25 | push @files, $File::Find::name; 26 | }, 27 | 'lib/' 28 | ); 29 | 30 | plan tests => scalar @files; 31 | critic_ok( $_ ) for @files; 32 | -------------------------------------------------------------------------------- /t/20-perltidy.t: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | use strict; 3 | use warnings; 4 | use File::Spec; 5 | use Test::More; 6 | use English qw(-no_match_vars); 7 | 8 | eval { require Test::PerlTidy; import Test::PerlTidy; }; 9 | 10 | if ( $EVAL_ERROR ) { 11 | my $msg = 'Test::Tidy required to criticise code'; 12 | plan( skip_all => $msg ); 13 | } 14 | 15 | my $rcfile = File::Spec->catfile( 't', 'perltidyrc' ); 16 | run_tests( 17 | perltidyrc => $rcfile, 18 | exclude => [ qr{\.t$}, qr{^blib/}, qr{^lib/Pg/SQL/Parser/SQL\.pm$}, qr{^lib/Pg/SQL/Parser/Lexer/Keywords\.pm$}, qr{^t/(?:07-parser|06-lexer)-data/.*\.pl$} ], 19 | ); 20 | -------------------------------------------------------------------------------- /t/perlcriticrc: -------------------------------------------------------------------------------- 1 | severity = 5 2 | include = RequireCarping 3 | -------------------------------------------------------------------------------- /t/perltidyrc: -------------------------------------------------------------------------------- 1 | --backup-file-extension=beforeTidy 2 | --block-brace-tightness=0 3 | --brace-tightness=0 4 | --closing-token-indentation=1 5 | --continuation-indentation=4 6 | --indent-columns=4 7 | --maximum-line-length=195 8 | --nocuddled-else 9 | --noopening-brace-on-new-line 10 | --nooutdent-labels 11 | --paren-tightness=0 12 | --square-bracket-tightness=0 13 | --vertical-tightness=0 14 | --vertical-tightness-closing=0 15 | --break-at-old-comma-breakpoints 16 | -------------------------------------------------------------------------------- /test/.gitignore: -------------------------------------------------------------------------------- 1 | data-* 2 | omnipitr 3 | -------------------------------------------------------------------------------- /test/run.test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export use_user="$( id -u -n )" 4 | export work_dir="$( pwd )" 5 | 6 | cd "$( dirname "${BASH_SOURCE[0]}" )" 7 | export test_dir="$( pwd )" 8 | 9 | # cd to where current script resides 10 | cd .. 11 | export omnipitr_dir="$( pwd )" 12 | 13 | cd "$test_dir" 14 | 15 | # load functions from test-lib 16 | while read source_file 17 | do 18 | source "$source_file" 19 | done < <( find test-lib/ -type f -name '[0-9]*' | sort -t/ -k2,2 -n ) 20 | 21 | cd "$work_dir" 22 | 23 | identify_current_pg_version 24 | 25 | setup_output_formatting 26 | 27 | echo "! Running test on $pg_version" 28 | 29 | prepare_test_environment 30 | 31 | echo "> Making master" 32 | 33 | make_master 34 | 35 | echo "< Master ready" 36 | 37 | echo "> Check dst-pipe in omnipitr-archive" 38 | 39 | verify_archived_xlogs 40 | 41 | echo "< Check dst-pipe in omnipitr-archive" 42 | 43 | echo "> Making backup off master" 44 | 45 | make_master_backup 46 | 47 | echo "< Master backup done and looks ok" 48 | 49 | echo "> Making backup off master (without xlogs)" 50 | 51 | make_master_sx_backup 52 | 53 | echo "< Master backup (without xlogs) done and looks ok" 54 | 55 | echo "> Starting standalone pg from master backup" 56 | 57 | make_standalone master 54002 58 | 59 | echo "< Standalone pg from master backup looks ok" 60 | 61 | echo "> Starting file-based slave out of master backup" 62 | 63 | make_normal_slave master-slave master 54003 64 | 65 | echo "< Slave looks ok." 66 | 67 | echo "> Make backup off normal slave" 68 | 69 | make_slave_backup 70 | 71 | echo "< Backup off slave worked" 72 | 73 | echo "> Make backup off normal slave (without xlogs)" 74 | 75 | make_slave_sx_backup 76 | 77 | echo "< Backup off slave (without xlogs) worked" 78 | 79 | echo "> Starting standalone pg from slave backup" 80 | 81 | make_standalone master-slave 54004 82 | 83 | echo "< Standalone pg from slave backup looks ok" 84 | 85 | echo "> Starting file-based slave out of slave backup" 86 | 87 | make_normal_slave slave-slave master-slave 54005 88 | 89 | echo "< Slave looks ok." 90 | 91 | echo "< Try to promote normal slave" 92 | 93 | test_promotion master-slave 54003 94 | 95 | echo "> Try to promote normal slave" 96 | 97 | echo "< Try to promote slave off slave" 98 | 99 | test_promotion slave-slave 54005 100 | 101 | echo "> Try to promote slave off slave" 102 | 103 | if (( pg_major_version >= 9 )) 104 | then 105 | 106 | echo "Running on Pg 9.0 (or later), testing streaming replication" 107 | 108 | echo "> Starting SR-based slave out of master backup" 109 | 110 | make_sr_slave master-sr-slave master 54006 111 | 112 | echo "< Slave looks ok." 113 | 114 | echo "> Make backup off SR slave" 115 | 116 | make_sr_slave_backup 117 | 118 | echo "< Backup off SR worked" 119 | 120 | echo "> Starting standalone pg from sr slave backup" 121 | 122 | make_standalone master-sr-slave 54007 123 | 124 | echo "< Standalone pg from sr slave backup looks ok" 125 | 126 | echo "> Starting file-based slave out of sr slave backup" 127 | 128 | make_normal_slave slave-of-sr-slave master-sr-slave 54008 129 | 130 | echo "< Slave looks ok." 131 | 132 | echo "> Starting SR-based slave out of slave backup" 133 | 134 | make_sr_slave sr-slave-from-slave master-slave 54009 135 | 136 | echo "< Slave looks ok." 137 | 138 | echo "> Starting SR-based slave out of SR slave backup" 139 | 140 | make_sr_slave sr-slave-from-sr-slave master-sr-slave 54010 141 | 142 | echo "< Slave looks ok." 143 | 144 | echo "< Try to promote normal sr slave" 145 | 146 | test_promotion master-sr-slave 54006 147 | 148 | echo "> Try to promote normal sr slave" 149 | 150 | echo "< Try to promote sr slave off slave" 151 | 152 | test_promotion sr-slave-from-slave 54009 153 | 154 | echo "> Try to promote sr slave off slave" 155 | 156 | echo "< Try to promote sr slave off sr slave" 157 | 158 | test_promotion sr-slave-from-sr-slave 54010 159 | 160 | echo "> Try to promote sr slave off sr slave" 161 | fi 162 | 163 | echo "All done." 164 | -------------------------------------------------------------------------------- /test/tags: -------------------------------------------------------------------------------- 1 | !_TAG_FILE_FORMAT 2 /extended format; --format=1 will not append ;" to lines/ 2 | !_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/ 3 | !_TAG_PROGRAM_AUTHOR Darren Hiebert /dhiebert@users.sourceforge.net/ 4 | !_TAG_PROGRAM_NAME Exuberant Ctags // 5 | !_TAG_PROGRAM_URL http://ctags.sourceforge.net /official site/ 6 | !_TAG_PROGRAM_VERSION 5.9~svn20110310 // 7 | generate_load ./test-lib/01.metainfo /^generate_load() {$/;" f 8 | get_master_sender_location ./test-lib/07.make.sr.slave /^get_master_sender_location() {$/;" f 9 | get_slave_receiver_location ./test-lib/07.make.sr.slave /^get_slave_receiver_location() {$/;" f 10 | identify_current_pg_version ./test-lib/01.metainfo /^identify_current_pg_version() {$/;" f 11 | make_master ./test-lib/02.make.master /^make_master() {$/;" f 12 | make_master_add_plpgsql ./test-lib/02.make.master /^make_master_add_plpgsql() {$/;" f 13 | make_master_backup ./test-lib/03.make.master.backup /^make_master_backup() {$/;" f 14 | make_master_initdb ./test-lib/02.make.master /^make_master_initdb() {$/;" f 15 | make_master_make_load_generator_structs ./test-lib/02.make.master /^make_master_make_load_generator_structs() {$/;" f 16 | make_master_mkdirs ./test-lib/02.make.master /^make_master_mkdirs() {$/;" f 17 | make_normal_slave ./test-lib/05.make.normal.slave /^make_normal_slave() {$/;" f 18 | make_slave_backup ./test-lib/06.make.slave.backup /^make_slave_backup() {$/;" f 19 | make_sr_slave ./test-lib/07.make.sr.slave /^make_sr_slave () {$/;" f 20 | make_sr_slave_backup ./test-lib/08.make.sr.slave.backup /^make_sr_slave_backup() {$/;" f 21 | make_standalone ./test-lib/04.make_standalone /^make_standalone() {$/;" f 22 | prepare_test_environment ./test-lib/01.metainfo /^prepare_test_environment() {$/;" f 23 | setup_output_formatting ./test-lib/01.metainfo /^setup_output_formatting() {$/;" f 24 | stop_load_generators ./test-lib/01.metainfo /^stop_load_generators() {$/;" f 25 | tail_n_grep_with_timeout ./test-lib/01.metainfo /^tail_n_grep_with_timeout() {$/;" f 26 | test_promotion ./test-lib/09.test.promotion /^test_promotion () {$/;" f 27 | -------------------------------------------------------------------------------- /test/test-lib/01.metainfo: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | tail_n_grep_with_timeout() { 4 | timeout=$1 match=$2 tail=$3 perl -e ' 5 | $SIG{ALRM} = sub { exit; }; 6 | alarm $ENV{timeout}; 7 | my $m = $ENV{match}; 8 | my $re = qr{$m}; 9 | my $pid = open my $fh, q{-|}, $ENV{tail} or die "Cannot run tail: $!\n"; 10 | while ( <$fh> ) { 11 | next unless $_ =~ $re; 12 | print; 13 | kill 15, $pid; 14 | close $fh; 15 | exit; 16 | }' 17 | } 18 | 19 | identify_current_pg_version() { 20 | pg_version="$( postgres -V | grep -E 'PostgreSQL.* [0-9]+\.[0-9]+\.[0-9]+[[:space:]]*$' | awk '{print $NF}' | cut -d. -f 1,2 )" 21 | if [[ -z "$pg_version" ]] 22 | then 23 | echo "Unknown PG version!" >&2 24 | exit 1 25 | fi 26 | pg_major_version=${pg_version%%.*} 27 | } 28 | 29 | prepare_test_environment() { 30 | killall postgres postmaster omnipitr-restore &>/dev/null 31 | killall -9 postgres postmaster omnipitr-restore &>/dev/null 32 | rm -rf data-* omnipitr 33 | } 34 | 35 | setup_output_formatting() { 36 | exec > >( perl -MPOSIX=strftime -pe 'print strftime("%Y-%m-%d %H:%M:%S %Z : ", localtime time)' ) 37 | } 38 | 39 | generate_load() { 40 | count="$1" 41 | sql="$( seq 1 100 | sed 's/.*/select fill_me_in(0.05);/'; echo "select remove_some_rows();"; echo "vacuum;" )" 42 | load_generator_pids='' 43 | ps_ppids='' 44 | for i in $( seq 1 $count ) 45 | do 46 | while true 47 | do 48 | psql -p 54001 -qAtX <<< "$sql" 49 | done &> /dev/null & 50 | load_generator_pids="$load_generator_pids $!" 51 | ps_ppids="$ps_ppids --ppid $!" 52 | done &> /dev/null 53 | disown $load_generator_pids 54 | } 55 | 56 | stop_load_generators() { 57 | psql_pids="$( ps $ps_ppids o pid= )" 58 | kill $load_generator_pids $psql_pids &> /dev/null 59 | kill -9 $load_generator_pids $psql_pids &> /dev/null 60 | } 61 | -------------------------------------------------------------------------------- /test/test-lib/02.make.master: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | make_master_mkdirs() { 3 | mkdir data-master omnipitr 4 | mkdir omnipitr/{longarchive,archive,state,backup,tmp,dst-pipe-archive} 5 | mkdir omnipitr/tmp/{archive,backup-master,backup-slave,restore} 6 | export TMPDIR="$(pwd)/omnipitr/tmp" 7 | chmod 700 data-master 8 | } 9 | 10 | create_temporary_script_for_dst_pipe() { 11 | cat > "$work_dir/omnipitr/tmp/helper-dst-pipe-archive.sh" << END_OF_SCRIPT 12 | #!/usr/bin/env bash 13 | cat - > "$work_dir/omnipitr/dst-pipe-archive/\$1" 14 | END_OF_SCRIPT 15 | chmod 755 "$work_dir/omnipitr/tmp/helper-dst-pipe-archive.sh" 16 | } 17 | 18 | make_master_initdb() { 19 | initdb -A trust -E UTF8 -D data-master > /dev/null 20 | perl -pi -e ' 21 | s/\A \s* (?: [#] \s* )? listen_addresses \s* = \s*.*/listen_addresses = \047*\047/x; 22 | s/\A \s* (?: [#] \s* )? port \s* = \s*.*/port = \04754001\047/x; 23 | s/\A \s* (?: [#] \s* )? log_destination \s* = \s*.*/log_destination = \047stderr\047/x; 24 | s/\A \s* (?: [#] \s* )? log_filename \s* = \s*.*/log_filename = \047pg.log\047/x; 25 | s/\A \s* (?: [#] \s* )? silent_mode \s* = \s*.*/silent_mode = on/x; 26 | s/\A \s* (?: [#] \s* )? logging_collector \s* = \s*.*/logging_collector = on/x; 27 | s/\A \s* (?: [#] \s* )? redirect_stderr \s* = \s*.*/redirect_stderr = on/x; 28 | s/\A \s* (?: [#] \s* )? log_min_duration_statement \s* = \s*.*/log_min_duration_statement = 0/x; 29 | s/\A \s* (?: [#] \s* )? log_line_prefix \s* = \s*.*/log_line_prefix = \047\%m \%u\@\%d \%p \%r \047/x; 30 | s/\A \s* (?: [#] \s* )? log_temp_files \s* = \s*.*/log_temp_files = 0/x; 31 | s/\A \s* (?: [#] \s* )? (log_checkpoints|log_connections|log_disconnections|log_lock_waits|archive_mode) \s* = \s*.*/$1 = on/x; 32 | s{\A \s* (?: [#] \s* )? archive_command \s* = \s*.*}{archive_command = \047$ENV{omnipitr_dir}/bin/omnipitr-archive -t $ENV{work_dir}/omnipitr/tmp/archive/ -PJ 5 -l $ENV{work_dir}/omnipitr/log.master -dl gzip=$ENV{work_dir}/omnipitr/archive -dl gzip=$ENV{work_dir}/omnipitr/longarchive -db $ENV{work_dir}/omnipitr/backup-archive/ -dp gzip=$ENV{work_dir}/omnipitr/tmp/helper-dst-pipe-archive.sh -s $ENV{work_dir}/omnipitr/state/ -v "\%p"\047}x; 33 | s/\A \s* (?: [#] \s* )? max_wal_senders \s* = \s*.*/max_wal_senders = 30/x; 34 | s/\A \s* (?: [#] \s* )? wal_level \s* = \s*.*/wal_level = \047hot_standby\047/x; 35 | s/\A \s* (?: [#] \s* )? archive_timeout \s* = \s*.*/archive_timeout = 60/x; 36 | ' data-master/postgresql.conf 37 | 38 | echo "local replication all trust 39 | host replication all 127.0.0.1/32 trust" >> data-master/pg_hba.conf 40 | } 41 | 42 | make_master_add_plpgsql() { 43 | if (( ${pg_version%%.*} < 9 )) 44 | then 45 | psql -d postgres -d template1 -p 54001 -qAtX -c "create language plpgsql" 46 | fi 47 | } 48 | 49 | make_master_make_load_generator_structs() { 50 | echo "\set VERBOSITY terse 51 | set client_min_messages = warning; 52 | create user postgres with superuser; 53 | create user replication with superuser; 54 | create database $use_user with owner $use_user; 55 | \c $use_user 56 | \set VERBOSITY terse 57 | set client_min_messages = warning; 58 | create table t0 (id serial primary key, when_tsz timestamptz default now(), payload text); 59 | create table t1 (id serial primary key, when_tsz timestamptz default now(), payload text); 60 | create table t2 (id serial primary key, when_tsz timestamptz default now(), payload text); 61 | create table t3 (id serial primary key, when_tsz timestamptz default now(), payload text); 62 | create table t4 (id serial primary key, when_tsz timestamptz default now(), payload text); 63 | create table t5 (id serial primary key, when_tsz timestamptz default now(), payload text); 64 | create table t6 (id serial primary key, when_tsz timestamptz default now(), payload text); 65 | create table t7 (id serial primary key, when_tsz timestamptz default now(), payload text); 66 | create table t8 (id serial primary key, when_tsz timestamptz default now(), payload text); 67 | create table t9 (id serial primary key, when_tsz timestamptz default now(), payload text); 68 | create function fill_me_in(float8) returns void as \$\$ 69 | begin 70 | insert into t0 (payload) values ( repeat('payload', 1000 ) ); 71 | perform pg_sleep( \$1 * random() ); 72 | insert into t1 (payload) values ( repeat('payload', 1000 ) ); 73 | perform pg_sleep( \$1 * random() ); 74 | insert into t2 (payload) values ( repeat('payload', 1000 ) ); 75 | perform pg_sleep( \$1 * random() ); 76 | insert into t3 (payload) values ( repeat('payload', 1000 ) ); 77 | perform pg_sleep( \$1 * random() ); 78 | insert into t4 (payload) values ( repeat('payload', 1000 ) ); 79 | perform pg_sleep( \$1 * random() ); 80 | insert into t5 (payload) values ( repeat('payload', 1000 ) ); 81 | perform pg_sleep( \$1 * random() ); 82 | insert into t6 (payload) values ( repeat('payload', 1000 ) ); 83 | perform pg_sleep( \$1 * random() ); 84 | insert into t7 (payload) values ( repeat('payload', 1000 ) ); 85 | perform pg_sleep( \$1 * random() ); 86 | insert into t8 (payload) values ( repeat('payload', 1000 ) ); 87 | perform pg_sleep( \$1 * random() ); 88 | insert into t9 (payload) values ( repeat('payload', 1000 ) ); 89 | perform pg_sleep( \$1 * random() ); 90 | end; 91 | \$\$ language plpgsql; 92 | create function remove_some_rows() returns void as \$\$ 93 | begin 94 | delete from t0 where when_tsz < now() - '10 minutes'::interval; 95 | delete from t1 where when_tsz < now() - '10 minutes'::interval; 96 | delete from t2 where when_tsz < now() - '10 minutes'::interval; 97 | delete from t3 where when_tsz < now() - '10 minutes'::interval; 98 | delete from t4 where when_tsz < now() - '10 minutes'::interval; 99 | delete from t5 where when_tsz < now() - '10 minutes'::interval; 100 | delete from t6 where when_tsz < now() - '10 minutes'::interval; 101 | delete from t7 where when_tsz < now() - '10 minutes'::interval; 102 | delete from t8 where when_tsz < now() - '10 minutes'::interval; 103 | delete from t9 where when_tsz < now() - '10 minutes'::interval; 104 | end; 105 | \$\$ language plpgsql; 106 | SELECT fill_me_in(0); 107 | " | psql -p 54001 -d template1 -qAtX | ( grep -v '^[[:space:]]*$' || true ) 108 | 109 | } 110 | 111 | verify_archived_xlogs() { 112 | while true 113 | do 114 | all_xlogs="$( find "$work_dir/omnipitr/archive" -type f -printf '%f\n' )" 115 | xlog_count="$( echo "$all_xlogs" | wc -l )" 116 | if (( $xlog_count > 4 )) 117 | then 118 | break 119 | fi 120 | sleep 1 121 | done 122 | 123 | # sleep 1 - allow for dst-pipe call 124 | while read -r xlog_filename 125 | do 126 | archive_md5="$( md5sum "$work_dir/omnipitr/archive/$xlog_filename" | cut -d\ -f1 )" 127 | pipe_md5="$( md5sum "$work_dir/omnipitr/dst-pipe-archive/$xlog_filename" | cut -d\ -f1 )" 128 | if [[ $archive_md5 != $pipe_md5 ]] 129 | then 130 | echo "Checking md5sum of $work_dir/omnipitr/archive/$xlog_filename and $work_dir/omnipitr/dst-pipe-archive/$xlog_filename showed difference ?!" >&2 131 | exit 1 132 | fi 133 | done <<< "$all_xlogs" 134 | 135 | } 136 | 137 | make_master() { 138 | make_master_mkdirs 139 | 140 | create_temporary_script_for_dst_pipe 141 | 142 | make_master_initdb 143 | 144 | pg_ctl -D data-master -s -w start 145 | 146 | make_master_add_plpgsql 147 | 148 | make_master_make_load_generator_structs 149 | } 150 | 151 | -------------------------------------------------------------------------------- /test/test-lib/03.make.master.backup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | make_master_backup() { 4 | rm -f $work_dir/omnipitr/tmp/omnipitr-helper-dst-pipe.out 5 | $omnipitr_dir/bin/omnipitr-backup-master -dg MD5,SHA-1 -p 54001 -t $work_dir/omnipitr/tmp/backup-master/ -D $work_dir/data-master/ -x $work_dir/omnipitr/backup-archive/ -dp gzip=$test_dir/test-lib/helper-dst-pipe.sh -dl gzip=$work_dir/omnipitr/backup/ -f "master-__FILETYPE__.tar__CEXT__" -l $work_dir/omnipitr/log -v 6 | 7 | if [[ ! -e $work_dir/omnipitr/backup/master-data.tar.gz ]] 8 | then 9 | echo "$work_dir/omnipitr/backup/master-data.tar.gz does not exist?!" >&2 10 | exit 1 11 | fi 12 | 13 | if [[ ! -e $work_dir/omnipitr/backup/master-xlog.tar.gz ]] 14 | then 15 | echo "$work_dir/omnipitr/backup/master-xlog.tar.gz does not exist?!" >&2 16 | exit 1 17 | fi 18 | 19 | data_size="$( du -k $work_dir/omnipitr/backup/master-data.tar.gz | awk '{print $1}')" 20 | xlog_size="$( du -k $work_dir/omnipitr/backup/master-xlog.tar.gz | awk '{print $1}')" 21 | 22 | if (( $data_size < 1024 )) 23 | then 24 | echo "$work_dir/omnipitr/backup/master-data.tar.gz exists but looks too small to be sensible!" >&2 25 | exit 1 26 | fi 27 | if (( $xlog_size < 8 )) 28 | then 29 | echo "$work_dir/omnipitr/backup/master-xlog.tar.gz exists but looks too small to be sensible!" >&2 30 | exit 1 31 | fi 32 | 33 | different_md5s="$( cat $work_dir/omnipitr/tmp/omnipitr-helper-dst-pipe.out $work_dir/omnipitr/backup/master-MD5.tar.gz | awk '$2~/data|xlog/' | tr '*' ' ' | sort | uniq | wc -l )" 34 | if (( $different_md5s != 2 )) 35 | then 36 | echo "MD5s are not the same in checksum file, and in file generated by dst-pipe?!" >&2 37 | exit 1 38 | fi 39 | 40 | # Rotate xlog to make sure there is something in replication archive. 41 | echo "create table xxx_tmp as select generate_series(1,10) as i; 42 | checkpoint; 43 | select pg_switch_xlog(); 44 | drop table xxx_tmp; 45 | checkpoint; 46 | " | psql -p 54001 &> /dev/null 47 | 48 | } 49 | -------------------------------------------------------------------------------- /test/test-lib/04.make_standalone: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | 4 | 5 | make_standalone() { 6 | source_codename="$1" 7 | use_port="$2" 8 | 9 | mkdir "data-${source_codename}-restored" 10 | chmod 700 "data-${source_codename}-restored" 11 | 12 | tar xzf omnipitr/backup/"${source_codename}"-data.tar.gz -C "data-${source_codename}-restored"/ 13 | tar xzf omnipitr/backup/"${source_codename}"-xlog.tar.gz -C "data-${source_codename}-restored"/ 14 | 15 | mv "data-${source_codename}-restored/data-${source_codename}"/* "data-${source_codename}-restored"/ 16 | rmdir "data-${source_codename}-restored/data-${source_codename}"/ 17 | 18 | export use_port 19 | perl -pi -e ' 20 | s/\A \s* (?: [#] \s* )? port \s* = \s*.*/port = \047$ENV{use_port}\047/x; 21 | s{\A \s* (?: [#] \s* )? archive_command \s* = \s*.*}{archive_command = \047/bin/true\047}x; 22 | ' "data-${source_codename}-restored"/postgresql.conf 23 | 24 | pg_ctl -D "data-${source_codename}-restored"/ -s -w start 25 | 26 | verification="$( psql -F, -qAtX -p $use_port -c ' 27 | select min(count), count(distinct count) from ( 28 | select r, count(*) from ( 29 | select distinct t0.when_tsz as r from t0 30 | union all 31 | select distinct t1.when_tsz as r from t1 32 | union all 33 | select distinct t2.when_tsz as r from t2 34 | union all 35 | select distinct t3.when_tsz as r from t3 36 | union all 37 | select distinct t4.when_tsz as r from t4 38 | union all 39 | select distinct t5.when_tsz as r from t5 40 | union all 41 | select distinct t6.when_tsz as r from t6 42 | union all 43 | select distinct t7.when_tsz as r from t7 44 | union all 45 | select distinct t8.when_tsz as r from t8 46 | union all 47 | select distinct t9.when_tsz as r from t9 48 | ) x group by r) y;' )" 49 | 50 | if [[ "$verification" != "10,1" ]] 51 | then 52 | echo "Something is wrong with standalone from $source_codename backup ($verification)" >&2 53 | exit 1 54 | fi 55 | } 56 | -------------------------------------------------------------------------------- /test/test-lib/05.make.normal.slave: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | make_normal_slave() { 4 | export codename="$1" 5 | backup_source="$2" 6 | export use_port="$3" 7 | directory="data-$codename" 8 | 9 | mkdir "$directory" 10 | chmod 700 "$directory" 11 | 12 | tar xzf omnipitr/backup/"$backup_source"-data.tar.gz -C "$directory/" 13 | 14 | mv "$directory/data-$backup_source"/* "$directory/" 15 | rmdir "$directory/data-$backup_source" 16 | 17 | perl -pi -e ' 18 | s/\A \s* (?: [#] \s* )? port \s* = \s*.*/port = \047$ENV{use_port}\047/x; 19 | s{\A \s* (?: [#] \s* )? archive_command \s* = \s*.*}{archive_command = \047/bin/true\047}x; 20 | s{\A \s* (?: [#] \s* )? hot_standby \s* = \s*.*}{hot_standby = off}x; 21 | ' "$directory/postgresql.conf" 22 | 23 | echo "restore_command = '$omnipitr_dir/bin/omnipitr-restore -t $work_dir/omnipitr/tmp/restore/ -l $work_dir/omnipitr/log.${codename} -p $work_dir/omnipitr/pause.${codename} -s gzip=$work_dir/omnipitr/archive -v -f $work_dir/${directory}.finish %f %p'" >> "$directory"/recovery.conf 24 | 25 | touch omnipitr/log.${codename} 26 | 27 | pg_ctl -D "$directory/" -s start 28 | 29 | last_archived_segment="$( tac omnipitr/log.master | grep -E -o -m 1 'Segment ./pg_wal/[0-9A-F]{24} successfully sent to all destinations.' | tr ' ' / | cut -d/ -f4 )" 30 | 31 | response="$( tail_n_grep_with_timeout 100 "Segment $last_archived_segment restored" "tail -n 50 -f omnipitr/log.${codename}" )" 32 | 33 | if ! ( echo "$response" | grep -q "Segment $last_archived_segment restored" ) 34 | then 35 | echo "Replication didn't start properly! ($last_archived_segment)" >&2 36 | exit 1 37 | fi 38 | 39 | # echo "Replication looks like working ($last_archived_segment restored), checking if it catches new segments" 40 | 41 | psql -p 54001 -qAtX -c "SELECT fill_me_in(1)" > /dev/null & 42 | psql_pid=$! 43 | 44 | next_segment="$( tail_n_grep_with_timeout 100 'Segment.*successfully sent to all destinations.' "tail -n 0 -f omnipitr/log.master" | sed 's/.*Segment/Segment/' | tr ' ' / 2>&1 | cut -d/ -f4 2>&1 )" 45 | kill $psql_pid 2> /dev/null 46 | 47 | if ! ( echo "$next_segment" | grep -Eq "^[0-9A-Fa-f]{24}$" ) 48 | then 49 | echo "Archiving doesn't work on master?!" >&2 50 | exit 1 51 | fi 52 | 53 | response="$( tail_n_grep_with_timeout 100 "Segment $next_segment restored" "tail -n 50 -f omnipitr/log.${codename}" )" 54 | 55 | if ! ( echo "$response" | grep -q "Segment $next_segment restored" ) 56 | then 57 | echo "Hmm .. 100 seconds and $next_segment is not restored? Something is wrong!" >&2 58 | exit 1 59 | fi 60 | 61 | # echo "Segment $next_segment also restored." 62 | 63 | } 64 | -------------------------------------------------------------------------------- /test/test-lib/06.make.slave.backup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | make_slave_backup() { 4 | 5 | generate_load 5 6 | 7 | rm -f $work_dir/omnipitr/tmp/omnipitr-helper-dst-pipe.out 8 | 9 | if (( pg_major_version >= 9 )) 10 | then 11 | $omnipitr_dir/bin/omnipitr-backup-slave -dg MD5,SHA-1 -P 54001 -cm -s gzip=$work_dir/omnipitr/archive -p $work_dir/omnipitr/pause.master-slave -t $work_dir/omnipitr/tmp/backup-slave/ -D $work_dir/data-master-slave/ -dl gzip=$work_dir/omnipitr/backup/ -f "master-slave-__FILETYPE__.tar__CEXT__" -l $work_dir/omnipitr/log-backup-slave -v -dp gzip=$test_dir/test-lib/helper-dst-pipe.sh 12 | else 13 | $omnipitr_dir/bin/omnipitr-backup-slave -dg MD5,SHA-1 -s gzip=$work_dir/omnipitr/archive -p $work_dir/omnipitr/pause.master-slave -t $work_dir/omnipitr/tmp/backup-slave/ -D $work_dir/data-master-slave/ -dl gzip=$work_dir/omnipitr/backup/ -f "master-slave-__FILETYPE__.tar__CEXT__" -l $work_dir/omnipitr/log-backup-slave -v -dp gzip=$test_dir/test-lib/helper-dst-pipe.sh 14 | fi 15 | 16 | stop_load_generators 17 | 18 | if [[ ! -e $work_dir/omnipitr/backup/master-slave-data.tar.gz ]] 19 | then 20 | echo "$work_dir/omnipitr/backup/master-slave-data.tar.gz does not exist?!" >&2 21 | exit 1 22 | fi 23 | 24 | if [[ ! -e $work_dir/omnipitr/backup/master-slave-xlog.tar.gz ]] 25 | then 26 | echo "$work_dir/omnipitr/backup/master-slave-xlog.tar.gz does not exist?!" >&2 27 | exit 1 28 | fi 29 | 30 | different_md5s="$( cat $work_dir/omnipitr/tmp/omnipitr-helper-dst-pipe.out $work_dir/omnipitr/backup/master-slave-MD5.tar.gz | awk '$2~/data|xlog/' | tr '*' ' ' | sort | uniq | wc -l )" 31 | if (( $different_md5s != 2 )) 32 | then 33 | echo "MD5s are not the same in checksum file, and in file generated by dst-pipe?!" >&2 34 | exit 1 35 | fi 36 | 37 | data_size="$( du -k $work_dir/omnipitr/backup/master-slave-data.tar.gz | awk '{print $1}')" 38 | xlog_size="$( du -k $work_dir/omnipitr/backup/master-slave-xlog.tar.gz | awk '{print $1}')" 39 | 40 | if (( $data_size < 1024 )) 41 | then 42 | echo "$work_dir/omnipitr/backup/master-slave-data.tar.gz exists but looks too small to be sensible!" >&2 43 | exit 1 44 | fi 45 | if (( $xlog_size < 8 )) 46 | then 47 | echo "$work_dir/omnipitr/backup/master-slave-xlog.tar.gz exists but looks too small to be sensible!" >&2 48 | exit 1 49 | fi 50 | 51 | } 52 | -------------------------------------------------------------------------------- /test/test-lib/07.make.sr.slave: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | get_slave_receiver_location() { 4 | ps ww o cmd= --ppid "$slave_postmaster_pid" | awk '$0~/^postgres: wal receiver process.*[0-9A-F]+\/[0-9A-F]+ *$/ {split($NF, f, "/"); printf "%08s%08s\n", f[1], f[2]}' 5 | } 6 | 7 | get_master_sender_location() { 8 | ps ww o cmd= --ppid "$( head -n 1 data-master/postmaster.pid )" | awk '$0~/^postgres: wal sender process.*[0-9A-F]+\/[0-9A-F]+ *$/ {split($NF, f, "/"); printf "%08s%08s\n", f[1], f[2]}' | LC_ALL=C sort | tail -n 1 9 | } 10 | 11 | make_sr_slave () { 12 | export codename="$1" 13 | backup_source="$2" 14 | export use_port="$3" 15 | directory="data-$codename" 16 | 17 | mkdir "$directory" 18 | chmod 700 "$directory" 19 | 20 | tar xzf omnipitr/backup/"$backup_source"-data.tar.gz -C "$directory/" 21 | 22 | mv "$directory/data-$backup_source"/* "$directory/" 23 | rmdir "$directory/data-$backup_source" 24 | 25 | perl -pi -e ' 26 | s/\A \s* (?: [#] \s* )? port \s* = \s*.*/port = \047$ENV{use_port}\047/x; 27 | s{\A \s* (?: [#] \s* )? archive_command \s* = \s*.*}{archive_command = \047/bin/true\047}x; 28 | s{\A \s* (?: [#] \s* )? hot_standby \s* = \s*.*}{hot_standby = on}x; 29 | ' "$directory/postgresql.conf" 30 | 31 | echo "restore_command = '$omnipitr_dir/bin/omnipitr-restore -sr -t $work_dir/omnipitr/tmp/sr-restore/ -l $work_dir/omnipitr/log.${codename} -p $work_dir/omnipitr/pause.${codename} -s gzip=$work_dir/omnipitr/archive -v -f $work_dir/${directory}.finish %f %p'" >> "$directory"/recovery.conf 32 | echo "standby_mode = 'on'" >> "$directory"/recovery.conf 33 | echo "primary_conninfo = 'port=54001 user=$use_user'" >> "$directory"/recovery.conf 34 | echo "trigger_file = '$work_dir/${directory}.finish'" >> "$directory"/recovery.conf 35 | 36 | touch omnipitr/log.${codename} 37 | 38 | pg_ctl -D "$directory/" -w -s start 39 | 40 | if [[ ! -e "${directory}/postmaster.pid" ]] 41 | then 42 | echo "$codename Pg didn't start?" >&2 43 | exit 1 44 | fi 45 | slave_postmaster_pid="$( head -n 1 "${directory}/postmaster.pid" )" 46 | 47 | i=0 48 | while true 49 | do 50 | location="$( get_slave_receiver_location )" 51 | if [[ "" != "$location" ]] 52 | then 53 | break 54 | fi 55 | i=$(( $i + 1 )) 56 | if (( $i > 360 )) 57 | then 58 | echo "There is no wal receiver process in $directory after 60 seconds?!" >&2 59 | exit 1 60 | fi 61 | sleep 1 62 | done 63 | 64 | generate_load 5 65 | 66 | sleep 5 67 | 68 | i=0 69 | while true 70 | do 71 | new_location="$( get_slave_receiver_location )" 72 | if [[ "$new_location" != "$location" ]] 73 | then 74 | break 75 | fi 76 | i=$(( $i + 1 )) 77 | if (( $i > 360 )) 78 | then 79 | echo "Wal receiver location didn't change in 60 seconds of load generation!" >&2 80 | exit 1 81 | fi 82 | sleep 1 83 | done 84 | 85 | sleep 5 86 | 87 | stop_load_generators 88 | 89 | i=0 90 | while true 91 | do 92 | master_location="$( get_master_sender_location )" 93 | slave_location="$( get_slave_receiver_location )" 94 | if [[ "$master_location" == "$slave_location" ]] 95 | then 96 | break 97 | fi 98 | i=$(( $i + 1 )) 99 | if (( $i > 720 )) 100 | then 101 | echo "Slave didn't catchup to master in 720 seconds!" >&2 102 | exit 1 103 | fi 104 | sleep 1 105 | done 106 | } 107 | -------------------------------------------------------------------------------- /test/test-lib/08.make.sr.slave.backup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | make_sr_slave_backup() { 4 | 5 | generate_load 5 6 | 7 | mkdir $work_dir/omnipitr/tmp/backup-sr-slave 8 | 9 | $omnipitr_dir/bin/omnipitr-backup-slave -dg MD5,SHA-1 -P 54001 -cm -p $work_dir/omnipitr/pause.master-slave -s gzip=$work_dir/omnipitr/longarchive -t $work_dir/omnipitr/tmp/backup-sr-slave/ -D $work_dir/data-master-sr-slave/ -dl gzip=$work_dir/omnipitr/backup/ -f "master-sr-slave-__FILETYPE__.tar__CEXT__" -l $work_dir/omnipitr/log-backup-sr-slave -v 10 | 11 | stop_load_generators 12 | 13 | if [[ ! -e $work_dir/omnipitr/backup/master-sr-slave-data.tar.gz ]] 14 | then 15 | echo "$work_dir/omnipitr/backup/master-sr-slave-data.tar.gz does not exist?!" >&2 16 | exit 1 17 | fi 18 | 19 | if [[ ! -e $work_dir/omnipitr/backup/master-sr-slave-xlog.tar.gz ]] 20 | then 21 | echo "$work_dir/omnipitr/backup/master-sr-slave-xlog.tar.gz does not exist?!" >&2 22 | exit 1 23 | fi 24 | 25 | 26 | data_size="$( du -k $work_dir/omnipitr/backup/master-sr-slave-data.tar.gz | awk '{print $1}')" 27 | xlog_size="$( du -k $work_dir/omnipitr/backup/master-sr-slave-xlog.tar.gz | awk '{print $1}')" 28 | 29 | if (( $data_size < 1024 )) 30 | then 31 | echo "$work_dir/omnipitr/backup/master-sr-slave-data.tar.gz exists but looks too small to be sensible!" >&2 32 | exit 1 33 | fi 34 | if (( $xlog_size < 8 )) 35 | then 36 | echo "$work_dir/omnipitr/backup/master-sr-slave-xlog.tar.gz exists but looks too small to be sensible!" >&2 37 | exit 1 38 | fi 39 | 40 | } 41 | -------------------------------------------------------------------------------- /test/test-lib/09.test.promotion: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test_promotion () { 4 | export codename="$1" 5 | export use_port="$2" 6 | export directory="data-$codename" 7 | 8 | finish_trigger="$work_dir/${directory}.finish" 9 | 10 | touch "$finish_trigger" 11 | 12 | response="$( tail_n_grep_with_timeout 120 "database system is ready to accept connections" "tail -n 50 -f $directory/log/pg.log" )" 13 | if ! ( echo "$response" | grep -q "database system is ready to accept connections" ) 14 | then 15 | echo "Promotion didn't work for $codename." >&2 16 | exit 1 17 | fi 18 | 19 | verification="$( psql -F, -qAtX -p $use_port -c ' 20 | select min(count), count(distinct count) from ( 21 | select r, count(*) from ( 22 | select distinct t0.when_tsz as r from t0 23 | union all 24 | select distinct t1.when_tsz as r from t1 25 | union all 26 | select distinct t2.when_tsz as r from t2 27 | union all 28 | select distinct t3.when_tsz as r from t3 29 | union all 30 | select distinct t4.when_tsz as r from t4 31 | union all 32 | select distinct t5.when_tsz as r from t5 33 | union all 34 | select distinct t6.when_tsz as r from t6 35 | union all 36 | select distinct t7.when_tsz as r from t7 37 | union all 38 | select distinct t8.when_tsz as r from t8 39 | union all 40 | select distinct t9.when_tsz as r from t9 41 | ) x group by r) y;' )" 42 | 43 | if [[ "$verification" != "10,1" ]] 44 | then 45 | echo "Something is wrong with promoted $codename." >&2 46 | exit 1 47 | fi 48 | } 49 | -------------------------------------------------------------------------------- /test/test-lib/10.make.master-sx.backup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | make_master_sx_backup() { 4 | $omnipitr_dir/bin/omnipitr-backup-master -dg MD5,SHA-1 -p 54001 -t $work_dir/omnipitr/tmp/backup-master/ -D $work_dir/data-master/ -x $work_dir/omnipitr/backup-archive/ -sx -dl gzip=$work_dir/omnipitr/backup/ -f "master-sx-__FILETYPE__.tar__CEXT__" -l $work_dir/omnipitr/log-sx -v 5 | 6 | if [[ ! -e $work_dir/omnipitr/backup/master-sx-data.tar.gz ]] 7 | then 8 | echo "$work_dir/omnipitr/backup/master-sx-data.tar.gz does not exist?!" >&2 9 | exit 1 10 | fi 11 | 12 | if [[ -e $work_dir/omnipitr/backup/master-sx-xlog.tar.gz ]] 13 | then 14 | echo "$work_dir/omnipitr/backup/master-sx-xlog.tar.gz does exist?!" >&2 15 | exit 1 16 | fi 17 | 18 | data_size="$( du -k $work_dir/omnipitr/backup/master-sx-data.tar.gz | awk '{print $1}')" 19 | 20 | if (( $data_size < 1024 )) 21 | then 22 | echo "$work_dir/omnipitr/backup/master-sx-data.tar.gz exists but looks too small to be sensible!" >&2 23 | exit 1 24 | fi 25 | 26 | # Rotate xlog to make sure there is something in replication archive. 27 | echo "create table xxx_tmp as select generate_series(1,10) as i; 28 | checkpoint; 29 | select pg_switch_xlog(); 30 | drop table xxx_tmp; 31 | checkpoint; 32 | " | psql -p 54001 &> /dev/null 33 | 34 | } 35 | -------------------------------------------------------------------------------- /test/test-lib/11.make.slave-sx.backup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | make_slave_sx_backup() { 4 | 5 | generate_load 5 6 | 7 | if (( pg_major_version >= 9 )) 8 | then 9 | $omnipitr_dir/bin/omnipitr-backup-slave -dg MD5,SHA-1 -P 54001 -cm -sx -s gzip=$work_dir/omnipitr/archive -p $work_dir/omnipitr/pause.master-slave -t $work_dir/omnipitr/tmp/backup-slave/ -D $work_dir/data-master-slave/ -dl gzip=$work_dir/omnipitr/backup/ -f "master-slave-sx-__FILETYPE__.tar__CEXT__" -l $work_dir/omnipitr/log-backup-slave-sx -v 10 | else 11 | $omnipitr_dir/bin/omnipitr-backup-slave -dg MD5,SHA-1 -sx -s gzip=$work_dir/omnipitr/archive -p $work_dir/omnipitr/pause.master-slave -t $work_dir/omnipitr/tmp/backup-slave/ -D $work_dir/data-master-slave/ -dl gzip=$work_dir/omnipitr/backup/ -f "master-slave-sx-__FILETYPE__.tar__CEXT__" -l $work_dir/omnipitr/log-backup-slave-sx -v 12 | fi 13 | 14 | stop_load_generators 15 | 16 | if [[ ! -e $work_dir/omnipitr/backup/master-slave-sx-data.tar.gz ]] 17 | then 18 | echo "$work_dir/omnipitr/backup/master-slave-sx-data.tar.gz does not exist?!" >&2 19 | exit 1 20 | fi 21 | 22 | if [[ -e $work_dir/omnipitr/backup/master-slave-sx-xlog.tar.gz ]] 23 | then 24 | echo "$work_dir/omnipitr/backup/master-slave-sx-xlog.tar.gz does not exist?!" >&2 25 | exit 1 26 | fi 27 | 28 | 29 | data_size="$( du -k $work_dir/omnipitr/backup/master-slave-sx-data.tar.gz | awk '{print $1}')" 30 | 31 | if (( $data_size < 1024 )) 32 | then 33 | echo "$work_dir/omnipitr/backup/master-slave-sx-data.tar.gz exists but looks too small to be sensible!" >&2 34 | exit 1 35 | fi 36 | 37 | } 38 | -------------------------------------------------------------------------------- /test/test-lib/helper-dst-pipe.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | export filename="$1" 3 | md5sum - | perl -pe 's/-/$ENV{"filename"}/' >> $TMPDIR/omnipitr-helper-dst-pipe.out 4 | --------------------------------------------------------------------------------