├── LICENSE
├── config-sample.cfg
├── README.pod
└── system_monitoring.pl


/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2009-2012, OmniTI Computer Consulting, Inc.
 2 | 
 3 | Permission to use, copy, modify, and distribute this software and its
 4 | documentation for any purpose, without fee, and without a written agreement
 5 | is hereby granted, provided that the above copyright notice and this
 6 | paragraph and the following two paragraphs appear in all copies.
 7 | 
 8 | IN NO EVENT SHALL OmniTI Computer Consulting, Inc. BE LIABLE TO ANY PARTY
 9 | FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES,
10 | INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
11 | DOCUMENTATION, EVEN IF OmniTI Computer Consulting, Inc. HAS BEEN ADVISED OF
12 | THE POSSIBILITY OF SUCH DAMAGE.
13 | 
14 | OmniTI Computer Consulting, Inc. SPECIFICALLY DISCLAIMS ANY WARRANTIES,
15 | INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
16 | FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN
17 | "AS IS" BASIS, AND OmniTI Computer Consulting, Inc. HAS NO OBLIGATIONS TO
18 | PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
19 | 


--------------------------------------------------------------------------------
/config-sample.cfg:
--------------------------------------------------------------------------------
  1 | # Global configuration, log directory
  2 | GLOBAL.logdir=/var/log/system_monitoring/
  3 | GLOBAL.pidfile=/var/log/system_monitoring/pidfile
  4 | GLOBAL.env.PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
  5 | GLOBAL.env.PGUSER=postgres
  6 | GLOBAL.env.PGDATABASE=template1
  7 | 
  8 | GLOBAL.var.psql               = psql -qAX -F"	" -c
  9 | GLOBAL.var.activity_query     = select now(), * from pg_stat_activity where current_query <> $$<IDLE>$$
 10 | GLOBAL.var.locks_query        = select * FROM pg_locks
 11 | GLOBAL.var.database_query     = SELECT *, pg_database_size(oid) as size FROM pg_database
 12 | GLOBAL.var.indexes_query      = SELECT *, pg_relation_size(indexrelid) as size FROM pg_stat_all_indexes
 13 | GLOBAL.var.indexes_io_query   = SELECT * FROM pg_statio_all_indexes
 14 | GLOBAL.var.tables_query       = SELECT *, pg_table_size(relid) as size, pg_total_relation_size(relid) as total_size FROM pg_stat_all_tables
 15 | GLOBAL.var.tables_io_query    = SELECT * FROM pg_statio_all_tables
 16 | 
 17 | # Checks configuration
 18 | check.iostat.type=persistent
 19 | check.iostat.exec=iostat -kx 5
 20 | 
 21 | check.vmstat.type=persistent
 22 | check.vmstat.exec=vmstat 5
 23 | 
 24 | check.mpstat.type=persistent
 25 | check.mpstat.exec=mpstat 5
 26 | 
 27 | check.ps.type=periodic
 28 | check.ps.exec=ps axo user,pid,ppid,pgrp,%cpu,%mem,rss,lstart,nice,nlwp,sgi_p,cputime,tty,wchan:25,args
 29 | check.ps.interval=30
 30 | 
 31 | check.df.type=periodic
 32 | check.df.exec=df -x tmpfs -x usbfs -l -P
 33 | check.df.interval=60
 34 | 
 35 | check.loadavg.type=periodic
 36 | check.loadavg.exec=< /proc/loadavg
 37 | check.loadavg.interval=10
 38 | 
 39 | check.mem.type=periodic
 40 | check.mem.exec=< /proc/meminfo
 41 | check.mem.interval=10
 42 | 
 43 | check.cpu.type=periodic
 44 | check.cpu.exec=< /proc/stat
 45 | check.cpu.header=/proc/stat content
 46 | check.cpu.interval=10
 47 | 
 48 | check.pg_stat_activity.type=periodic
 49 | check.pg_stat_activity.exec=@psql 'COPY ( @activity_query ) TO STDOUT'
 50 | check.pg_stat_activity.header=!@psql '@activity_query LIMIT 0' | sed -ne '1p'
 51 | check.pg_stat_activity.interval=30
 52 | 
 53 | check.pg_locks.type=periodic
 54 | check.pg_locks.exec=@psql 'COPY ( @locks_query ) TO STDOUT'
 55 | check.pg_locks.header=!@psql '@locks_query limit 0' | sed -ne '1p'
 56 | check.pg_locks.interval=30
 57 | 
 58 | check.pg_stat_database.type=periodic
 59 | check.pg_stat_database.exec=@psql 'COPY ( @database_query ) TO STDOUT'
 60 | check.pg_stat_database.header=!@psql '@database_query LIMIT 0' | sed -ne '1p'
 61 | check.pg_stat_database.interval=300
 62 | 
 63 | check.pg_stat_all_indexes.type=periodic
 64 | check.pg_stat_all_indexes.exec=@psql 'COPY ( @indexes_query ) TO STDOUT'
 65 | check.pg_stat_all_indexes.header=!@psql '@indexes_query LIMIT 0' | sed -ne '1p'
 66 | check.pg_stat_all_indexes.interval=300
 67 | 
 68 | check.pg_stat_all_tables.type=periodic
 69 | check.pg_stat_all_tables.exec=@psql 'COPY ( @tables_query ) TO STDOUT'
 70 | check.pg_stat_all_tables.header=!@psql '@tables_query LIMIT 0' | sed -ne '1p'
 71 | check.pg_stat_all_tables.interval=300
 72 | 
 73 | check.pg_statio_all_indexes.type=periodic
 74 | check.pg_statio_all_indexes.exec=@psql 'COPY ( @indexes_io_query ) TO STDOUT'
 75 | check.pg_statio_all_indexes.header=!@psql '@indexes_io_query LIMIT 0' | sed -ne '1p'
 76 | check.pg_statio_all_indexes.interval=300
 77 | 
 78 | check.pg_statio_all_tables.type=periodic
 79 | check.pg_statio_all_tables.exec=@psql 'COPY ( @tables_io_query ) TO STDOUT'
 80 | check.pg_statio_all_tables.header=!@psql '@tables_io_query LIMIT 0' | sed -ne '1p'
 81 | check.pg_statio_all_tables.interval=300
 82 | 
 83 | # Compress logs older than 1 hour
 84 | check.cleanup.type=periodic
 85 | check.cleanup.interval=300
 86 | check.cleanup.exec=find . -type f -name '*.log' -mmin +60 -exec gzip {} +
 87 | check.cleanup.ignore=1
 88 | 
 89 | # Remove logs older than 2 weeks ( 14 * 24 * 60 )
 90 | check.cleanup2.type=periodic
 91 | check.cleanup2.interval=86400
 92 | check.cleanup2.exec=find . -type f -name '*.log.gz' -mmin +20160 -delete
 93 | check.cleanup2.ignore=1
 94 | 
 95 | # Remove empty directories, left over after removed logfiles
 96 | check.cleanup3.type=periodic
 97 | check.cleanup3.interval=86400
 98 | check.cleanup3.exec=find . -depth -type d -mmin +1440 -delete 2>/dev/null
 99 | check.cleanup3.ignore=1
100 | 


--------------------------------------------------------------------------------
/README.pod:
--------------------------------------------------------------------------------
  1 | =head1 NAME
  2 | 
  3 | system_monitoring.pl - job scheduler with single-second precision
  4 | 
  5 | =head1 SYNOPSIS
  6 | 
  7 |     system_monitoring.pl -d some_config.ini
  8 | 
  9 | =head1 DESCRIPTION
 10 | 
 11 | This tool is not strictly for PostgreSQL, but I use it on virtually any Pg server I have to look after, and I found it very useful.
 12 | 
 13 | You most likely all know tools like Cacti, Ganglia, Munin and so on.
 14 | 
 15 | system.monitoring is similar in the way that it gets data from system. But it's totally different in a way that it doesn't make any graphs. It just stores the data.
 16 | 
 17 | But, addition of new data to be stored is trivial.
 18 | 
 19 | Most generally system_monitoring.pl is simply job scheduler - like cron. But, unlike cron, it is not limited to 1 minute as the shortest interval. It can easily run jobs every second. Or run jobs that never end.
 20 | 
 21 | It will run every command you give it to run, and log its output. That's all. But since it's small, and the logged format is normal plain text, it became extremely useful for me.
 22 | 
 23 | Let's take a look at L<sample config|https://github.com/omniti-labs/system_monitoring/blob/master/config-sample.cfg>.
 24 | 
 25 | At the beginning I set some variables - most importantly - where the logs will go.
 26 | 
 27 | Then I set environment variables that will be used for all "checks" (it can be overridden for every check). And then I set some additional variables which make running checks a bit nicer.
 28 | 
 29 | Then, I have the actual checks.
 30 | 
 31 | There are three kinds of them: persistent (command), periodic command, and periodic file.
 32 | 
 33 | persistent check example looks like this:
 34 | 
 35 |     check.iostat.type=persistent
 36 |     check.iostat.exec=iostat -kx 5
 37 | 
 38 | On start, system_monitoring.pl will run iostat -kx 5, and then will log everything it will output, when it will output, to the iostat logfile.
 39 | 
 40 | Periodic check can looks like this:
 41 | 
 42 |     check.ps.type=periodic
 43 |     check.ps.exec=ps axo user,pid,ppid,pgrp,%cpu,%mem,rss,lstart,nice,nlwp,sgi_p,cputime,tty,wchan:25,args
 44 |     check.ps.interval=30
 45 | 
 46 | This will run, every 30 seconds, this ps command, and log output.
 47 | 
 48 | If first character of exec for check is "&lt;" the rest of exec line is assumed to be file that should be opened (to read), and it's content be copied to log for given check. Hence:
 49 | 
 50 |     check.mem.type=periodic
 51 |     check.mem.exec=< /proc/meminfo
 52 |     check.mem.interval=10
 53 | 
 54 | Will log meminfo every 10 seconds to logs.
 55 | 
 56 | Logs are stored in tree structure which is based on when even happened. Exact filename for check is F<LOGDIR/YYYY/MM/DD/CHECK_NAME-HOUR.log>. With default logdir, current meminfo log is: F</var/log/system_monitoring/2012/01/22/mem-2012-01-22-16.log>
 57 | 
 58 | It looks like this:
 59 | 
 60 |     2012-01-22 16:00:02 UTC BSGMA   MemTotal:     528655576 kB
 61 |     2012-01-22 16:00:02 UTC BSGMA   MemFree:      14354712 kB
 62 |     2012-01-22 16:00:02 UTC BSGMA   Buffers:       1099748 kB
 63 |     2012-01-22 16:00:02 UTC BSGMA   Cached:       465002160 kB
 64 |     2012-01-22 16:00:02 UTC BSGMA   SwapCached:       5304 kB
 65 |     2012-01-22 16:00:02 UTC BSGMA   Active:       426886732 kB
 66 |     2012-01-22 16:00:02 UTC BSGMA   Inactive:     55076184 kB
 67 |     2012-01-22 16:00:02 UTC BSGMA   HighTotal:           0 kB
 68 |     2012-01-22 16:00:02 UTC BSGMA   HighFree:            0 kB
 69 |     2012-01-22 16:00:02 UTC BSGMA   LowTotal:     528655576 kB
 70 |     2012-01-22 16:00:02 UTC BSGMA   LowFree:      14354712 kB
 71 |     2012-01-22 16:00:02 UTC BSGMA   SwapTotal:     2048276 kB
 72 |     2012-01-22 16:00:02 UTC BSGMA   SwapFree:      1902552 kB
 73 |     2012-01-22 16:00:02 UTC BSGMA   Dirty:            5776 kB
 74 |     2012-01-22 16:00:02 UTC BSGMA   Writeback:         236 kB
 75 |     2012-01-22 16:00:02 UTC BSGMA   AnonPages:    15869780 kB
 76 |     2012-01-22 16:00:02 UTC BSGMA   Mapped:       107046492 kB
 77 |     2012-01-22 16:00:02 UTC BSGMA   Slab:         10872504 kB
 78 |     2012-01-22 16:00:02 UTC BSGMA   PageTables:   20694796 kB
 79 |     2012-01-22 16:00:02 UTC BSGMA   NFS_Unstable:        0 kB
 80 |     2012-01-22 16:00:02 UTC BSGMA   Bounce:              0 kB
 81 |     2012-01-22 16:00:02 UTC BSGMA   CommitLimit:  266376064 kB
 82 |     2012-01-22 16:00:02 UTC BSGMA   Committed_AS: 158592164 kB
 83 |     2012-01-22 16:00:02 UTC BSGMA   VmallocTotal: 34359738367 kB
 84 |     2012-01-22 16:00:02 UTC BSGMA   VmallocUsed:    271152 kB
 85 |     2012-01-22 16:00:02 UTC BSGMA   VmallocChunk: 34359466935 kB
 86 |     2012-01-22 16:00:02 UTC BSGMA   HugePages_Total:     0
 87 |     2012-01-22 16:00:02 UTC BSGMA   HugePages_Free:      0
 88 |     2012-01-22 16:00:02 UTC BSGMA   HugePages_Rsvd:      0
 89 |     2012-01-22 16:00:02 UTC BSGMA   Hugepagesize:     2048 kB
 90 |     2012-01-22 16:00:12 UTC BSGPA   MemTotal:     528655576 kB
 91 | 
 92 | And so on.
 93 | 
 94 | As you can see nothing really complicated.
 95 | 
 96 | There are 2 more features. One of them are checks that are configured so that their output is ignored and not logged - this is for cleanup jobs - check last three in example config.
 97 | 
 98 | The other feature are headers.
 99 | 
100 | For example, let's look at pg_stat_database check. The way it's written, it logs data like:
101 | 
102 |     $ tail pg_stat_database-2012-01-22-15.log
103 |     2012-01-22 15:59:06 UTC BSF9w   template1       10      6       t       t       -1      11510   15545653        1663    \N      {=c/postgres,postgres=CTc/postgres}     4689516
104 |     2012-01-22 15:59:06 UTC BSF9w   postgres        10      6       f       t       -1      11510   15253018        1663    \N      \N      42800983660
105 |     2012-01-22 15:59:06 UTC BSF9w   pg_audit        10      6       f       t       -1      11510   15252690        1663    \N      \N      4779628
106 |     2012-01-22 15:59:06 UTC BSF9w   template0       10      6       t       f       -1      11510   3886088170      1663    \N      {=c/postgres,postgres=CTc/postgres}     4583020
107 |     2012-01-22 15:59:06 UTC BSF9w   magicdb 10      6       f       t       -1      11510   3922373574      1663    {"search_path=public, check_postgres, ltree, pgcrypto"} \N      716819345004
108 |     2012-01-22 15:59:37 UTC BSGFA   template1       10      6       t       t       -1      11510   15545653        1663    \N      {=c/postgres,postgres=CTc/postgres}     4689516
109 |     2012-01-22 15:59:37 UTC BSGFA   postgres        10      6       f       t       -1      11510   15253018        1663    \N      \N      42800983660
110 |     2012-01-22 15:59:37 UTC BSGFA   pg_audit        10      6       f       t       -1      11510   15252690        1663    \N      \N      4779628
111 |     2012-01-22 15:59:37 UTC BSGFA   template0       10      6       t       f       -1      11510   3886088170      1663    \N      {=c/postgres,postgres=CTc/postgres}     4583020
112 |     2012-01-22 15:59:37 UTC BSGFA   magicdb 10      6       f       t       -1      11510   3922373574      1663    {"search_path=public, check_postgres, ltree, pgcrypto"} \N      716820033132
113 | 
114 | which is nice, but it would be cool to be able to know what which columns mean.
115 | 
116 | So we have I<check.pg_stat_database.header> line, which makes system_montoring, to log, whenever it rotates logfile (i.e. every hour) to first log header:
117 | 
118 |     $ head -n 1 pg_stat_database-2012-01-22-15.log
119 |     2012-01-22 15:00:01 UTC :h   datname datdba  encoding        datistemplate   datallowconn    datconnlimit    datlastsysoid   datfrozenxid    dattablespace   datconfig       datacl  size
120 | 
121 | And that's about it. The one additional feature is that you can run system_monitoring and request it to show you data for given check for given time (or closest possible time after):
122 | 
123 |     $ ./system_monitoring.pl -s loadavg -t '2012-01-22 18:04:00' config-sample.cfg
124 |     2012-01-22 18:04:00 CET Kg      0.01 0.10 0.12 1/575 20320
125 | 
126 | And that's all. But thanks to its simplicity it's trivial to use the data it logs for some graphing tool, or just read it.
127 | 
128 | =head1 COPYRIGHT
129 | 
130 | The system_monitoring project is Copyright (c) 2012 OmniTI. All rights reserved.
131 | 
132 | 


--------------------------------------------------------------------------------
/system_monitoring.pl:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env perl
  2 | 
  3 | package main;
  4 | use strict;
  5 | use warnings;
  6 | 
  7 | my $program = Monitoring->new();
  8 | $program->run();
  9 | 
 10 | exit;
 11 | 
 12 | package Monitoring;
 13 | use strict;
 14 | use warnings;
 15 | use Carp;
 16 | use English qw( -no_match_vars );
 17 | use Time::HiRes qw( time sleep );
 18 | use Time::Local qw( timelocal );
 19 | use POSIX qw( strftime setsid );
 20 | use Getopt::Long;
 21 | use File::Spec;
 22 | use File::Path qw( mkpath );
 23 | use MIME::Base64;
 24 | use IO::Select;
 25 | use IO::Handle;
 26 | 
 27 | sub new {
 28 |     my $class = shift;
 29 |     return bless {}, $class;
 30 | }
 31 | 
 32 | sub run {
 33 |     my $self = shift;
 34 |     $self->{ 'job_id' } = 1;
 35 | 
 36 |     $self->read_command_line_options();
 37 | 
 38 |     $self->read_config();
 39 | 
 40 |     $self->validate_config();
 41 | 
 42 |     if ( $self->{ 'show_check' } ) {
 43 |         $self->show_data();
 44 |         return;
 45 |     }
 46 | 
 47 |     $self->daemonize();
 48 | 
 49 |     $self->{ 'select' } = IO::Select->new();
 50 |     $self->start_persistent_processes();
 51 | 
 52 |     $self->main_loop();
 53 |     return;
 54 | }
 55 | 
 56 | sub show_data {
 57 |     my $self = shift;
 58 |     croak( sprintf 'Requested check (%s) is not defined in config (%s)!', $self->{ 'show_check' }, $self->{ 'config_file' } ) unless $self->{ 'checks_hash' }->{ $self->{ 'show_check' } };
 59 | 
 60 |     my @files = $self->find_best_files_for_show();
 61 | 
 62 |     for my $use_file ( @files ) {
 63 | 
 64 |         my $input;
 65 |         if ( $use_file =~ m{\.gz\z} ) {
 66 |             open $input, '-|', 'gzip --decompress --stdout ' . quotemeta( $use_file ) or croak( "Cannot gzip-decompress $use_file: $OS_ERROR" );
 67 |         }
 68 |         elsif ( $use_file =~ m{\.bz2\z} ) {
 69 |             open $input, '-|', 'bzip2 --decompress --stdout ' . quotemeta( $use_file ) or croak( "Cannot bzip2-decompress $use_file: $OS_ERROR" );
 70 |         }
 71 |         else {
 72 |             open $input, '<', $use_file or croak( "Cannot open $use_file: $OS_ERROR" );
 73 |         }
 74 | 
 75 |         my $C = $self->{ 'checks_hash' }->{ $self->{ 'show_check' } };
 76 | 
 77 |         my $state           = 0;
 78 |         my $last_time_str   = 0;
 79 |         my $last_time_epoch = 0;
 80 |         my $data_marker;
 81 | 
 82 |         while ( <$input> ) {
 83 |             croak( "Bad line in log: $_" ) unless m{^((\d\d\d\d)-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)) \S+\t(\S+)};
 84 |             my ( $time_str, $line_code, @elements ) = ( $1, $8, $2, $3, $4, $5, $6, $7 );
 85 |             next if $line_code =~ m{\A(?:\?\?|:h)\z};    # ignore ?? and :h
 86 |             $elements[ 1 ]--;                            # month should be 0-11, and not 1-12
 87 | 
 88 |             my $time_epoch = $time_str eq $last_time_str ? $last_time_epoch : timelocal( reverse @elements );
 89 |             $last_time_str   = $time_str;
 90 |             $last_time_epoch = $time_epoch;
 91 | 
 92 |             my $line_marker = $C->{ 'type' } eq 'periodic' ? $line_code : $time_str;
 93 | 
 94 |             if ( $state == 0 ) {
 95 |                 next if $time_epoch < $self->{ 'show_time' };
 96 |                 $state = 1;
 97 |                 print;
 98 |                 $data_marker = $line_marker;
 99 |                 next;
100 |             }
101 |             else {
102 |                 return unless $data_marker eq $line_marker;
103 |                 print;
104 |             }
105 |         }
106 |         close $input;
107 |     }
108 | 
109 |     return;
110 | }
111 | 
112 | sub find_best_files_for_show {
113 |     my $self = shift;
114 | 
115 |     my @use_files = ();
116 | 
117 |     my $use_ts = $self->{ 'show_time' };
118 | 
119 |     while ( $use_ts < time() ) {
120 |         my @t                = localtime( $use_ts );
121 |         my $directory_prefix = strftime( '%Y/%m/%d', @t );
122 |         my $full_directory   = File::Spec->catfile( $self->{ 'logdir' }, $directory_prefix );
123 |         my $file_suffix      = strftime( '-%Y-%m-%d-%H.log', @t );
124 | 
125 |         my $file_name = $self->{ 'show_check' } . $file_suffix;
126 |         my $full_path = File::Spec->catfile( $full_directory, $file_name );
127 | 
128 |         for my $ext ( ( '', '.gz', '.bz2' ) ) {
129 |             next unless -f $full_path . $ext;
130 |             push @use_files, $full_path . $ext;
131 |             return @use_files if 2 == scalar @use_files;
132 |             last;
133 |         }
134 |         $use_ts += 3600;
135 |     }
136 |     croak( "There is no data for this check and this time." ) if 0 == scalar @use_files;
137 |     return @use_files;
138 | }
139 | 
140 | sub daemonize {
141 |     my $self = shift;
142 |     unless ( $self->{ 'daemonize' } ) {
143 |         $self->handle_pidfile();
144 |         return;
145 |     }
146 | 
147 |     $self->handle_pidfile();
148 | 
149 |     #<<<  do not let perltidy touch this
150 |     open( STDIN,  '<', '/dev/null' ) || croak( "can't read /dev/null: $!" );
151 |     open( STDOUT, '>', '/dev/null' ) || croak( "can't write to /dev/null: $!" );
152 |     defined( my $pid = fork() )      || croak( "can't fork: $!" );
153 |     if ( $pid ) {
154 |         # parent
155 |         sleep 1; # time for slave to rewrite pidfile
156 |         exit
157 |     }
158 |     $self->write_pidfile();
159 |     ( setsid() != -1 )               || croak( "Can't start a new session: $!" );
160 |     open( STDERR, '>&', \*STDOUT )   || croak( "can't dup stdout: $!" );
161 |     #>>>
162 |     return;
163 | }
164 | 
165 | sub handle_pidfile {
166 |     my $self = shift;
167 |     return unless $self->{ 'pidfile' };
168 |     $self->verify_existing_pidfile();
169 |     $self->write_pidfile();
170 |     return;
171 | }
172 | 
173 | sub write_pidfile {
174 |     my $self = shift;
175 |     return unless $self->{ 'pidfile' };
176 |     my $pidfilename = $self->{ 'pidfile' };
177 |     open my $pidfile, '>', $pidfilename or croak( "Cannot write to pidfile ($pidfilename): $OS_ERROR\n" );
178 |     print $pidfile $PID . "\n";
179 |     close $pidfile;
180 |     return;
181 | }
182 | 
183 | sub verify_existing_pidfile {
184 |     my $self = shift;
185 |     return unless $self->{ 'pidfile' };
186 |     my $pidfilename = $self->{ 'pidfile' };
187 |     return unless -e $pidfilename;
188 |     croak( "Pidfile ($pidfilename) exists, but is not a file?!\n" ) unless -f $pidfilename;
189 |     open my $pidfile, '<', $pidfilename or croak( "Cannot read from pidfile ($pidfilename): $OS_ERROR\n" );
190 |     my $old_pid_line = <$pidfile>;
191 |     close $pidfile;
192 | 
193 |     croak( "Bad format of pidfile ($pidfilename)!\n" ) unless $old_pid_line =~ m{\A(\d+)\s*\z};
194 |     my $old_pid = $1;
195 |     return if 0 == kill( 0, $old_pid );
196 |     croak( "Old process ($old_pid) still exists!\n" );
197 | }
198 | 
199 | sub main_loop {
200 |     my $self = shift;
201 |     while ( 1 ) {
202 |         $self->{ 'current_time' } = time();
203 |         $self->update_logger_filehandles();
204 | 
205 |         my $timeout = $self->calculate_timeout();
206 |         my @ready   = $self->{ 'select' }->can_read( $timeout );
207 |         for my $fh ( @ready ) {
208 |             $self->handle_read( $fh );
209 |         }
210 |         $self->start_periodic_processes();
211 |     }
212 | }
213 | 
214 | sub handle_read {
215 |     my $self = shift;
216 |     my $fh   = shift;
217 | 
218 |     my $C;
219 |     for my $tmp ( $self->checks ) {
220 |         next unless $tmp->{ 'input' };
221 |         my $tmp_fh = $tmp->{ 'input' };
222 |         next if "$tmp_fh" ne "$fh";    # Stringified reference to io handle
223 |         $C = $tmp;
224 |         last;
225 |     }
226 |     croak( "Data from unknown input?! It shouldn't *ever* happen\n" ) unless $C;
227 | 
228 |     my $read_data = '';
229 |     while ( 1 ) {
230 |         my $buffer;
231 |         my $read_bytes = sysread( $fh, $buffer, 8192 );
232 |         $read_data .= $buffer;
233 |         last if 8192 > $read_bytes;
234 |     }
235 |     $C->{ 'buffer' } .= $read_data unless $C->{ 'ignore' };
236 | 
237 |     if ( '' eq $read_data ) {
238 |         if ( $C->{ 'logduration' } ) {
239 |             $C->{ 'end_time' } = time();
240 |         }
241 |         $self->{ 'select' }->remove( $fh );
242 |         close $fh;
243 |         delete $C->{ 'input' };
244 |         return unless 'periodic' eq $C->{ 'type' };
245 | 
246 |         if ( $C->{ 'next_call' } - $self->{ 'current_time' } < $C->{ 'minwait' } ) {
247 |             $C->{ 'next_call' } = $self->{ 'current_time' } + $C->{ 'minwait' };
248 |         }
249 | 
250 |         # $C->{ 'next_call' } = $self->{ 'current_time' } + $C->{ 'interval' } if $self->{ 'current_time' } < $C->{ 'next_call' };
251 |         $C->{ 'buffer' } .= "\n" if ( defined $C->{ 'buffer' } ) && ( $C->{ 'buffer' } =~ /[^\n]\z/ );
252 |         if ( $C->{ 'logduration' } ) {
253 |             $C->{ 'buffer' } .= sprintf 'Time to finish: %.3fs%s', $C->{ 'end_time' } - $C->{ 'start_time' }, "\n";
254 |         }
255 |         $self->print_log( $C ) unless $C->{ 'ignore' };
256 |         return;
257 |     }
258 | 
259 |     delete $C->{ 'buffer' } if $C->{ 'ignore' };
260 |     $self->print_log( $C ) unless $C->{ 'ignore' };
261 |     return;
262 | }
263 | 
264 | sub print_log {
265 |     my $self = shift;
266 |     my $C    = shift;
267 | 
268 |     my $timestamp = strftime( '%Y-%m-%d %H:%M:%S %Z', localtime( $self->{ 'current_time' } ) );
269 |     my $job_id = $C->{ 'job_id' };
270 |     $job_id = '??' unless defined $job_id;
271 |     my $line_prefix = sprintf( '%s%s%s%s', $timestamp, "\t", $job_id, "\t" );
272 | 
273 |     while ( $C->{ 'buffer' } =~ s{\A([^\n]*\n)}{} ) {
274 |         my $line = $1;
275 |         print { $C->{ 'fh' } } $line_prefix . $line;
276 |     }
277 |     $C->{ 'fh' }->flush();
278 |     return;
279 | }
280 | 
281 | sub set_env_for_check {
282 |     my $self = shift;
283 |     my $C    = shift;
284 | 
285 |     for my $source ( $self, $C ) {
286 |         next unless $source->{ 'env' };
287 |         while ( my ( $key, $value ) = each %{ $source->{ 'env' } } ) {
288 |             if ( 0 == length $value ) {
289 |                 delete $ENV{ $key };
290 |             }
291 |             else {
292 |                 $ENV{ $key } = $value;
293 |             }
294 |         }
295 |     }
296 |     return;
297 | }
298 | 
299 | sub run_check {
300 |     my $self    = shift;
301 |     my $C       = shift;
302 |     my $command = $C->{ 'exec' };
303 | 
304 |     $self->set_env_for_check( $C );
305 | 
306 |     my $mode = '-|';
307 |     $mode = '<' if $command =~ s/\A\s*<\s*//;
308 | 
309 |     if ( $C->{ 'logduration' } ) {
310 |         $C->{ 'start_time' } = time();
311 |     }
312 | 
313 |     open my $fh, $mode, $command or croak( "Cannot open [$command] in mode [$mode]: $OS_ERROR\n" );
314 |     $self->{ 'select' }->add( $fh );
315 |     $C->{ 'input' } = $fh;
316 | 
317 |     my $encoded_job_id = encode_base64( pack( 'N', $self->{ 'job_id' } ) );
318 |     $self->{ 'job_id' }++;
319 | 
320 |     $encoded_job_id =~ s/\s+//g;    # new line characters and anything like
321 |                                     # this is irrelevant
322 |     $encoded_job_id =~ s/=*\z//;    # trailing = are irrelevant, it just fill
323 |                                     # space to 4 bytes
324 |     $encoded_job_id =~ s/\AA*//;    # leading A characters are like leasing 0s
325 |                                     # in numbers. And since we don't really
326 |                                     # care about full base64 (and its
327 |                                     # reversability), we can remove it
328 | 
329 |     $C->{ 'job_id' } = $encoded_job_id;
330 | 
331 |     return;
332 | }
333 | 
334 | sub start_periodic_processes {
335 |     my $self = shift;
336 |     for my $C ( $self->checks ) {
337 |         next unless 'periodic' eq $C->{ 'type' };
338 |         next if defined $C->{ 'input' };
339 |         next if ( defined $C->{ 'next_call' } ) && ( $C->{ 'next_call' } > $self->{ 'current_time' } );
340 |         $C->{ 'next_call' } = $self->{ 'current_time' } + $C->{ 'interval' };
341 |         $self->run_check( $C );
342 |     }
343 |     return;
344 | }
345 | 
346 | sub start_persistent_processes {
347 |     my $self = shift;
348 |     for my $C ( $self->checks ) {
349 |         next unless 'persistent' eq $C->{ 'type' };
350 |         $self->run_check( $C );
351 |     }
352 |     return;
353 | }
354 | 
355 | sub calculate_timeout {
356 |     my $self = shift;
357 | 
358 |     my $nearest = undef;
359 | 
360 |     for my $C ( $self->checks ) {
361 |         next if 'persistent' eq $C->{ 'type' };
362 |         next if defined $C->{ 'input' };
363 |         return 0 unless defined $C->{ 'next_call' };
364 |         if ( defined $nearest ) {
365 |             $nearest = $C->{ 'next_call' } if $C->{ 'next_call' } < $nearest;
366 |         }
367 |         else {
368 |             $nearest = $C->{ 'next_call' };
369 |         }
370 |     }
371 | 
372 |     $nearest = $self->{ 'current_time' } unless defined $nearest;
373 |     my $sleep_time = $nearest - $self->{ 'current_time' };
374 | 
375 |     return $sleep_time < 0.5 ? 0.5 : $sleep_time;    # limit sleep time to 0.5s to avoid too aggresive calls.
376 | }
377 | 
378 | sub update_logger_filehandles {
379 |     my $self = shift;
380 | 
381 |     my $file_suffix = strftime( '-%Y-%m-%d-%H.log', localtime( $self->{ 'current_time' } ) );
382 |     return if ( defined $self->{ 'previous-suffix' } ) && ( $self->{ 'previous-suffix' } eq $file_suffix );
383 |     $self->{ 'previous-suffix' } = $file_suffix;
384 | 
385 |     my $directory_prefix = strftime( '%Y/%m/%d', localtime( $self->{ 'current_time' } ) );
386 |     my $full_directory = File::Spec->catfile( $self->{ 'logdir' }, $directory_prefix );
387 | 
388 |     mkpath( [ $full_directory ], 0, oct( "750" ) ) unless -e $full_directory;
389 | 
390 |     for my $C ( $self->checks ) {
391 |         next if $C->{ 'ignore' };
392 | 
393 |         if ( $C->{ 'fh' } ) {
394 |             close $C->{ 'fh' };
395 |             delete $C->{ 'fh' };
396 |         }
397 | 
398 |         my $full_name = File::Spec->catfile( $full_directory, $C->{ 'name' } . $file_suffix );
399 |         open my $fh, '>>', $full_name or croak( "Cannot write to $full_name: $OS_ERROR\n" );
400 |         $C->{ 'fh' } = $fh;
401 | 
402 |         if (   ( $C->{ 'header' } )
403 |             && ( !-s $full_name ) )
404 |         {
405 | 
406 |             # File is empty
407 |             my $tmp_job_id = $C->{ 'job_id' };
408 |             $C->{ 'job_id' } = ':h';
409 |             my $tmp_buffer = $C->{ 'buffer' };
410 |             $C->{ 'buffer' } = $C->{ 'header' };
411 |             $self->print_log( $C );
412 |             $C->{ 'job_id' } = $tmp_job_id;
413 |             $C->{ 'buffer' } = $tmp_buffer;
414 |         }
415 |     }
416 | 
417 |     return;
418 | }
419 | 
420 | sub checks {
421 |     my $self = shift;
422 |     return @{ $self->{ 'checks' } };
423 | }
424 | 
425 | sub validate_config {
426 |     my $self = shift;
427 | 
428 |     croak( "GLOBAL.logdir was not provided in config!\n" ) unless defined $self->{ 'logdir' };
429 |     croak( "There are no checks to be run!\n" )            unless defined $self->{ 'pre_checks' };
430 | 
431 |     croak( "Cannot chdir to " . $self->{ 'logdir' } . ": $OS_ERROR\n" ) unless chdir $self->{ 'logdir' };
432 | 
433 |     my @checks = ();
434 |     while ( my ( $check, $C ) = each %{ $self->{ 'pre_checks' } } ) {
435 |         $C->{ 'name' } = $check;
436 |         push @checks, $C;
437 | 
438 |         croak( "Bad type " . $C->{ 'type' } . " in check $check!\n" ) unless $C->{ 'type' } =~ m{\A(?:persistent|periodic)\z};
439 |         next unless $C->{ 'type' } eq 'periodic';
440 | 
441 |         croak( "Undefined interval for check $check!\n" ) unless defined $C->{ 'interval' };
442 |         croak( "Bad interval (" . $C->{ 'interval' } . ") in check $check!\n" ) unless $C->{ 'interval' } =~ m{\A[1-9]\d*\z};
443 |         $C->{ 'minwait' } = 0 unless defined $C->{ 'minwait' };
444 |         croak( "Bad minwait (" . $C->{ 'minwait' } . ") in check $check!\n" ) unless $C->{ 'minwait' } =~ m{\A\d+\z};
445 | 
446 |         $self->process_config_vars( $C );
447 | 
448 |         if ( $C->{ 'header' } ) {
449 |             my $header = $C->{ 'header' };
450 |             if ( $header =~ s/^!\s*// ) {
451 |                 $header .= ' 2>&1' unless $header =~ m{\b2>};
452 |                 $self->set_env_for_check( $C );
453 |                 $header = `$header`;
454 |             }
455 |             $header =~ s/\s*\z/\n/;
456 |             $C->{ 'header' } = $header;
457 |         }
458 | 
459 |         # redirect stderr to stdout if it hasn't been redirected in the exec command itself, and it's not just file to read
460 |         next if $C->{ 'exec' } =~ m{\A\s*<};
461 |         next if $C->{ 'exec' } =~ m{\b2>};
462 |         $C->{ 'exec' } .= ' 2>&1';
463 |     }
464 | 
465 |     $self->{ 'checks' } = \@checks;
466 |     $self->{ 'checks_hash' } = { map { ( $_->{ 'name' } => $_ ) } @checks };
467 |     delete $self->{ 'pre_checks' };
468 | 
469 |     return;
470 | }
471 | 
472 | sub process_config_vars {
473 |     my $self = shift;
474 |     my $C    = shift;
475 |     return unless $self->{ 'var_re' };
476 |     my $re = $self->{ 'var_re' };
477 | 
478 |     for my $field ( qw( exec header ) ) {
479 |         next unless $C->{ $field };
480 |         $C->{ $field } =~ s/\@$re/$self->{'var'}->{$1}/eg;
481 |     }
482 |     return;
483 | }
484 | 
485 | sub read_config {
486 |     my $self = shift;
487 | 
488 |     my $config_file_name = $self->{ 'config_file' };
489 | 
490 |     open my $fh, '<', $config_file_name or croak( "Cannot open config file ($config_file_name) : $OS_ERROR\n" );
491 |     while ( my $line = <$fh> ) {
492 |         next if $line =~ m{^\s*#};     # comment
493 |         next if $line =~ m{^\s*\z};    # empty line
494 |         $line =~ s{\A\s*}{};           # removing leading spaces
495 |         $line =~ s{\s*\z}{};           # removing trailing spaces
496 |         if ( $line =~ m{ \A GLOBAL\.(logdir|pidfile) \s* = \s* (\S.*) \z }xmsi ) {
497 |             $self->{ lc $1 } = $2;
498 |             next;
499 |         }
500 |         if ( $line =~ m{ \A GLOBAL\.env\.([^\s=]+) \s* = \s* (.*) \z }xmsi ) {
501 |             $self->{ 'env' }->{ $1 } = $2;
502 |             next;
503 |         }
504 |         if ( $line =~ m{ \A GLOBAL\.var\.([A-Za-z0-9_]+) \s* = \s* (.*) \z }xmsi ) {
505 |             $self->{ 'var' }->{ $1 } = $2;
506 |             next;
507 |         }
508 |         elsif ( $line =~ m{ \A check\.([A-Za-z0-9_]+)\.(type|exec|interval|minwait|header|logduration|ignore) \s* = \s* (\S.*) \z }xmsi ) {
509 |             $self->{ 'pre_checks' }->{ $1 }->{ $2 } = $3;
510 |             next;
511 |         }
512 |         elsif ( $line =~ m{ \A check\.([A-Za-z0-9_]+)\.env\.([^\s=]+) \s* = \s* (.*) \z }xmsi ) {
513 |             $self->{ 'pre_checks' }->{ $1 }->{ 'env' }->{ $2 } = $3;
514 |             next;
515 |         }
516 |         croak( "Unknown line: [ $line ]\n" );
517 |     }
518 |     close $fh;
519 |     return unless $self->{ 'var' };
520 |     my @all_vars = sort { length( $b ) <=> length( $a ) } keys %{ $self->{ 'var' } };
521 |     my $vars_as_string = join '|', @all_vars;
522 |     my $vars_re = qr{($vars_as_string)};
523 |     $self->{ 'var_re' } = $vars_re;
524 |     return;
525 | }
526 | 
527 | sub read_command_line_options {
528 |     my $self = shift;
529 | 
530 |     my $daemonize  = undef;
531 |     my $show_check = undef;
532 |     my $show_time  = undef;
533 |     exit( 1 ) unless GetOptions(
534 |         'daemonize|d' => \$daemonize,
535 |         'show|s=s'    => \$show_check,
536 |         'time|t=s'    => \$show_time,
537 |     );
538 | 
539 |     if ( $show_time ) {
540 |         croak( "-t value has bad format (not YYYY-MM-DD HH:MI:SS)\n" ) unless $show_time =~ m{\A(\d\d\d\d)-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)\z};
541 |         my @elements = ( $1, $2, $3, $4, $5, $6 );
542 |         $elements[ 1 ]--;    # month is 0-11, and not 1-12!
543 |         $show_time = timelocal( reverse @elements );
544 |     }
545 |     croak( "You cannot give -s without -t\n" ) if defined $show_check && !defined $show_time;
546 |     croak( "You cannot give -t without -s\n" ) if defined $show_time  && !defined $show_check;
547 |     $self->{ 'daemonize' }  = $daemonize;
548 |     $self->{ 'show_check' } = $show_check;
549 |     $self->{ 'show_time' }  = $show_time;
550 | 
551 |     croak( "You have to provide name of config file! Check: perldoc $PROGRAM_NAME\n" ) if 0 == scalar @ARGV;
552 |     $self->{ 'config_file' } = shift @ARGV;
553 | 
554 |     return;
555 | }
556 | 
557 | 1;
558 | 
559 | =head1 system_monitoring.pl
560 | 
561 | =head2 USAGE
562 | 
563 |   system_monitoring.pl [-d] <config_file>
564 | 
565 |   system_monitoring.pl -s check -t when <config_file>
566 | 
567 | =head2 DESCRIPTION
568 | 
569 | system_monitoring.pl script is meant to provide single and solution for
570 | logging system data which change more often than it's practical for systems
571 | like cacti/nagios.
572 | 
573 | It is meant to be run on some low-privilege account, and gather the data,
574 | which are partitioned automatically by source, and time, and stored in
575 | simple text files.
576 | 
577 | After running, system_monitor.pl will check config, and if there are no
578 | errors - will start processing checks.
579 | 
580 | All checks work in parallel, so there is no chance single check could lock
581 | whole system_monitoring.pl.
582 | 
583 | When run with -d option, it will daemonize itself and detach from terminal.
584 | 
585 | When run with -s .. -t ... options, it will print values for given (-s)
586 | check for date being given (-t) or closest later.
587 | 
588 | -t value (time to search for) has to be given in format:
589 | 
590 |   YYYY-MM-DD HH:MI:SS
591 | 
592 | For example:
593 | 
594 |   2012-01-23 16:34:56
595 | 
596 | =head2 Configuration file
597 | 
598 | Format of the configuration file is kept as simple as possible, to make this
599 | script very portable - which in this particular case means: no external
600 | (aside from core perl) dependencies.
601 | 
602 | Each line should be one of:
603 | 
604 | =over
605 | 
606 | =item * Comment (starts with #)
607 | 
608 | =item * Empty line (just white space characters)
609 | 
610 | =item * Setting
611 | 
612 | =back
613 | 
614 | Where setting line looks like:
615 | 
616 |     PARAM=value
617 | 
618 | with optional leading, trailing or around "=" whitespace.
619 | 
620 | Recognized parameters are:
621 | 
622 | =over
623 | 
624 | =item * GLOBAL.logdir - full path to log directory
625 | 
626 | =item * GLOBAL.pidfile- full path to file which should contain pid of
627 | currently running system_monitoring.pl
628 | 
629 | =item * GLOBAL.env.* - setting environment variables
630 | 
631 | =item * GLOBAL.var.* - setting variable to be used as expansion in header
632 | and exec lines. For example: GLOBAL.var.psql=/long/path/psql lets you later
633 | use check.XXX.exec=@psql. Variable names are limited to /^[A-Za-z0-9_]$/
634 | 
635 | =item * check.XXX.type - type of check with name XXX
636 | 
637 | =item * check.XXX.exec - what should be executed to get data for check XXX
638 | 
639 | =item * check.XXX.header - whenever first write to new file for given check
640 | is done, it should be printed first. If header value starts with ! it is
641 | treated (sans the ! character) as command to run that will output header. It
642 | has to be noted, though, that it's locking call - but it's only evaluated
643 | once - at the startup of monitoring script (this is intentional to 
644 | 
645 | =item * check.XXX.interval - how often to run check XXX
646 | 
647 | =item * check.XXX.minwait - minimal wait for next run of the check
648 | 
649 | =item * check.XXX.logduration - if set to true value, system_monitoring will
650 | log how long it took for the command to finish
651 | 
652 | =item * check.XXX.ignore - should output be ignored?
653 | 
654 | =item * check.XXX.env.* - setting environment variables
655 | 
656 | =back
657 | 
658 | There are only two supported types:
659 | 
660 | =over
661 | 
662 | =item * persistent - which means given program is to be run in background,
663 | and whatever it will return should be logged. Such program "interval" will
664 | be ignored.
665 | 
666 | =item * periodic - which means that given program is to be run periodically
667 | as it will exit after returning data
668 | 
669 | =back
670 | 
671 | env parameters are used to set environment variables. You can set them
672 | globally for all checks, via GLOBAL.env., or for any given check itself -
673 | using check.XXX.env.
674 | 
675 | For example:
676 | 
677 |     GLOBAL.env.PGUSER=postgres
678 | 
679 | Will set environment variable PGUSER to value postgres.
680 | 
681 | If you'd want to make sure that given env variable is not set, you can use
682 | syntax with lack of value:
683 | 
684 |     check.whatever.env.PGUSER=
685 | 
686 | "exec" parameter is simply command line, to be run via shell, that will run
687 | the program.
688 | 
689 | If exec parameter starts with '<' character (with optional whitespace
690 | characters after), it is treated as filename to be read, and logged.
691 | 
692 | Due to the way it is internally processed - using "<" approach makes sense
693 | only for periodic checks - in case of permenent checks it would simply copy
694 | the file at start of system_monitoring.pl, and ignore any changes to it
695 | afterwards. If you'd like to have something like 'tail -f' - use tail -f.
696 | 
697 | interval is time (in seconds) how often given program (of periodic type)
698 | should be run.
699 | 
700 | If job takes 25 seconds, and it scheduled to be run every 30 seconds, it
701 | will pause 5 seconds between end of previous iteration and starting new one,
702 | unless minwait parameter will be set to some higher value.
703 | 
704 | ignore is optional parameter which is checked using Perl boolean logic (any
705 | value other than empty string or 0 ar treated as true). Since
706 | system_monitoring doesn't let setting empty string as value for option -
707 | it's best to not include ignore option for checks you want to log, and just
708 | add '...ignore=1' for those that you want to ignore.
709 | 
710 | If ignore is set, system_monitoring will not log output from such check.
711 | 
712 | This is helpful to build-in compression of older logs, using for example:
713 | 
714 |     check.cleanup.type=periodic
715 |     check.cleanup.interval=300
716 |     check.cleanup.exec=find /var/log/monitoring -type f -name '*.log' -mmin +120 -print0 | xargs -0 gzip
717 |     check.cleanup.ignore=1
718 | 
719 | "XXX" (name of check) can consist only of upper and lower case letters,
720 | digits, and character _. That is it has to match regular expression:
721 | 
722 |     /\A[A-Za-z0-9_]+\z/
723 | 
724 | Output from all programs will be logged in files named:
725 | 
726 |     /logdir/YYYY/MM/DD/XXX-YYY-MM-DD-HH.log
727 | 
728 | where YYYY, MM, DD and HH are date and time parts of current (as of logging
729 | moment) time.
730 | 
731 | HH is 0 padded 24-hour style hour.
732 | 
733 | Example configuration:
734 | 
735 |     # Global configuration, log directory
736 |     GLOBAL.logdir=/var/tmp/monitoring
737 | 
738 |     # Logging iostat output in 10 second intervals
739 |     check.iostat.type=persistent
740 |     check.iostat.exec=iostat -kx 10
741 | 
742 |     # Logging "ps auxwwn" every 30 seconds.
743 |     check.ps.type=periodic
744 |     check.ps.exec=ps auxwwn
745 |     check.ps.interval=30
746 | 
747 | =head2 INTERNALS
748 | 
749 | Program itself is very short:
750 | 
751 |     my $program = Monitoring->new();
752 |     $program->run();
753 | 
754 | This creates $program as object of Monitoring class (defined in the same
755 | file), and calls method run() on it.
756 | 
757 | =head3 METHODS
758 | 
759 | =head4 new
760 | 
761 | Just object constructor. Nothing to see there.
762 | 
763 | =head4 run
764 | 
765 | Initialization of stuff, and call to main_loop. Reads and validates config
766 | (by calls to appropriate methods), initializes IO::Select object for
767 | asynchronous I/O, starts persistent checks (again, using special metod), and
768 | enters main_loop();
769 | 
770 | =head4 main_loop
771 | 
772 | The core of the program. Infinite loop, which - upon every iteration:
773 | 
774 | =over
775 | 
776 | =item * updates logging filehandles
777 | 
778 | =item * checks if there is anything to read in input filehandles (from
779 | checks)
780 | 
781 | =item * reads whatever is to be read from checks
782 | 
783 | =item * runs new periodic checks if the time has come to do it
784 | 
785 | =back
786 | 
787 | Checking for data in input filehandles is done with timeout, which is
788 | calculated to finish when next check will have to be run, so the program
789 | doesn't use virtually no CPU unless there are some data to be worked on.
790 | 
791 | =head4 handle_read
792 | 
793 | Since all we get from IO::Select is filehandle to read from, this method has
794 | first to find which check given filehandle belongs to.
795 | 
796 | Afterwards, it reads whatever is available in the filehandle. In case there
797 | is error on the filehandle - it closes the filehandle - as it means that
798 | output for given check ended.
799 | 
800 | Every line from check is prefixed with timestamp and logged to appropriate
801 | logfile.
802 | 
803 | Additionally, when closing the filehandle (on error), it sets when given
804 | check should be run next time.
805 | 
806 | =head4 run_check
807 | 
808 | Simple helper function which runs external program (or opens filehandle for
809 | reading from file), and puts it into check data.
810 | 
811 | =head4 start_periodic_processes
812 | 
813 | Iterates over all periodic processes, checks which should be already run,
814 | and runs them.
815 | 
816 | =head4 start_persistent_processes
817 | 
818 | Iterates over all persistent processes and runs them. This is done only
819 | once, from run() method.
820 | 
821 | =head4 calculate_timeout
822 | 
823 | Helper function which calculates how long should main_loop() wait for data
824 | from IO::Select before it has to run another round of
825 | start_periodic_processes().
826 | 
827 | =head4 update_logger_filehandles
828 | 
829 | Checks if current timestamp has changed enough to require swapping files,
830 | and if yes - closes old ones and opens new ones - making all necessary
831 | directories to make it happen.
832 | 
833 | =head4 checks
834 | 
835 | Wrapper to be able to write:
836 | 
837 |     for my $C ( $self->checks ) {
838 | 
839 | instead of:
840 | 
841 |     for my $C ( @{ $self->{ 'checks'} } ) {
842 | 
843 | =head4 validate_config
844 | 
845 | Verifies that config values make sense, and reorganizes them into final data
846 | structure (checks hashes in $self->{'checks'} arrayref).
847 | 
848 | =head4 read_config
849 | 
850 | Just like name suggests - reads given config to memory. Very simple parser
851 | based on regular expressions.
852 | 
853 | =head2 LICENSE
854 | 
855 | Copyright (c) 2010,2011, OmniTI, Inc.
856 | 
857 | Permission to use, copy, modify, and distribute this software and its
858 | documentation for any purpose, without fee, and without a written agreement
859 | is hereby granted, provided that the above copyright notice and this
860 | paragraph and the following two paragraphs appear in all copies.
861 | 
862 | IN NO EVENT SHALL OmniTI, Inc. BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT,
863 | SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS,
864 | ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF
865 | OmniTI, Inc. HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
866 | 
867 | OmniTI, Inc. SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT
868 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
869 | PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS,
870 | AND OmniTI, Inc. HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT,
871 | UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
872 | 
873 | =head2 COPYRIGHT
874 | 
875 | The system_monitoring project is Copyright (c) 2010,2011 OmniTI. All rights reserved.
876 | 
877 | 


--------------------------------------------------------------------------------