├── .gitignore ├── .shipit ├── Changes ├── MANIFEST ├── Makefile.PL ├── Makefile.package ├── README.md ├── SPECS └── fluent-agent-lite.spec ├── bin ├── fluent-agent-lite └── install.sh ├── lib └── Fluent │ └── AgentLite.pm ├── misc └── start-fluent-agent-lite └── package ├── fluent-agent-lite.conf └── fluent-agent-lite.init /.gitignore: -------------------------------------------------------------------------------- 1 | META.yml 2 | MYMETA* 3 | Makefile 4 | blib/ 5 | inc/ 6 | extlib/ 7 | pm_to_blib 8 | tmp/archive 9 | *~ 10 | -------------------------------------------------------------------------------- /.shipit: -------------------------------------------------------------------------------- 1 | # auto-generated shipit config file. 2 | steps = FindVersion, ChangeVersion, CheckChangeLog, Commit, Tag, MakeDist, UploadCPAN 3 | 4 | # svn.tagpattern = MyProj-%v 5 | # svn.tagpattern = http://code.example.com/svn/tags/MyProj-%v 6 | 7 | # CheckChangeLog.files = ChangeLog, MyProj.CHANGES 8 | git.tagpattern = %v 9 | git.push_to = origin 10 | 11 | -------------------------------------------------------------------------------- /Changes: -------------------------------------------------------------------------------- 1 | revision history for Perl module Fluent::AgentLite 2 | 3 | {{$NEXT}} 4 | - add force start option (w/o reading log file) 5 | - archive file maintenance 6 | 7 | 0.9 Thu Jun 27 15:43:37 JST 2013 8 | - maintenance update (update of package scripts) 9 | 10 | 0.8 Sat Apr 13 15:24:26 JST 2013 11 | - first version for CPAN dist 12 | 13 | -------------------------------------------------------------------------------- /MANIFEST: -------------------------------------------------------------------------------- 1 | bin/fluent-agent-lite 2 | Changes 3 | inc/Module/Install.pm 4 | inc/Module/Install/Base.pm 5 | inc/Module/Install/Can.pm 6 | inc/Module/Install/Fetch.pm 7 | inc/Module/Install/Makefile.pm 8 | inc/Module/Install/Metadata.pm 9 | inc/Module/Install/Scripts.pm 10 | inc/Module/Install/Win32.pm 11 | inc/Module/Install/WriteAll.pm 12 | lib/Fluent/AgentLite.pm 13 | Makefile.PL 14 | MANIFEST This list of files 15 | META.yml 16 | README.md 17 | -------------------------------------------------------------------------------- /Makefile.PL: -------------------------------------------------------------------------------- 1 | use inc::Module::Install; 2 | 3 | name 'Fluent-AgentLite'; 4 | all_from 'lib/Fluent/AgentLite.pm'; 5 | license 'perl'; 6 | 7 | requires 'Data::MessagePack' => '0.35_01'; 8 | requires 'JSON::XS'; 9 | 10 | requires 'Time::Piece'; 11 | requires 'Time::HiRes'; 12 | requires 'Time::Piece'; 13 | requires 'Log::Minimal'; 14 | 15 | install_script 'bin/fluent-agent-lite'; 16 | 17 | WriteAll; 18 | -------------------------------------------------------------------------------- /Makefile.package: -------------------------------------------------------------------------------- 1 | version = $(shell git tag | tail -1 | sed -e 's/^v//') 2 | 3 | archive: 4 | rm -rf tmp/archive/fluent-agent-lite-$(version) 5 | mkdir -p tmp/archive 6 | (cd tmp/archive ; git clone ../../.git fluent-agent-lite-$(version) ; cd fluent-agent-lite-$(version) ; rm -rf .git .shipit) 7 | (cd tmp/archive ; tar czf fluent-agent-lite-$(version).tar.gz fluent-agent-lite-$(version)) 8 | 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fluent-agent-lite 2 | 3 | * http://github.com/tagomoris/fluent-agent-lite 4 | 5 | ## DESCRIPTION 6 | 7 | 'fluent-agent-lite' is a log transfer agent, for Fluentd's 'forward' input. 8 | 9 | This agent reads specified files, and sends each lines to fluentd servers. One log line will be packed one fluentd message, that has one attribute ('message' or specified in configuration) with entire line (not terminated by newline). 10 | 11 | ### VERSION 12 | 13 | 1.0 https://github.com/tagomoris/fluent-agent-lite/releases/tag/v1.0 14 | 15 | ## PACKAGING TARBALL 16 | 17 | To get tarball of latest tagged version: 18 | 19 | make -f Makefile.package 20 | 21 | Got tarball on `tmp/archive/fluent-agent-lite-vX.Y.tar.gz`. 22 | 23 | To do this, versions (tags) MUST be v1.0 not but v0.10 for sorting order... 24 | 25 | ## INSTALL 26 | 27 | On RHEL/CentOS, you can use .spec file to build rpm with your customized default config file. 28 | 29 | ### RHEL/CentOS 30 | 31 | To build your rpm package, do 5 steps below. 32 | 33 | 1. Download (newest version) tarball, and place it on SOURCES/ . 34 | 2. Download package/fluent-agent-lite.conf, and place it on SOURCES/ . 35 | 3. Fix SOURCES/package/fluent-agent-lite.conf as you want (ex: server name or servers list), and add servers list for your own. 36 | 4. Download SPECS/fluent-agent-lite.spec, and place it on SPECS/ . 37 | 5. run 'rpmbuild -ba SPECS/fluent-agent-lite.spec' 38 | 39 | To install each RHEL/CentOS host, use yum server, or copy and rpm -i on each host. 40 | 41 | NOTE: `yum install perl-devel` and `QA_RPATHS=$[0x001] rpmbuild -ba` may help you if `rpmbuild` fails on your build environment. 42 | 43 | ### Other Linux or Unix-like OS 44 | 45 | On each host, do steps below. 46 | 47 | 1. Download and extract tarball, or clone repository, and move into extracted directory. 48 | 2. Do 'bin/install.sh'. 49 | 50 | ## Configuration 51 | 52 | All of configurations are written in configuration shell-script file (/etc/fluent-agent-lite.conf). Configurable values are below: 53 | 54 | ### LOGS 55 | 56 | Pairs of tag and file, such as: 57 | 58 | LOGS=$(cat <<"EOF" 59 | www /var/log/nginx/www_access.log 60 | app /var/log/apache2/app_access.log 61 | EOF 62 | 63 | Or, you can use this syntax: 64 | 65 | LOGS=$(cat /etc/fluent-agent.logs) 66 | 67 | ## in fluent-agent.logs 68 | www /var/log/nginx/www_access.log 69 | app /var/log/apache2/app_access.log 70 | 71 | ### TAG_PREFIX 72 | 73 | Prefix of each tags, specified in 'LOGS'. 'TAG_PREFIX="service"' with 'LOGS' above, you will get fluentd messages with tags 'service.www' and 'service.app'. 74 | 75 | ### FIELD_NAME 76 | 77 | Log line attribute name in fluentd message (default: 'message'). 78 | 79 | ### PRIMARY_SERVER, SECONDARY_SERVER 80 | 81 | Fluentd server name and port (SERVERNAME:PORT), as primary server. 'fluent-agent-lite' try to connect to primary server at first, and if fails, then try to connect secondary server (if it specified). 82 | 83 | Default port is 24224 (if omitted). 84 | 85 | ### PRIMARY_SERVERS_LIST, SECONDARY_SERVERS_LIST 86 | 87 | File path to specify primary(secondary) servers' list. 'fluent-agent-lite' reads this file when executed, and choose one of servers randomly at each connection trial. 88 | 89 | You cannot specify both of 'PRIMARY\_SERVER' and 'PRIMARY\_SERVERS\_LIST', and both of 'SECONDARY\_SERVER' and 'SECONDARY\_SERVERS\_LIST' 90 | 91 | ### READ_BUFFER_SIZE 92 | 93 | Bytes size which 'fluent-agent-lite' try to read from 'tail' at once (Default: 1MB). 94 | 95 | ### PROCESS_NICE 96 | 97 | Nice value for 'fluent-agent-lite'. If you want to execute fluent-agent-lite in server with high loadavg, 'PROCESS_NICE="-1"' can help you. 98 | 99 | ### TAIL_PATH 100 | 101 | Path of tail command (Default: /usr/bin/tail). 102 | 103 | ### TAIL_INTERVAL 104 | 105 | 'sleep interval' of tail command in seconds (Default: 1.0). For high throughput log file, you can specify 'TAIL_INTERVAL="0.5"' or any other values (but over "0.1"). 106 | 107 | Caution: This cofiguration is for GNU tail only. 108 | 109 | ### PING_TAG, PING_DATA, PING_INTERVAL 110 | 111 | Ping message tag/data and emit interval specification. Without PING\_TAG, fluent-agent-lite doesn't emit ping\_messages. 112 | 113 | Actual 'data' field of ping message is 'PING_DATA PATH\_OF\_INPUT\_FILE'. 114 | 115 | ### DRAIN_LOG_TAG 116 | 117 | Tag name of drain\_log (messages count per drain/send to server), which is emitted to configured fluentd sever as fluentd message. Default is none (not to send drain\_log). 118 | 119 | ### KEEPALIVE_TIME 120 | 121 | Connection keepalive time in seconds. 0 means infinity (Default: 1800, minimum: 120) 122 | 123 | ### RECONNECT_WAIT_MAX 124 | 125 | The maximum wait time for TCP socket reconnection in seconds. (Default: 3600, minimum: 0.5) 126 | 127 | ### RECONNECT_WAIT_INCR_RATE 128 | 129 | The rate to increment the reconnect time. `next_wait_time = current_wait_time * rate` (Default: 1.5, minimum: 1.0) 130 | 131 | ### LOG_PATH 132 | 133 | Log file path for 'fluent-agent-lite' (Default: /tmp/fluent-agent.log). 134 | 135 | ### LOG_VERBOSE 136 | 137 | If you specify 'LOG_VERBOSE="yes"', 'fluent-agent-lite' writes logs with level info/debug (Default: warn/crit only). 138 | 139 | ## Run 140 | 141 | With properly configured '/etc/fluent-agent-lite.conf', you can run and stop all transfer agent as below. 142 | 143 | # /etc/init.d/fluent-agent-lite start 144 | # /etc/init.d/fluent-agent-lite restart 145 | # /etc/init.d/fluent-agent-lite stop 146 | 147 | Check running processes: 148 | 149 | # /etc/init.d/fluent-agent-lite status 150 | 151 | And reset fluentd connections of each processes. (To re-connect primary server instead of secondary server immediately.) 152 | 153 | # /etc/init.d/fluent-agent-lite reload 154 | 155 | To reflect change of config file, you must do 'restart'. 156 | 157 | * * * * * 158 | 159 | ## License 160 | 161 | Copyright 2012- TAGOMORI Satoshi (tagomoris) 162 | 163 | Licensed under the Apache License, Version 2.0 (the "License"); 164 | you may not use this file except in compliance with the License. 165 | You may obtain a copy of the License at 166 | 167 | http://www.apache.org/licenses/LICENSE-2.0 168 | 169 | Unless required by applicable law or agreed to in writing, software 170 | distributed under the License is distributed on an "AS IS" BASIS, 171 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 172 | See the License for the specific language governing permissions and 173 | limitations under the License. 174 | -------------------------------------------------------------------------------- /SPECS/fluent-agent-lite.spec: -------------------------------------------------------------------------------- 1 | %define name fluent-agent-lite 2 | %define version 1.0 3 | %define prefix /usr/local 4 | %define build_perl_path /usr/bin/perl 5 | 6 | %define _use_internal_dependency_generator 0 7 | 8 | %global debug_package %{nil} 9 | 10 | Name: fluent-agent-lite 11 | Version: %{version} 12 | Release: original 13 | Summary: Log transfer agent service over fluentd protocol 14 | 15 | Group: Applications/System 16 | License: Apache Software License v2 17 | URL: https://github.com/tagomoris/fluent-agent-lite 18 | # Source0: (tarball from) https://github.com/tagomoris/fluent-agent-lite/releases/tag/v%{version} 19 | Source0: fluent-agent-lite-%{version}.tar.gz 20 | # Source1: fluent-agent-lite.conf 21 | # Source2: fluent-agent.servers.primary 22 | # Source3: fluent-agent.servers.secondary 23 | BuildRoot: %{_tmppath}/%{name}-root 24 | 25 | ExclusiveArch: x86_64 i386 26 | AutoReq: no 27 | 28 | %description 29 | Log transfer agent service over fluentd protocol. 30 | 31 | %prep 32 | %setup -q -n fluent-agent-lite-%{version} 33 | 34 | %build 35 | 36 | %install 37 | rm -rf $RPM_BUILD_ROOT 38 | env PREFIX=$RPM_BUILD_ROOT PERL_PATH=%{build_perl_path} bin/install.sh 39 | # install -m 644 $RPM_SOURCE_DIR/fluent-agent-lite.conf $RPM_BUILD_ROOT/etc/fluent-agent-lite.conf 40 | # install -m 644 $RPM_SOURCE_DIR/fluent-agent.servers.primary $RPM_BUILD_ROOT/etc/fluent-agent.servers.primary 41 | # install -m 644 $RPM_SOURCE_DIR/fluent-agent.servers.secondary $RPM_BUILD_ROOT/etc/fluent-agent.servers.secondary 42 | 43 | %clean 44 | rm -rf $RPM_BUILD_ROOT 45 | 46 | %files 47 | %defattr(-,root,root) 48 | %config(noreplace) %{_sysconfdir}/fluent-agent-lite.conf 49 | # %config %{_sysconfdir}/fluent-agent.servers.primary 50 | # %config %{_sysconfdir}/fluent-agent.servers.secondary 51 | %{_sysconfdir}/init.d/fluent-agent-lite 52 | %{prefix}/* 53 | # %doc README 54 | 55 | %changelog 56 | * Wed Dec 11 2013 TAGOMORI Satoshi 57 | - add force start option 58 | - change archiving 59 | * Thu Jun 27 2013 TAGOMORI Satoshi 60 | - add configuration pattern for log files 61 | - fix to check input log files 62 | * Sat Apr 13 2013 TAGOMORI Satoshi 63 | - fix not to read lines already exists just after start-up 64 | - CPANize 65 | * Tue Mar 19 2013 TAGOMORI Satoshi 66 | - add i386 support for specs 67 | - add keepalive time option 68 | - fix to remove .pid file when stop 69 | * Wed Aug 29 2012 TAGOMORI Satoshi 70 | - add ping_message options 71 | * Wed Mar 21 2012 TAGOMORI Satoshi 72 | - fix to send PackedForward object 73 | - bugfix about installer / init script 74 | - add feature about drain log 75 | * Thu Mar 15 2012 TAGOMORI Satoshi 76 | - bugfix about path of perl 77 | * Wed Mar 14 2012 TAGOMORI Satoshi 78 | - initial packaging attempt 79 | -------------------------------------------------------------------------------- /bin/fluent-agent-lite: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | 3 | use strict; 4 | use warnings; 5 | use English; 6 | 7 | use Fcntl; 8 | 9 | use FindBin; 10 | use lib "$FindBin::Bin/../lib"; 11 | use lib "$FindBin::Bin/../extlib/lib/perl5"; 12 | 13 | use Log::Minimal; 14 | local $Log::Minimal::PRINT = sub { 15 | my ( $time, $type, $message, $trace,$raw_message) = @_; 16 | warn "$time [$type] ($PID) $message at $trace\n"; 17 | }; 18 | 19 | use Fluent::AgentLite; 20 | 21 | my $HUPPED = undef; 22 | $SIG{ HUP } = sub { $HUPPED = 1; }; # to reconnect 23 | my $TERMINATED = undef; 24 | $SIG{ INT } = $SIG{ TERM } = sub { $TERMINATED = 1; }; # to terminate 25 | 26 | my $checker_terminated = sub { $TERMINATED; }; 27 | my $checker_reconnect = sub { 28 | if (shift) { 29 | $HUPPED = undef; 30 | } else { 31 | $HUPPED or $TERMINATED; 32 | } 33 | }; 34 | 35 | use Getopt::Std qw//; 36 | my %commandline_options; 37 | Getopt::Std::getopts('f:p:s:b:n:t:i:l:P:S:d:k:w:r:jvFh', \%commandline_options); 38 | 39 | sub HELP_MESSAGE { 40 | print <) { 89 | chomp $line; 90 | push @list, server_entry($line) if length($line) > 0; 91 | } 92 | close($fd); 93 | return @list; 94 | } 95 | 96 | my $output_tag = shift @ARGV; 97 | my $input_file = shift @ARGV; 98 | 99 | my $fieldname = 'message'; 100 | if ($commandline_options{f}) { 101 | $fieldname = $commandline_options{f}; 102 | } 103 | 104 | my @primary_servers; 105 | my @secondary_servers; 106 | if (scalar(@ARGV) > 0) { 107 | push @primary_servers, server_entry(shift @ARGV); 108 | push @secondary_servers, server_entry(shift @ARGV) if scalar(@ARGV) > 0; 109 | } 110 | if ($commandline_options{p}) { 111 | die "primary server list file not exists" unless -f $commandline_options{p}; 112 | push @primary_servers, load_server_list($commandline_options{p}); 113 | } 114 | if ($commandline_options{s}) { 115 | die "secondary server list file not exists" unless -f $commandline_options{s}; 116 | push @secondary_servers, load_server_list($commandline_options{s}); 117 | } 118 | 119 | my $buffer_size = 1024 * 1024; 120 | if ($commandline_options{b} and $commandline_options{b} > 0) { 121 | $buffer_size = $commandline_options{b}; 122 | } 123 | my $nice = undef; 124 | if (defined $commandline_options{n}) { 125 | $nice = $commandline_options{n}; 126 | } 127 | my $tail_path = '/usr/bin/tail'; 128 | if ($commandline_options{t}) { 129 | die "not executable file specified as tail" unless -x $commandline_options{t}; 130 | $tail_path = $commandline_options{t}; 131 | } 132 | my $sleep_interval = undef; 133 | if ($commandline_options{i}) { 134 | my $tail_interval_valid = (system('tail -s 1 /dev/null > /dev/null 2>&1') == 0); 135 | die "tail interval specified, but your tail doesn't have -s option" unless $tail_interval_valid; 136 | $sleep_interval = $commandline_options{i}; 137 | } 138 | my $logpath = '/tmp/fluent-agent.log'; 139 | if ($commandline_options{l}) { 140 | $logpath = $commandline_options{l}; 141 | } 142 | 143 | my $ping_message = undef; 144 | if ($commandline_options{P}) { 145 | $commandline_options{P} =~ /^([^:]+):(.+)$/; 146 | $ping_message = +{ 147 | tag => $1, 148 | data => $2 . ' ' . $input_file , 149 | interval => ($commandline_options{S} or 60), 150 | }; 151 | } 152 | 153 | my $drain_log_tag = undef; 154 | if ($commandline_options{d}) { 155 | $drain_log_tag = $commandline_options{d}; 156 | } 157 | my $keepalive_time = undef; 158 | if (defined $commandline_options{k} and $commandline_options{k} >= 0) { 159 | $keepalive_time = $commandline_options{k}; 160 | } 161 | my $reconnect_wait_max = undef; 162 | if (defined $commandline_options{w} and $commandline_options{w} >= 0) { 163 | $reconnect_wait_max = $commandline_options{w}; 164 | } 165 | my $reconnect_wait_incr_rate = undef; 166 | if (defined $commandline_options{r} and $commandline_options{r} >= 0) { 167 | $reconnect_wait_incr_rate = $commandline_options{r}; 168 | } 169 | 170 | my $output_format = undef; # default: MessagePack 171 | if ($commandline_options{j}) { 172 | $output_format = 'json'; 173 | } 174 | 175 | my $force_start = undef; 176 | if ($commandline_options{F}) { 177 | $force_start = 1; 178 | } 179 | 180 | open(STDOUT, ">> $logpath") or die "failed to reopen STDOUT to $logpath"; 181 | open(STDERR, ">> $logpath") or die "failed to reopen STDERR to $logpath"; 182 | 183 | use IO::Handle; 184 | autoflush STDOUT 1; 185 | autoflush STDERR 1; 186 | 187 | sub build_tail_command { 188 | my @command; 189 | if ($nice) { 190 | push @command, 'nice', '-n', $nice; 191 | } 192 | push @command, $tail_path; 193 | if ($sleep_interval) { 194 | push @command, '-s', $sleep_interval; 195 | } 196 | push @command, '-n', '0', '-F', $input_file; 197 | infof 'tail command line: %s', \@command; 198 | return @command; 199 | } 200 | 201 | sub open_tail { 202 | my $pid = open(my $tailfd, '-|', build_tail_command()) 203 | or die "failed to exec tail"; 204 | my $io = IO::Handle->new(); 205 | $io->fdopen($tailfd, "r"); 206 | $io->blocking(0); # set nonblock 207 | return ($io, $pid); 208 | } 209 | 210 | sub open_stdin { 211 | my $io = IO::Handle->new(); 212 | $io->fdopen(fileno(STDIN), "r"); 213 | $io->blocking(0); 214 | return ($io, undef); 215 | } 216 | 217 | sub close_fd { 218 | my $fd = shift; 219 | eval { 220 | $fd->blocking(1); 221 | close($fd); 222 | }; 223 | # throw away errors 224 | 1; 225 | } 226 | 227 | sub main { 228 | my ($tailfd, $tailpid); 229 | if ( $input_file eq "-" ) { 230 | ($tailfd, $tailpid) = open_stdin(); 231 | } elsif ( -f $input_file || $force_start ) { 232 | close(STDIN) or die "failed to close STDIN"; 233 | ($tailfd, $tailpid) = open_tail(); 234 | } else { 235 | die 'cannot find input file ', $input_file; 236 | } 237 | 238 | my $agent = Fluent::AgentLite->new( 239 | $output_tag, 240 | \@primary_servers, 241 | \@secondary_servers, 242 | { 243 | buffer_size => $buffer_size, 244 | ping_message => $ping_message, 245 | drain_log_tag => $drain_log_tag, 246 | keepalive_time => $keepalive_time, 247 | reconnect_wait_max => $reconnect_wait_max, 248 | reconnect_wait_incr_rate => $reconnect_wait_incr_rate, 249 | output_format => $output_format, 250 | }, 251 | ); 252 | $agent->execute( { 253 | fieldname => $fieldname, 254 | tailfd => $tailfd, 255 | checker => { 256 | term => $checker_terminated, 257 | reconnect => $checker_reconnect, 258 | }, 259 | } ); 260 | 261 | if (defined $tailpid) { 262 | infof "Sending TERM to pid %s", $tailpid; 263 | kill(TERM => $tailpid); 264 | } 265 | # agent exits (killed, or on error), and close pipe 266 | close_fd($tailfd); 267 | } 268 | 269 | infof "starting to forward logs."; 270 | 271 | main(); 272 | 273 | infof "process exit."; 274 | 275 | exit 0; 276 | -------------------------------------------------------------------------------- /bin/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | ARG="$1" 6 | CLEAN= 7 | if [ "x"$ARG = "xclean" ]; then 8 | CLEAN="y" 9 | fi 10 | 11 | cd $(dirname $0)"/../" 12 | 13 | SOURCEDIR=$(pwd) 14 | 15 | if [ "x"$PREFIX = "x" ]; then 16 | PREFIX= 17 | fi 18 | INSTALLDIR=$PREFIX/usr/local/fluent-agent-lite 19 | 20 | if [ "x"$PERL_PATH = "x" ]; then 21 | PERL_PATH="perl" 22 | fi 23 | 24 | if [ -d $INSTALLDIR -a "x"$CLEAN = "xy" ]; then 25 | rm -rf $INSTALLDIR 26 | fi 27 | 28 | mkdir -p $INSTALLDIR 29 | 30 | cp -rp bin lib Makefile.PL $INSTALLDIR 31 | 32 | cd $INSTALLDIR 33 | 34 | curl -s -L http://cpanmin.us > $INSTALLDIR/bin/cpanm 35 | chmod +x $INSTALLDIR/bin/cpanm 36 | $PERL_PATH $INSTALLDIR/bin/cpanm -n -lextlib inc::Module::Install 37 | $PERL_PATH $INSTALLDIR/bin/cpanm -n -lextlib --reinstall --installdeps . 38 | 39 | cd $SOURCEDIR 40 | 41 | mkdir -p $PREFIX/etc/init.d 42 | cp package/fluent-agent-lite.init $PREFIX/etc/init.d/fluent-agent-lite 43 | chmod +x $PREFIX/etc/init.d/fluent-agent-lite 44 | 45 | if [ ! -f $PREFIX/etc/fluent-agent-lite.conf -o "x"$CLEAN = x"y" ]; then 46 | cp package/fluent-agent-lite.conf $PREFIX/etc/fluent-agent-lite.conf 47 | fi 48 | -------------------------------------------------------------------------------- /lib/Fluent/AgentLite.pm: -------------------------------------------------------------------------------- 1 | package Fluent::AgentLite; 2 | 3 | use strict; 4 | use warnings; 5 | use English; 6 | use Carp; 7 | 8 | use POSIX qw(:errno_h); 9 | 10 | use Time::HiRes; 11 | 12 | use Time::Piece; 13 | use Log::Minimal; 14 | 15 | use IO::Socket::INET; 16 | use Data::MessagePack; 17 | 18 | our $VERSION = '1.0'; 19 | 20 | use constant READ_WAIT => 0.1; # 0.1sec 21 | 22 | use constant SOCKET_TIMEOUT => 5; # 5sec 23 | 24 | use constant CONNECTION_KEEPALIVE_INFINITY => 0; 25 | use constant CONNECTION_KEEPALIVE_TIME => 1800; # 30min 26 | use constant CONNECTION_KEEPALIVE_MIN => 120; # min 2min 27 | use constant CONNECTION_KEEPALIVE_MARGIN_MAX => 30; # max 30sec 28 | 29 | use constant RECONNECT_WAIT_MIN => 0.5; # 0.5sec 30 | use constant RECONNECT_WAIT_MAX => 3600; # 60min 31 | use constant RECONNECT_WAIT_INCR_RATE_MIN => 1.0; 32 | use constant RECONNECT_WAIT_INCR_RATE => 1.5; 33 | 34 | use constant SEND_RETRY_MAX => 4; 35 | 36 | sub connection_keepalive_time { 37 | my ($keepalive_time) = @_; 38 | $keepalive_time + int(CONNECTION_KEEPALIVE_MARGIN_MAX * 2 * rand()) - CONNECTION_KEEPALIVE_MARGIN_MAX; 39 | } 40 | 41 | sub new { 42 | my $this = shift; 43 | my ($tag, $primary_servers, $secondary_servers, $configuration) = @_; 44 | my $self = { 45 | tag => $tag, 46 | servers => { 47 | primary => $primary_servers, 48 | secondary => $secondary_servers, 49 | }, 50 | buffer_size => $configuration->{buffer_size}, 51 | ping_message => $configuration->{ping_message}, 52 | drain_log_tag => $configuration->{drain_log_tag}, 53 | keepalive_time => $configuration->{keepalive_time}, 54 | reconnect_wait_incr_rate => $configuration->{reconnect_wait_incr_rate}, 55 | reconnect_wait_max => $configuration->{reconnect_wait_max}, 56 | output_format => $configuration->{output_format}, 57 | }; 58 | 59 | if (defined $self->{output_format} and $self->{output_format} eq 'json') { 60 | *pack = *pack_json; 61 | *pack_drainlog = *pack_drainlog_json; 62 | } 63 | 64 | srand (time ^ $PID ^ unpack("%L*", `ps axww | gzip`)); 65 | 66 | bless $self, $this; 67 | } 68 | 69 | sub execute { 70 | my $self = shift; 71 | my $args = shift; 72 | 73 | my $fieldname = $args->{fieldname}; 74 | my $tailfd = $args->{tailfd}; 75 | 76 | my $check_terminated = ($args->{checker} || {})->{term} || sub { 0 }; 77 | my $check_reconnect = ($args->{checker} || {})->{reconnect} || sub { 0 }; 78 | 79 | my $packer = Data::MessagePack->new(); 80 | 81 | my $reconnect_wait = RECONNECT_WAIT_MIN; 82 | my $reconnect_wait_max = RECONNECT_WAIT_MAX; 83 | if (defined $self->{reconnect_wait_max}) { 84 | $reconnect_wait_max = $self->{reconnect_wait_max}; 85 | if ($reconnect_wait_max < RECONNECT_WAIT_MIN) { 86 | warnf 'Reconnect wait max is too short. Set minimum value %s', RECONNECT_WAIT_MIN; 87 | $reconnect_wait_max = RECONNECT_WAIT_MIN; 88 | } 89 | } 90 | my $reconnect_wait_incr_rate = RECONNECT_WAIT_INCR_RATE; 91 | if (defined $self->{reconnect_wait_incr_rate}) { 92 | $reconnect_wait_incr_rate = $self->{reconnect_wait_incr_rate}; 93 | if ($reconnect_wait_incr_rate < RECONNECT_WAIT_INCR_RATE_MIN) { 94 | warnf 'Reconnect wait incr rate is too small. Set minimum value %s', RECONNECT_WAIT_INCR_RATE_MIN; 95 | $reconnect_wait_incr_rate = RECONNECT_WAIT_INCR_RATE_MIN; 96 | } 97 | } 98 | 99 | my $last_ping_message = time; 100 | if ($self->{ping_message}) { 101 | $last_ping_message = time - $self->{ping_message}->{interval} * 2; 102 | } 103 | my $keepalive_time = CONNECTION_KEEPALIVE_TIME; 104 | if (defined $self->{keepalive_time}) { 105 | $keepalive_time = $self->{keepalive_time}; 106 | if ($keepalive_time < CONNECTION_KEEPALIVE_MIN and $keepalive_time != CONNECTION_KEEPALIVE_INFINITY) { 107 | warnf 'Keepalive time setting is too short. Set minimum value %s', CONNECTION_KEEPALIVE_MIN; 108 | $keepalive_time = CONNECTION_KEEPALIVE_MIN; 109 | } 110 | } 111 | 112 | my $pending_packed; 113 | my $continuous_line; 114 | my $disconnected_primary = 0; 115 | my $expiration_enable = $keepalive_time != CONNECTION_KEEPALIVE_INFINITY; 116 | 117 | while(not $check_terminated->()) { 118 | # at here, connection initialized (after retry wait if required) 119 | 120 | # connect to servers 121 | my $primary = $self->choose($self->{servers}->{primary}); 122 | my $secondary; 123 | 124 | my $sock = $self->connect($primary) unless $disconnected_primary; 125 | if (not $sock and $self->{servers}->{secondary}) { 126 | $secondary = $self->choose($self->{servers}->{secondary}); 127 | $sock = $self->connect($self->choose($self->{servers}->{secondary})); 128 | } 129 | $disconnected_primary = 0; 130 | unless ($sock) { 131 | # failed to connect both of primary / secondary 132 | warnf 'failed to connect servers, primary: %s, secondary: %s', $primary, ($secondary || 'none'); 133 | warnf 'waiting %s seconds to reconnect', $reconnect_wait; 134 | 135 | Time::HiRes::sleep($reconnect_wait); 136 | $reconnect_wait *= $reconnect_wait_incr_rate; 137 | $reconnect_wait = $reconnect_wait_max if $reconnect_wait > $reconnect_wait_max; 138 | next; 139 | } 140 | 141 | # succeed to connect. set keepalive disconnect time 142 | my $connecting = $secondary || $primary; 143 | 144 | my $expired = time + connection_keepalive_time($keepalive_time) if $expiration_enable; 145 | $reconnect_wait = RECONNECT_WAIT_MIN; 146 | 147 | while(not $check_reconnect->()) { 148 | # connection keepalive expired 149 | if ($expiration_enable and time > $expired) { 150 | infof "connection keepalive expired."; 151 | last; 152 | } 153 | 154 | # ping message (if enabled) 155 | my $ping_packed = undef; 156 | if ($self->{ping_message} and time >= $last_ping_message + $self->{ping_message}->{interval}) { 157 | $ping_packed = $self->pack_ping_message($packer, $self->{ping_message}->{tag}, $self->{ping_message}->{data}); 158 | $last_ping_message = time; 159 | } 160 | 161 | # drain (sysread) 162 | my $lines = 0; 163 | if (not $pending_packed) { 164 | my $buffered_lines; 165 | ($buffered_lines, $continuous_line, $lines) = $self->drain($tailfd, $continuous_line); 166 | 167 | if ($buffered_lines) { 168 | $pending_packed = $self->pack($packer, $fieldname, $buffered_lines); 169 | if ($self->{drain_log_tag}) { 170 | $pending_packed .= $self->pack_drainlog($packer, $self->{drain_log_tag}, $lines); 171 | } 172 | } 173 | if ($ping_packed) { 174 | $pending_packed ||= ''; 175 | $pending_packed .= $ping_packed; 176 | } 177 | unless ($pending_packed) { 178 | Time::HiRes::sleep READ_WAIT; 179 | next; 180 | } 181 | } 182 | # send 183 | my $written = $self->send($sock, $pending_packed); 184 | unless ($written) { # failed to write (socket error). 185 | $disconnected_primary = 1 unless $secondary; 186 | last; 187 | } 188 | 189 | $pending_packed = undef; 190 | } 191 | if ($check_reconnect->()) { 192 | infof "SIGHUP (or SIGTERM) received"; 193 | $disconnected_primary = 0; 194 | $check_reconnect->(1); # clear SIGHUP signal 195 | } 196 | infof "disconnecting to current server"; 197 | if ($sock) { 198 | $sock->close; 199 | $sock = undef; 200 | } 201 | infof "disconnected."; 202 | } 203 | if ($check_terminated->()) { 204 | warnf "SIGTERM received"; 205 | } 206 | infof "process exit"; 207 | } 208 | 209 | sub drain { 210 | # if broken child process (undefined return value of $fd->sysread()) 211 | # if content exists, return it. 212 | # else die 213 | my ($self,$fd, $continuous_line) = @_; 214 | my $readlimit = $self->{buffer_size}; 215 | my $readsize = 0; 216 | my $readlines = 0; 217 | my @buffered_lines; 218 | 219 | my $chunk; 220 | while ($readsize < $readlimit) { 221 | my $bytes = $fd->sysread($chunk, $readlimit); 222 | if (defined $bytes and $bytes == 0) { # EOF (child process exit) 223 | last if $readsize > 0; 224 | warnf "failed to read from child process, maybe killed."; 225 | confess "give up to read tailing fd, see logs"; 226 | } 227 | if (not defined $bytes and $! == EAGAIN) { # I/O Error (no data in fd): "Resource temporarily unavailable" 228 | last; 229 | } 230 | if (not defined $bytes) { # Other I/O error... what? 231 | warnf "I/O error with tail fd: $!"; 232 | last; 233 | } 234 | 235 | $readsize += $bytes; 236 | my $terminated_line = chomp $chunk; 237 | my @lines = split(m!\n!, $chunk); 238 | if ($continuous_line) { 239 | $lines[0] = $continuous_line . $lines[0]; 240 | $continuous_line = undef; 241 | } 242 | if (not $terminated_line) { 243 | $continuous_line = pop @lines; 244 | } 245 | if (scalar(@lines) > 0) { 246 | push @buffered_lines, @lines; 247 | $readlines += scalar(@lines); 248 | } 249 | } 250 | if ($readlines < 1) { 251 | return undef, $continuous_line, 0; 252 | } 253 | 254 | return (\@buffered_lines, $continuous_line, $readlines); 255 | } 256 | 257 | # MessagePack 'PackedForward' object 258 | # see lib/fluent/plugin/in_forward.rb in fluentd 259 | sub pack { 260 | my ($self,$packer,$fieldname,$lines) = @_; 261 | my $t = time; 262 | my $event_stream = join('', map { $packer->pack([$t, {$fieldname => $_}]) } @$lines); 263 | return $packer->pack([$self->{tag}, $event_stream]); 264 | } 265 | 266 | sub pack_json { 267 | use JSON::XS qw/encode_json/; 268 | my ($self,$packer,$fieldname,$lines) = @_; 269 | my $t = time; 270 | return encode_json([$self->{tag}, [map { [$t, {$fieldname => $_}] } @$lines ]]); 271 | } 272 | 273 | # MessagePack 'Message' object 274 | sub pack_ping_message { 275 | my ($self,$packer,$ping_tag,$ping_data) = @_; 276 | my $t = time; 277 | return $packer->pack([$ping_tag, $t, {'data' => $ping_data}]); 278 | } 279 | 280 | # MessagePack 'Message' object 281 | sub pack_drainlog { 282 | my ($self,$packer,$drain_log_tag,$drain_lines) = @_; 283 | my $t = time; 284 | return $packer->pack([$drain_log_tag, $t, {'drain' => $drain_lines}]); 285 | } 286 | 287 | sub pack_drainlog_json { 288 | use JSON::XS qw/encode_json/; 289 | my ($self,$packer,$drain_log_tag,$drain_lines) = @_; 290 | my $t = time; 291 | return encode_json([$drain_log_tag, $t, {'drain' => $drain_lines}]); 292 | } 293 | 294 | # choose a server [host, port] randomly from arg arrayref 295 | sub choose { 296 | my ($self,$servers) = @_; 297 | my $num = scalar(@$servers); 298 | return $servers->[int(rand() * $num)]; 299 | } 300 | 301 | sub connect { 302 | my ($self,$server) = @_; 303 | my $sock = IO::Socket::INET->new( 304 | PeerAddr => $server->[0], 305 | PeerPort => $server->[1], 306 | Proto => 'tcp', 307 | Timeout => SOCKET_TIMEOUT, 308 | ReuseAddr => 1, 309 | ); 310 | if ($sock) { 311 | infof 'connected to server: %s', $server; 312 | } else { 313 | warnf 'failed to connect to server %s : %s', $server, $!; 314 | } 315 | $sock; 316 | } 317 | 318 | sub send { 319 | my ($self,$sock,$data) = @_; 320 | my $length = length($data); 321 | my $written = 0; 322 | my $retry = 0; 323 | 324 | local $SIG{"PIPE"} = sub { die $! }; 325 | 326 | eval { 327 | while ($written < $length) { 328 | my $wbytes = $sock->syswrite($data, $length, $written); 329 | if ($wbytes) { 330 | $written += $wbytes; 331 | } 332 | else { 333 | die "failed $retry times to send data: $!" if $retry > SEND_RETRY_MAX; 334 | $retry += 1; 335 | } 336 | } 337 | }; 338 | if ($@) { 339 | my $error = $@; 340 | warnf "Cannot send data: $error"; 341 | return undef; 342 | } 343 | $written; 344 | } 345 | 346 | sub close { 347 | my ($self,$sock) = @_; 348 | $sock->close if $sock; 349 | } 350 | 351 | 1; 352 | -------------------------------------------------------------------------------- /misc/start-fluent-agent-lite: -------------------------------------------------------------------------------- 1 | #!perl 2 | 3 | use strict; 4 | use warnings; 5 | 6 | use Getopt::Long qw(:config posix_default no_ignore_case no_ignore_case_always); 7 | use Pod::Usage; 8 | 9 | use Data::Validator; 10 | use Log::Minimal env_debug => 'SFAL_DEBUG'; 11 | use Proclet; 12 | 13 | my $PROG = substr($0, rindex($0, '/') + 1); 14 | 15 | my $Debug = 0; 16 | 17 | MAIN: { 18 | my %arg; 19 | GetOptions( 20 | \%arg, 21 | 'agent-path|a=s', 22 | 'perl-path=s', 23 | 'tag-prefix=s', 24 | 'f|field-name=s', 25 | 'primary-server=s', 26 | 'secondary-server=s', 27 | 'p|primary-server-list=s', 28 | 's|secondary-server-list=s', 29 | 'b|raed-buffer-size=i', 30 | 'n|process-nice=i', 31 | 't|tail-path=s', 32 | 'i|tail-interval=f', 33 | 'tag-file|T=s@', 34 | 'l|log-path=s', 35 | 'P|ping=s', 36 | 'S|ping-interval=i', 37 | 'd|drain-log-tag=s', 38 | 'k|keepalive-time=i', 39 | 'w|reconnect-wait-max=f', 40 | 'r|reconnect-wait-incr-rate=f', 41 | 'j|json', 42 | 'v|log-verbose', 43 | 'F|force', 44 | 'debug' => \$Debug, 45 | 'help|h|?' => sub { pod2usage(-verbose=>1) }, 46 | ) or pod2usage(); 47 | 48 | $ENV{SFAL_DEBUG} = 1 if $Debug; 49 | debugf("arg: %s", ddf(\%arg)); 50 | 51 | 52 | my @opts_common; 53 | for my $k (qw(f b n t i l P S d k w r)) { 54 | push @opts_common, "-$k", $arg{$k} if exists $arg{$k}; 55 | } 56 | for my $k (qw(f v F)) { 57 | push @opts_common, "-$k" if exists $arg{$k}; 58 | } 59 | 60 | 61 | my @opts_server = (); 62 | if ($arg{'primary-server'}) { 63 | push @opts_server, $arg{'primary-server'}; 64 | push @opts_server, $arg{'secondary-server'} if exists $arg{'secondary-server'}; 65 | } elsif ($arg{p}) { 66 | for my $k (qw(p s)) { 67 | push @opts_common, "-$k", $arg{$k} if exists $arg{$k}; 68 | } 69 | } else { 70 | pod2usage("Invalid option: requires -p or --primary-server"); 71 | } 72 | 73 | 74 | my @opts_target_file = (); 75 | my $tag_prefix = exists $arg{'tag-prefix'} ? $arg{'tag-prefix'}."." : ""; 76 | for my $tagfile (@{ $arg{'tag-file'} }) { 77 | my($tag, $file) = split /=/, $tagfile, 2; 78 | push @opts_target_file, ["${tag_prefix}${tag}", $file]; 79 | } 80 | unless (@opts_target_file) { 81 | pod2usage("Invalid option: requires --tag-file TAG=TARGET_FILE"); 82 | } 83 | 84 | my $agent_path = exists $arg{'agent-path'} ? $arg{'agent-path'} : '/usr/local/fluent-agent-lite/bin/fluent-agent-lite'; 85 | if (! -f $agent_path) { 86 | pod2usage("No such file: $agent_path"); 87 | } 88 | 89 | my $perl_path = exists $arg{'perl-path'} ? $arg{'perl-path'} : 'perl'; 90 | 91 | debugf("agent path: %s", $agent_path); 92 | debugf("perl path: %s", $perl_path); 93 | debugf("common: %s", join(' ', @opts_common)); 94 | debugf("target_file: %s", ddf(\@opts_target_file)); 95 | debugf("server: %s", join(' ', @opts_server)); 96 | 97 | my $proclet = Proclet->new( 98 | color => 1, 99 | ); 100 | 101 | for my $target_file (@opts_target_file) { 102 | my @args = ($perl_path, $agent_path, @opts_common, @$target_file, @opts_server); 103 | $proclet->service( 104 | code => sub { 105 | exec { $args[0] } @args; 106 | }, 107 | ); 108 | } 109 | 110 | $proclet->run; 111 | exit 0; 112 | } 113 | 114 | __END__ 115 | 116 | =head1 NAME 117 | 118 | B - fluent-agent-lite launcher script 119 | 120 | =head1 SYNOPSIS 121 | 122 | B --primary-server HOST:PORT [--secondary-server HOST:PORT] -T TAG=FILE [-T TAG=FILE ...] [OTHER OPTIONS] 123 | 124 | B -p SERVER_LIST_FILE [-s SERVER_LIST_FILE] -T TAG=FILE [-T TAG=FILE ...] [OTHER OPTIONS] 125 | 126 | B B<-h> | B<--help> | B<-?> 127 | 128 | $ start-fluent-agent-lite --primary-server 127.0.0.1 --tag-prefix service \ 129 | -T www=/var/log/nginx/www_access.log \ 130 | -T app=/var/log/apache2/app_access.log 131 | 132 | =head1 DESCRIPTION 133 | 134 | This script is launcher script for fluent-agent-lite. 135 | 136 | fluent-agent-lite can read from only one file, so we need to run several fluent-agent-lite(s) if read and transfer multiple files. 137 | 138 | start-fluent-agent-lite starts fluent-agent-lite(s) using L as many as target files (specified -T or --tag-file options) 139 | 140 | =head1 OPTIONS 141 | 142 | =over 4 143 | 144 | =item B<--agent-path> Str, B<-a> Str 145 | 146 | path of fluent-agent-lite (DEFAULT: fluent-agent-lite (search in $PATH)) 147 | 148 | =item B<--tag-prefix> Str 149 | 150 | Prefix of each tags, specified in '--tag-file' option 151 | 152 | =item B<-f> Str, B<--field-name> Str 153 | 154 | fieldname of fluentd log message attribute (DEFAULT: message) 155 | 156 | =item B<--primary-server> SERVERNAME[:PORT] (Str) 157 | 158 | Fluentd server name and port (SERVERNAME:PORT), as primary server. 'fluent-agent-lite' try to connect to primary server at first, and if fails, then try to connect secondary server (if it specified). 159 | 160 | Default port is 24224 (if omitted). 161 | 162 | =item B<--secondary-server> SERVERNAME[:PORT] (Str) 163 | 164 | Secondary fluentd server name and port. 165 | 166 | =item B<-p> Str, B<--primary-server-list> Str 167 | 168 | primary servers list file (server[:port] per line, random selected one server) 169 | 170 | =item B<-s> Str, B<--secondary-server-list> Str 171 | 172 | secondary servers list file (server[:port] per line, random selected one server) 173 | 174 | =item B<-b> Int, B<--raed-buffer-size> Int 175 | 176 | log tailing buffer size (DEFAULT: 1MB) 177 | 178 | =item B<-n> Int, B<--process-nice> Int 179 | 180 | tail process nice (DEFAULT: 0) 181 | 182 | =item B<-t> Str, B<--tail-path> Str 183 | 184 | tail path (DEFAULT: /usr/bin/tail) 185 | 186 | =item B<-i> Num, B<--tail-interval> Num 187 | 188 | tail -F sleep interval (GNU tail ONLY, DEFAULT: tail default) 189 | 190 | =item B<--tag-file> TAG=TARGET_FILE, B<-T> TAG=TARGET_FILE 191 | 192 | Pairs of tag and target file. 193 | 194 | You can specify more than one this option. 195 | 196 | =item B<-l> Str, B<--log-path> Str 197 | 198 | log file path (DEFAULT: /tmp/fluent-agent.log) 199 | 200 | =item B<-P> TAG:DATA, B<--ping> TAG:DATA 201 | 202 | send a ping message per minute with specified TAG and DATA (DEFAULT: not to send) 203 | 204 | =item B<-S> Int, B<--ping-interval> Int 205 | 206 | ping message interval seconds (DEFAULT: 60) 207 | 208 | =item B<-d> Str, B<--drain-log-tag> Str 209 | 210 | emits drain log to fluentd: messages per drain/send (DEFAULT: not to emits) 211 | 212 | =item B<-k> Int, B<--keepalive-time> Int 213 | 214 | connection keepalive time in seconds. 0 means infinity (DEFAULT: 1800, minimum: 120) 215 | 216 | =item B<-w> Num, B<--reconnect-wait-max> Num 217 | 218 | the maximum wait time for TCP socket reconnection in seconds (DEFAULT: 3600, minimum: 0.5) 219 | 220 | =item B<-r> Num, B<--reconnect-wait-incr-rate> Num 221 | 222 | the rate to increment the reconnect time (DEFAULT: 1.5, minimum: 1.0) 223 | 224 | =item B<-j>, B<--json> 225 | 226 | use JSON for message structure in transfering (highly experimental) 227 | 228 | =item B<-v>, B<--log-verbose> 229 | 230 | output logs of level debug and info (DEFAULT: warn/crit only) 231 | 232 | =item B<-F>, B<--force> 233 | 234 | force start even if input file is not found 235 | 236 | =item B<--debug> 237 | 238 | increase debug level of start-fluent-agent-lite 239 | -d -d more verbosely. 240 | 241 | =back 242 | 243 | =head1 SEE ALSO 244 | 245 | L, L 246 | 247 | =head1 AUTHOR 248 | 249 | HIROSE Masaaki 250 | 251 | =cut 252 | 253 | # for Emacsen 254 | # Local Variables: 255 | # mode: cperl 256 | # cperl-indent-level: 4 257 | # cperl-close-paren-offset: -4 258 | # cperl-indent-parens-as-block: t 259 | # indent-tabs-mode: nil 260 | # coding: utf-8 261 | # End: 262 | 263 | # vi: set ts=4 sw=4 sts=0 et ft=perl fenc=utf-8 : 264 | -------------------------------------------------------------------------------- /package/fluent-agent-lite.conf: -------------------------------------------------------------------------------- 1 | # fluentd tag prefix of all LOGS 2 | TAG_PREFIX="" 3 | 4 | # fluentd message log attribute name (default: message) 5 | # FIELD_NAME="message" 6 | 7 | ### LOGS: tag /path/to/log/file 8 | # 9 | # LOGS=$(cat <<"EOF" 10 | # apache2 /var/log/apache2/access.log 11 | # # yourservice /var/log/yourservice/access.log 12 | # EOF 13 | # ) 14 | 15 | ### or, read from external file 16 | # LOGS=$(cat /etc/fluent-agent.logs) 17 | 18 | # SERVERNAME[:PORTNUM] 19 | # port number is optional (default: 24224) 20 | PRIMARY_SERVER="primary.fluentd.local:24224" 21 | 22 | ### or, PRIMARY SERVER LIST FILE of servers 23 | # PRIMARY_SERVERS_LIST="/etc/fluent-agent.servers.primary" 24 | 25 | # secondary server setting is optional... 26 | # SECONDARY_SERVER="secondary.fluentd.local:24224" 27 | 28 | # SECONDARY_SERVERS_LIST is available as like as PRIMARY_SERVERS_LIST 29 | 30 | # max bytes to try read as one action from tail (default: 1MB) 31 | # READ_BUFFER_SIZE=1048576 32 | 33 | # PROCESS_NICE default: 0 34 | # PROCESS_NICE=-1 35 | 36 | # TAIL_INTERVAL=0.1 37 | # TAIL_PATH=/usr/bin/tail 38 | 39 | # Tag , data and interval of ping message (not to output ping message when tag not specified) 40 | # PING_TAG=ping 41 | # PING_DATA=`hostname` 42 | # PING_INTERVAL=60 43 | 44 | # Tag name of 'drain_log' (none: not to output drain_log) 45 | # DRAIN_LOG_TAG= 46 | 47 | # connection keepalive time in seconds. 0 means infinity (DEFAULT: 1800) 48 | # KEEPALIVE_TIME=1800 49 | 50 | # The maximum wait time for TCP socket reconnection in seconds (Default: 3600, minimum: 0.5) 51 | # RECONNECT_WAIT_MAX=3600 52 | 53 | # The rate to increment the reconnect time (Default: 1.5) 54 | # RECONNECT_WAIT_INCR_RATE=1.5 55 | 56 | # LOG_PATH=/tmp/fluent-agent.log 57 | # LOG_VERBOSE=false 58 | 59 | # PERL_PATH=/usr/bin/perl 60 | -------------------------------------------------------------------------------- /package/fluent-agent-lite.init: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # fluent-agent-lite 4 | # chkconfig: - 85 15 5 | # description: log transfer agent, for Fluentd's 'forward' input. 6 | # config: /etc/fluent-agent-lite.conf 7 | 8 | prog="fluent-agent-lite" 9 | 10 | scriptpath="/usr/local/fluent-agent-lite/bin/"$prog 11 | 12 | if [ ! -x $scriptpath ]; then 13 | echo "Not installed properly" 14 | exit 1 15 | fi 16 | 17 | CONFIG_FILE=/etc/fluent-agent-lite.conf 18 | if [ ! -r $CONFIG_FILE ]; then 19 | echo "Config file does not exists: "$CONFIG_FILE 20 | exit 1 21 | fi 22 | source $CONFIG_FILE 23 | 24 | PID_FILE=/var/run/fluent-agent-lite.pid 25 | 26 | RETVAL=0 27 | 28 | start() { 29 | echo -n $"Starting $prog: " 30 | 31 | touch $PID_FILE 32 | if [ ! -w $PID_FILE ]; then 33 | echo "Cannot touch pid file: "$PID_FILE 34 | exit 1 35 | fi 36 | pids=$(cat $PID_FILE | wc -l | awk '{print $1;}') 37 | if [ "x""$pids" != 'x0' ]; then 38 | echo "process already running? check pidfile: "$PID_FILE 39 | return 1; 40 | fi 41 | 42 | lines=$(echo "$LOGS" | grep -v '^#' | grep -v '^$' | wc -l | awk '{print $1;}') 43 | if [ x"$LOGS" = "x" -o $lines -lt 1 ]; then 44 | echo "No input logfiles specified." 45 | return 1; 46 | fi 47 | if [ "x"$FORCE_START = "x" -o "x"$FORCE_START = "xno" -o "x"$FORCE_START = "x0" -o "x"$FORCE_START = "xfalse" ]; then 48 | for (( i = 0; i < $lines; i++ )); do 49 | lineno=$((i + 1)) 50 | line=$(echo "$LOGS" | grep -v '^#' | tail -n $lineno | head -n 1) 51 | path=$(echo $line | awk '{print $2;}') 52 | if [ ! -r "$path" ]; then 53 | echo "$path does not exist or is not readable." 54 | return 1; 55 | fi 56 | done 57 | fi 58 | for (( i = 0; i < $lines; i++ )); do 59 | COMMANDLINE=$(build_command_line $i) 60 | $COMMANDLINE & 61 | 62 | CPID=$! 63 | disown $CPID > /dev/null 2>&1 64 | echo $CPID >> $PID_FILE 65 | done 66 | RETVAL=1 67 | for (( retries = 0 ; retries < 10 ; retries++ )); do 68 | running=$(cat $PID_FILE | while read p ; do if ps -p $p > /dev/null; then echo -n ""; else echo -n "1"; fi; done); 69 | if [ "x"$running = "x" ]; then 70 | RETVAL=0 71 | break 72 | fi 73 | sleep 1 74 | done 75 | if [ $RETVAL = 0 ]; then 76 | echo "ok." 77 | else 78 | echo "failed. (timeout after 10sec)" 79 | fi 80 | return $RETVAL 81 | } 82 | stop() { 83 | echo -n $"Stopping $prog: " 84 | touch $PID_FILE 85 | if [ ! -w $PID_FILE ]; then 86 | echo "Cannot touch pid file: "$PID_FILE 87 | exit 1 88 | fi 89 | cat $PID_FILE | while read pid ; do 90 | kill -s TERM $pid > /dev/null 2>&1 91 | done 92 | RETVAL=1 93 | for (( retries = 0 ; retries < 10 ; retries++ )); do 94 | running=$(cat $PID_FILE | while read p ; do if ps -p $p > /dev/null; then echo -n "1"; fi; done); 95 | if [ "x"$running = "x" ]; then 96 | RETVAL=0 97 | break 98 | fi 99 | sleep 1 100 | done 101 | 102 | if [ $RETVAL = 0 ]; then 103 | echo "ok." 104 | rm -f $PID_FILE 105 | else 106 | echo "failed. (timeout after 10sec)" 107 | fi 108 | return $RETVAL 109 | } 110 | reload() { 111 | echo -n $"Reloading $prog: " 112 | cat $PID_FILE | while read pid ; do 113 | kill -s HUP $pid > /dev/null 2>&1 114 | done 115 | echo "reset connections." 116 | echo "To run with new config file, do 'restart'." 117 | RETVAL=0 118 | return $RETVAL 119 | } 120 | status() { 121 | echo -n "Checking $prog: " 122 | if [ ! -f $PID_FILE ]; then 123 | working=0 124 | else 125 | working=$(cat $PID_FILE | wc -l | awk '{print $1;}') 126 | fi 127 | if [ "x"$working != x"0" ]; then 128 | status=$(cat $PID_FILE | while read p ; do if ps -p $p > /dev/null; then echo -n "" ; else echo -n "1"; fi; done); 129 | fi 130 | RETVAL=1 131 | if [ "x""$working" = "x0" -o "x"$status = "x" ]; then 132 | RETVAL=0 133 | fi 134 | if [ $RETVAL = 0 ]; then 135 | if [ "x"$working = "x0" ]; then 136 | echo "not running, ok" 137 | # ref: http://refspecs.linuxbase.org/LSB_3.1.1/LSB-Core-generic/LSB-Core-generic/iniscrptact.html 138 | RETVAL=3 139 | else 140 | echo "ok." 141 | fi 142 | else 143 | echo "not running correctly." 144 | fi 145 | return $RETVAL 146 | } 147 | 148 | PREPARE_GLOBAL="" 149 | 150 | if [ "x"$PERL_PATH = "x" ]; then 151 | PERL_PATH="perl" 152 | fi 153 | 154 | NICE_PART="" 155 | TAG_PREFIX_STRING="" 156 | OPTIONS_PART="" 157 | ARGS_SERVER_PART="" 158 | function prepare_build { 159 | if [ "x"$PREPARE_GLOBAL != "x" ]; then 160 | return 161 | fi 162 | 163 | if [ "x"$TAG_PREFIX != "x" ]; then 164 | TAG_PREFIX_STRING=$TAG_PREFIX"." 165 | fi 166 | 167 | if [ "x"$PRIMARY_SERVER != "x" ]; then 168 | if [ "x"$SECONDARY_SERVER != "x" ]; then 169 | ARGS_SERVER_PART=$PRIMARY_SERVER" "$SECONDARY_SERVER 170 | else 171 | ARGS_SERVER_PART=$PRIMARY_SERVER 172 | fi 173 | else 174 | if [ "x"$PRIMARY_SERVERS_LIST != "x" ]; then 175 | OPTIONS_PART="-p "$PRIMARY_SERVERS_LIST 176 | if [ "x"$SECONDARY_SERVERS_LIST != "x" ]; then 177 | OPTIONS_PART=$OPTIONS_PART" -s "$SECONDARY_SERVERS_LIST 178 | fi 179 | else 180 | echo "Invalid configuration... PRIMARY_SERVER/PRIMARY_SERVERS not found" 181 | exit 1 182 | fi 183 | fi 184 | if [ "x"$FIELD_NAME != "x" ]; then 185 | OPTIONS_PART=$OPTIONS_PART" -f "$FIELD_NAME 186 | fi 187 | if [ "x"$READ_BUFFER_SIZE != "x" ]; then 188 | OPTIONS_PART=$OPTIONS_PART" -b "$READ_BUFFER_SIZE 189 | fi 190 | if [ "x"$PROCESS_NICE != "x" ]; then 191 | NICE_PART="nice -n "$PROCESS_NICE 192 | # OPTIONS_PART=$OPTIONS_PART" -n "$PROCESS_NICE 193 | # if perl script run with 'nice -n -1', then tail (forked by perl) runs also run as -1. 194 | fi 195 | if [ "x"$TAIL_PATH != "x" ]; then 196 | if [ ! -x $TAIL_PATH ]; then 197 | echo "invalid tail path: "$TAIL_PATH 198 | exit 1 199 | fi 200 | OPTIONS_PART=$OPTIONS_PART" -t "$TAIL_PATH 201 | fi 202 | if [ "x"$TAIL_INTERVAL != "x" ]; then 203 | OPTIONS_PART=$OPTIONS_PART" -i "$TAIL_INTERVAL 204 | fi 205 | if [ "x"$PING_TAG != "x" ]; then 206 | if [ "x"$PING_DATA = "x" ]; then 207 | echo "ping data PING_DATA not specified" 208 | exit 1 209 | fi 210 | OPTIONS_PART=$OPTIONS_PART" -P "$PING_TAG":"$PING_DATA 211 | if [ "x"$PING_INTERVAL != "x" ]; then 212 | OPTIONS_PART=$OPTIONS_PART" -S "$PING_INTERVAL 213 | fi 214 | fi 215 | if [ "x"$DRAIN_LOG_TAG != "x" ]; then 216 | OPTIONS_PART=$OPTIONS_PART" -d "$DRAIN_LOG_TAG 217 | fi 218 | if [ "x"$KEEPALIVE_TIME != "x" ]; then 219 | OPTIONS_PART=$OPTIONS_PART" -k "$KEEPALIVE_TIME 220 | fi 221 | if [ "x"$RECONNECT_WAIT_MAX != "x" ]; then 222 | OPTIONS_PART=$OPTIONS_PART" -w "$RECONNECT_WAIT_MAX 223 | fi 224 | if [ "x"$RECONNECT_WAIT_INCR_RATE != "x" ]; then 225 | OPTIONS_PART=$OPTIONS_PART" -r "$RECONNECT_WAIT_INCR_RATE 226 | fi 227 | if [ "x"$LOG_PATH != "x" ]; then 228 | OPTIONS_PART=$OPTIONS_PART" -l "$LOG_PATH 229 | fi 230 | if [ "x"$LOG_VERBOSE != "x" ]; then 231 | if [ $LOG_VERBOSE != "no" -a $LOG_VERBOSE != "0" -a $LOG_VERBOSE != "false" ]; then 232 | OPTIONS_PART=$OPTIONS_PART" -v" 233 | fi 234 | fi 235 | if [ "x"$FORCE_START != "x" ]; then 236 | if [ $FORCE_START != "no" -a $FORCE_START != "0" -a $FORCE_START != "false" ]; then 237 | OPTIONS_PART=$OPTIONS_PART" -F" 238 | fi 239 | fi 240 | PREPARE_GLOBAL="x" 241 | } 242 | 243 | function build_command_line { 244 | pos=$1 245 | 246 | prepare_build 247 | 248 | lineno=$((pos + 1)) 249 | line=$(echo "$LOGS" | grep -v '^#' | tail -n $lineno | head -n 1) 250 | 251 | tag=$(echo $line | awk '{print $1;}') 252 | tag=$TAG_PREFIX_STRING$tag 253 | 254 | path=$(echo $line | awk '{print $2;}') 255 | 256 | echo -n $NICE_PART $PERL_PATH $scriptpath $OPTIONS_PART $tag $path $ARGS_SERVER_PART 257 | return 0 258 | } 259 | 260 | case "$1" in 261 | start) 262 | start 263 | ;; 264 | stop) 265 | stop 266 | ;; 267 | status) 268 | status 269 | ;; 270 | restart) 271 | stop && start 272 | ;; 273 | reload) 274 | reload 275 | ;; 276 | *) 277 | echo $"Usage: $prog {start|stop|restart|reload|status}" 278 | exit 1 279 | esac 280 | 281 | exit $RETVAL 282 | --------------------------------------------------------------------------------