├── .gitignore
├── Gemfile
├── LICENSE
├── README.md
├── Rakefile
├── fluent-plugin-remote-syslog.gemspec
├── lib
    └── fluentd
    │   └── plugin
    │       ├── out_syslog.rb
    │       └── out_syslog_buffered.rb
└── test
    ├── helper.rb
    └── plugin
        └── test_out_syslog.rb


/.gitignore:
--------------------------------------------------------------------------------
1 | .idea
2 | 


--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
1 | source "http://rubygems.org"
2 | 
3 | gemspec


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2017 Docebo
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | fluent-plugin-remote-syslog
 2 | ===========================
 3 | 
 4 | fluentd plugin for streaming logs out to a remote syslog server or syslog SaaS service (like Papertrail)
 5 | 
 6 | #Available Plugins:
 7 | * out_syslog: registers itself as "syslog", and is the non-buffered implementation, communicating through UDP
 8 | * out_syslog_buffered: registers itself as "syslog_buffered", and is the buffered implementation, communicating through TCP
 9 | 
10 | #Plugin Settings:
11 | Both plugins have the same configuration options:
12 | 
13 | * remote_syslog: fqdn or ip of the remote syslog instance
14 | * port: the port, where the remote syslog instance is listening
15 | * hostname: hostname to be set for syslog messages
16 | * remove_tag_prefix: remove tag prefix for tag placeholder. 
17 | * tag_key: use the field specified in tag_key from record to set the syslog key
18 | * facility: Syslog log facility
19 | * severity: Syslog log severity
20 | * use_record: Use severity and facility from record if available
21 | * payload_key: Use the field specified in payload_key from record to set payload
22 | 
23 | #Configuration example:
24 | ```
25 | <match site.*>
26 |   type syslog_buffered
27 |   remote_syslog your.syslog.host
28 |   port 25
29 |   hostname ${hostname}
30 |   facility local0
31 |   severity debug
32 | </match>
33 | ```
34 | 
35 | 
36 | Contributors:
37 | 
38 | * Andrea Spoldi 
39 | * [deathowl](http://github.com/deathowl)
40 | 


--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env rake
 2 | require "bundler/gem_tasks"
 3 | 
 4 | require 'rake/testtask'
 5 | Rake::TestTask.new(:test) do |test|
 6 |   test.libs << 'lib' << 'test'
 7 |   test.pattern = 'test/**/test_*.rb'
 8 |   test.verbose = true
 9 | end
10 | 
11 | task :default => :test


--------------------------------------------------------------------------------
/fluent-plugin-remote-syslog.gemspec:
--------------------------------------------------------------------------------
 1 | # encoding: utf-8
 2 | $:.push File.expand_path('../lib', __FILE__)
 3 | 
 4 | Gem::Specification.new do |gem|
 5 |   gem.name        = "fluent-plugin-remote-syslog"
 6 |   gem.description = "Output plugin for streaming logs out to a remote syslog"
 7 |   gem.homepage    = "https://github.com/docebo/fluent-plugin-remote-syslog"
 8 |   gem.summary     = gem.description
 9 |   gem.version     = "1.0"
10 |   gem.authors     = ["Andrea Spoldi"]
11 |   gem.email       = "devops@docebo.com"
12 |   gem.has_rdoc    = false
13 |   gem.license     = 'MIT'
14 |   gem.files       = `git ls-files`.split("\n")
15 |   gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
16 |   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17 |   gem.require_paths = ['lib']
18 | 
19 |   gem.add_dependency "fluentd", "~> 0.10.45"
20 |   gem.add_dependency "fluent-mixin-config-placeholders", "~> 0.2.0"
21 |   gem.add_dependency "syslog_protocol"
22 |   gem.add_development_dependency "rake", ">= 0.9.2"
23 | end


--------------------------------------------------------------------------------
/lib/fluentd/plugin/out_syslog.rb:
--------------------------------------------------------------------------------
  1 | require 'fluent/mixin/config_placeholders'
  2 | module Fluent
  3 | 
  4 |   class SyslogOutput < Fluent::Output
  5 |     # First, register the plugin. NAME is the name of this plugin
  6 |     # and identifies the plugin in the configuration file.
  7 |     Fluent::Plugin.register_output('syslog', self)
  8 | 
  9 |     # This method is called before starting.
 10 | 
 11 |     config_param :remote_syslog, :string, :default => nil
 12 |     config_param :port, :integer, :default => 25
 13 |     config_param :hostname, :string, :default => ""
 14 |     config_param :remove_tag_prefix, :string, :default => nil
 15 |     config_param :tag_key, :string, :default => nil
 16 |     config_param :facility, :string, :default => 'user'
 17 |     config_param :severity, :string, :default => 'debug'
 18 |     config_param :use_record, :string, :default => nil
 19 |     config_param :payload_key, :string, :default => 'message'
 20 | 
 21 | 
 22 |     def initialize
 23 |       super
 24 |       require 'socket'
 25 |       require 'syslog_protocol'
 26 |     end
 27 | 
 28 |     def configure(conf)
 29 |       super
 30 |       if not conf['remote_syslog']
 31 |         raise Fluent::ConfigError.new("remote syslog required")
 32 |       end
 33 |       @socket = UDPSocket.new
 34 |       @packet = SyslogProtocol::Packet.new
 35 |       if remove_tag_prefix = conf['remove_tag_prefix']
 36 |           @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
 37 |       end
 38 |       @facilty = conf['facility']
 39 |       @severity = conf['severity']
 40 |       @use_record = conf['use_record']
 41 |       @payload_key = conf['payload_key']
 42 |       if not @payload_key
 43 |         @payload_key = "message"
 44 |       end
 45 |     end
 46 | 
 47 | 
 48 |     # This method is called when starting.
 49 |     def start
 50 |       super
 51 |     end
 52 | 
 53 |     # This method is called when shutting down.
 54 |     def shutdown
 55 |       super
 56 |     end
 57 | 
 58 |     # This method is called when an event reaches Fluentd.
 59 |     # 'es' is a Fluent::EventStream object that includes multiple events.
 60 |     # You can use 'es.each {|time,record| ... }' to retrieve events.
 61 |     # 'chain' is an object that manages transactions. Call 'chain.next' at
 62 |     # appropriate points and rollback if it raises an exception.
 63 |     def emit(tag, es, chain)
 64 |       tag = tag.sub(@remove_tag_prefix, '') if @remove_tag_prefix
 65 |       chain.next
 66 |       es.each {|time,record|
 67 |         @packet.hostname = hostname
 68 |         if @use_record
 69 |           @packet.facility = record['facility'] || @facilty
 70 |           @packet.severity = record['severity'] || @severity
 71 |         else
 72 |           @packet.facility = @facilty
 73 |           @packet.severity = @severity
 74 |         end
 75 |         if record['time']
 76 |           time = Time.parse(record['time'])
 77 |         else
 78 |           time = Time.now
 79 |         end
 80 |         @packet.time = time
 81 |         @packet.tag      = if tag_key
 82 |                               record[tag_key][0..31].gsub(/[\[\]]/,'') # tag is trimmed to 32 chars for syslog_protocol gem compatibility
 83 |                            else
 84 |                               tag[0..31] # tag is trimmed to 32 chars for syslog_protocol gem compatibility
 85 |                            end
 86 |         packet = @packet.dup
 87 |         packet.content = record[@payload_key]
 88 |         @socket.send(packet.assemble, 0, @remote_syslog, @port)
 89 |     }
 90 |     end
 91 |   end
 92 |   class Time < Time
 93 |     def timezone(timezone = 'UTC')
 94 |       old = ENV['TZ']
 95 |       utc = self.dup.utc
 96 |       ENV['TZ'] = timezone
 97 |       output = utc.localtime
 98 |       ENV['TZ'] = old
 99 |       output
100 |     end
101 |   end
102 | 
103 | end
104 | 


--------------------------------------------------------------------------------
/lib/fluentd/plugin/out_syslog_buffered.rb:
--------------------------------------------------------------------------------
  1 | require 'fluent/mixin/config_placeholders'
  2 | module Fluent
  3 |   class SyslogBufferedOutput < Fluent::BufferedOutput
  4 |     # First, register the plugin. NAME is the name of this plugin
  5 |     # and identifies the plugin in the configuration file.
  6 |     Fluent::Plugin.register_output('syslog_buffered', self)
  7 | 
  8 |     # This method is called before starting.
  9 | 
 10 |     config_param :remote_syslog, :string, :default => ""
 11 |     config_param :port, :integer, :default => 25
 12 |     config_param :hostname, :string, :default => ""
 13 |     config_param :remove_tag_prefix, :string, :default => nil
 14 |     config_param :tag_key, :string, :default => nil
 15 |     config_param :facility, :string, :default => 'user'
 16 |     config_param :severity, :string, :default => 'debug'
 17 |     config_param :use_record, :string, :default => nil
 18 |     config_param :payload_key, :string, :default => 'message'
 19 |     config_param :max_size, :integer, :default => 4096
 20 | 
 21 | 
 22 |     def initialize
 23 |       super
 24 |       require 'socket'
 25 |       require 'syslog_protocol'
 26 |       require 'timeout'
 27 |     end
 28 | 
 29 |     def configure(conf)
 30 |       super
 31 |       if not conf['remote_syslog']
 32 |         raise Fluent::ConfigError.new("remote syslog required")
 33 |       end
 34 |         @socket = create_tcp_socket(conf['remote_syslog'], conf['port'])
 35 |       @packet = SyslogProtocol::Packet.new
 36 |       if remove_tag_prefix = conf['remove_tag_prefix']
 37 |         @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
 38 |       end
 39 |       @facilty = conf['facility']
 40 |       @severity = conf['severity']
 41 |       @use_record = conf['use_record']
 42 |       @payload_key = conf['payload_key']
 43 |       if not @payload_key
 44 |         @payload_key = "message"
 45 |       end
 46 |       if conf['max_size']
 47 |         @max_size = conf['max_size'].to_i
 48 |       end
 49 |     end
 50 | 
 51 |     def format(tag, time, record)
 52 |       [tag, time, record].to_msgpack
 53 |     end
 54 | 
 55 |     def create_tcp_socket(host, port)
 56 |       begin
 57 |         Timeout.timeout(10) do
 58 |           begin
 59 |             socket = TCPSocket.new(host, port)
 60 |           rescue Errno::ENETUNREACH
 61 |             retry
 62 |           end
 63 |         end
 64 |         socket = TCPSocket.new(host, port)
 65 |         secs = Integer(1)
 66 |         usecs = Integer((1 - secs) * 1_000_000)
 67 |         optval = [secs, usecs].pack("l_2")
 68 |         socket.setsockopt Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, optval
 69 |       rescue SocketError, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EPIPE, Timeout::Error, OpenSSL::SSL::SSLError, Timeout::Error => e
 70 |         log.warn "out:syslog: failed to open tcp socket  #{@remote_syslog}:#{@port} :#{e}"
 71 |         socket = nil
 72 |       end
 73 |       socket
 74 |     end
 75 | 
 76 |     # This method is called when starting.
 77 |     def start
 78 |       super
 79 |     end
 80 | 
 81 |     # This method is called when shutting down.
 82 |     def shutdown
 83 |       super
 84 |     end
 85 | 
 86 | 
 87 |     def write(chunk)
 88 |       chunk.msgpack_each {|(tag,time,record)|
 89 |             send_to_syslog(tag, time, record)
 90 |           }
 91 |     end
 92 | 
 93 |     def send_to_syslog(tag, time, record)
 94 |       tag = tag.sub(@remove_tag_prefix, '') if @remove_tag_prefix
 95 |       @packet.hostname = hostname
 96 |       if @use_record
 97 |         @packet.facility = record['facility'] || @facilty
 98 |         @packet.severity = record['severity'] || @severity
 99 |       else
100 |         @packet.facility = @facilty
101 |         @packet.severity = @severity
102 |       end
103 |       if record['time']
104 |         time = Time.parse(record['time'])
105 |       else
106 |         time = Time.now
107 |       end
108 |       @packet.time = time
109 |       @packet.tag      = if tag_key
110 |                            record[tag_key][0..31].gsub(/[\[\]]/,'') # tag is trimmed to 32 chars for syslog_protocol gem compatibility
111 |                          else
112 |                            tag[0..31] # tag is trimmed to 32 chars for syslog_protocol gem compatibility
113 |                          end
114 |       packet = @packet.dup
115 |       packet.content = record[@payload_key]
116 |       begin
117 |         if not @socket
118 |           @socket = create_tcp_socket(@remote_syslog, @port)
119 |         end
120 |         if @socket
121 |           begin
122 |             @socket.write packet.assemble(@max_size) + "\n"
123 |             @socket.flush
124 |           rescue SocketError, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EPIPE, Timeout::Error, OpenSSL::SSL::SSLError => e
125 |             log.warn "out:syslog: connection error by #{@remote_syslog}:#{@port} :#{e}"
126 |             @socket = nil
127 |             raise #{e}
128 |           end
129 |         else
130 |           log.warn "out:syslog: Socket connection couldn't be reestablished"
131 |           raise #{e}
132 |         end
133 |       end
134 |     end
135 | 
136 | 
137 |   end
138 | 
139 |   class Time
140 |     def timezone(timezone = 'UTC')
141 |       old = ENV['TZ']
142 |       utc = self.dup.utc
143 |       ENV['TZ'] = timezone
144 |       output = utc.localtime
145 |       ENV['TZ'] = old
146 |       output
147 |     end
148 |   end
149 | end
150 | 
151 | 


--------------------------------------------------------------------------------
/test/helper.rb:
--------------------------------------------------------------------------------
 1 | require 'rubygems'
 2 | require 'bundler'
 3 | begin
 4 |   Bundler.setup(:default, :development)
 5 | rescue Bundler::BundlerError => e
 6 |   $stderr.puts e.message
 7 |   $stderr.puts "Run `bundle install` to install missing gems"
 8 |   exit e.status_code
 9 | end
10 | require 'test/unit'
11 | 
12 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13 | $LOAD_PATH.unshift(File.dirname(__FILE__))
14 | require 'fluent/test'
15 | unless ENV.has_key?('VERBOSE')
16 |   nulllogger = Object.new
17 |   nulllogger.instance_eval {|obj|
18 |     def method_missing(method, *args)
19 |       # pass
20 |     end
21 |   }
22 |   $log = nulllogger
23 | end
24 | 
25 | require 'fluentd/plugin/out_syslog'
26 | require 'fluentd/plugin/out_syslog_buffered'
27 | 
28 | 
29 | class Test::Unit::TestCase
30 | end


--------------------------------------------------------------------------------
/test/plugin/test_out_syslog.rb:
--------------------------------------------------------------------------------
  1 | require 'helper'
  2 | 
  3 | class SyslogOutputTest < Test::Unit::TestCase
  4 |   def setup
  5 |     Fluent::Test.setup
  6 |   end
  7 | 
  8 |   CONFIG = %[
  9 |     remote_syslog 127.0.0.1
 10 |     port    25
 11 |     hostname      testhost
 12 |     remove_tag_prefix        test
 13 |     severity  debug
 14 |     facility user
 15 |     payload_key message
 16 |   ]
 17 | 
 18 |   def create_driver(conf=CONFIG,tag='test')
 19 |     Fluent::Test::OutputTestDriver.new(Fluent::SyslogOutput, tag).configure(conf)
 20 |   end
 21 | 
 22 |   def test_configure
 23 |     assert_raise(Fluent::ConfigError) {
 24 |       d = create_driver('')
 25 |     }
 26 |     assert_raise(Fluent::ConfigError) {
 27 |       d = create_driver %[
 28 |       hostname      testhost
 29 |       remove_tag_prefix        test
 30 |       ]
 31 |     }
 32 |     assert_nothing_raised {
 33 |       d = create_driver %[
 34 |         remote_syslog 127.0.0.1
 35 |       ]
 36 |     }
 37 |     assert_nothing_raised {
 38 |       d = create_driver %[
 39 |         remote_syslog 127.0.0.1
 40 |         port    639
 41 |       ]
 42 |     }
 43 |     assert_nothing_raised {
 44 |       d = create_driver %[
 45 |         remote_syslog 127.0.0.1
 46 |         port    25
 47 |         hostname      deathstar
 48 |       ]
 49 |     }
 50 |     assert_nothing_raised {
 51 |       d = create_driver %[
 52 |        remote_syslog 127.0.0.1
 53 |         port    25
 54 |         hostname      testhost
 55 |         remove_tag_prefix        test123
 56 |       ]
 57 |     }
 58 |     assert_nothing_raised {
 59 |       d = create_driver %[
 60 |         remote_syslog 127.0.0.1
 61 |         port    25
 62 |         hostname      testhost
 63 |         remove_tag_prefix        test
 64 |         tag_key   tagtag
 65 |         severity  debug
 66 |       ]
 67 |     }
 68 |     assert_nothing_raised {
 69 |       d = create_driver %[
 70 |         remote_syslog 127.0.0.1
 71 |         port    25
 72 |         hostname      testhost
 73 |         remove_tag_prefix        test
 74 |         tag_key   tagtag
 75 |         severity  debug
 76 |         facility user
 77 |       ]
 78 |     }
 79 |     assert_nothing_raised {
 80 |       d = create_driver %[
 81 |         remote_syslog 127.0.0.1
 82 |         port    25
 83 |         hostname      testhost
 84 |         remove_tag_prefix        test
 85 |         tag_key   tagtag
 86 |         severity  debug
 87 |         facility user
 88 |         payload_key message
 89 |       ]
 90 |     }
 91 |     d = create_driver %[
 92 |         remote_syslog 127.0.0.1
 93 |         port    25
 94 |         hostname      testhost
 95 |         remove_tag_prefix        test
 96 |         tag_key   tagtag
 97 |         severity  debug
 98 |         facility user
 99 |         payload_key message
100 |     ]
101 |     assert_equal 25, d.instance.port
102 |     assert_equal "127.0.0.1", d.instance.remote_syslog
103 |     assert_equal "testhost", d.instance.hostname
104 |     assert_equal Regexp.new('^' + Regexp.escape("test")), d.instance.remove_tag_prefix
105 |     assert_equal "tagtag", d.instance.tag_key
106 |     assert_equal "debug", d.instance.severity
107 |     assert_equal "user", d.instance.facility
108 |     assert_equal "message", d.instance.payload_key
109 | 
110 |   end
111 |   def test_emit
112 |     d1 = create_driver(CONFIG, 'test.in')
113 |     d1.run do
114 |       d1.emit({'message' => 'asd asd'})
115 |       d1.emit({'message' => 'dsa xasd'})
116 |       d1.emit({'message' => 'ddd ddddd'})
117 |       d1.emit({'message' => '7sssss8 ssssdasd'})
118 |       d1.emit({'message' => 'aaassddffg asdasdasfasf'})
119 |     end
120 |     assert_equal 0, d1.emits.size
121 | 
122 |   end
123 | 
124 |   def test_emit_with_time_and_without_time
125 |     d1 = create_driver(CONFIG, 'test.in')
126 |     d1.run do
127 |       d1.emit({'message' => 'asd asd', 'time' => '2007-01-31 12:22:26'})
128 |       d1.emit({'message' => 'dsa xasd'})
129 |       d1.emit({'message' => 'ddd ddddd', 'time' => '2007-03-01 12:22:26'})
130 |       d1.emit({'message' => '7sssss8 ssssdasd', 'time' => '2011-03-01 12:22:26'})
131 |       d1.emit({'message' => 'aaassddffg asdasdasfasf', 'time' => '2016-03-01 12:22:26'})
132 |     end
133 |     assert_equal 0, d1.emits.size
134 | 
135 |   end
136 | 
137 | 
138 | end
139 | 


--------------------------------------------------------------------------------