├── .gitignore ├── Gemfile ├── Gemfile.lock ├── README.md ├── Rakefile ├── lib └── logstash │ └── inputs │ └── log4j2.rb └── logstash-input-log4j2.gemspec /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | /.config 4 | /coverage/ 5 | /InstalledFiles 6 | /pkg/ 7 | /spec/reports/ 8 | /test/tmp/ 9 | /test/version_tmp/ 10 | /tmp/ 11 | 12 | ## Specific to RubyMotion: 13 | .dat* 14 | .repl_history 15 | build/ 16 | 17 | ## Documentation cache and generated files: 18 | /.yardoc/ 19 | /_yardoc/ 20 | /doc/ 21 | /rdoc/ 22 | 23 | ## Environment normalisation: 24 | /.bundle/ 25 | /lib/bundler/man/ 26 | 27 | # for a library or gem, you might want to ignore these files since the code is 28 | # intended to run in multiple environments; otherwise, check them in: 29 | # Gemfile.lock 30 | # .ruby-version 31 | # .ruby-gemset 32 | 33 | # unless supporting rvm < 1.11.0 or doing something fancy, ignore this: 34 | .rvmrc 35 | 36 | # vendor files (jars) 37 | vendor/ 38 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec 3 | gem "logstash", :github => "elastic/logstash", :branch => "2.1" 4 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | logstash-input-log4j2 (5.2-java) 5 | logstash-core (>= 2.0.0.beta2, < 3.0.0) 6 | 7 | GEM 8 | remote: https://rubygems.org/ 9 | specs: 10 | cabin (0.7.2) 11 | clamp (0.6.5) 12 | coderay (1.1.0) 13 | concurrent-ruby (0.9.2-java) 14 | diff-lcs (1.2.5) 15 | ffi (1.9.10-java) 16 | filesize (0.0.4) 17 | gem_publisher (1.5.0) 18 | gems (0.8.3) 19 | i18n (0.6.9) 20 | insist (1.0.0) 21 | jrjackson (0.3.7) 22 | jruby-openssl (0.9.12-java) 23 | kramdown (1.9.0) 24 | logstash-core (2.1.1-java) 25 | cabin (~> 0.7.0) 26 | clamp (~> 0.6.5) 27 | concurrent-ruby (= 0.9.2) 28 | filesize (= 0.0.4) 29 | gems (~> 0.8.3) 30 | i18n (= 0.6.9) 31 | jrjackson (~> 0.3.7) 32 | jruby-openssl (>= 0.9.11) 33 | minitar (~> 0.5.4) 34 | pry (~> 0.10.1) 35 | rubyzip (~> 1.1.7) 36 | stud (~> 0.0.19) 37 | thread_safe (~> 0.3.5) 38 | treetop (< 1.5.0) 39 | logstash-devutils (0.0.18-java) 40 | gem_publisher 41 | insist (= 1.0.0) 42 | kramdown 43 | minitar 44 | rake 45 | rspec (~> 3.1.0) 46 | rspec-wait 47 | stud (>= 0.0.20) 48 | method_source (0.8.2) 49 | minitar (0.5.4) 50 | polyglot (0.3.5) 51 | pry (0.10.3-java) 52 | coderay (~> 1.1.0) 53 | method_source (~> 0.8.1) 54 | slop (~> 3.4) 55 | spoon (~> 0.0) 56 | rake (10.4.2) 57 | rspec (3.1.0) 58 | rspec-core (~> 3.1.0) 59 | rspec-expectations (~> 3.1.0) 60 | rspec-mocks (~> 3.1.0) 61 | rspec-core (3.1.7) 62 | rspec-support (~> 3.1.0) 63 | rspec-expectations (3.1.2) 64 | diff-lcs (>= 1.2.0, < 2.0) 65 | rspec-support (~> 3.1.0) 66 | rspec-mocks (3.1.3) 67 | rspec-support (~> 3.1.0) 68 | rspec-support (3.1.2) 69 | rspec-wait (0.0.8) 70 | rspec (>= 2.11, < 3.5) 71 | rubyzip (1.1.7) 72 | slop (3.6.0) 73 | spoon (0.0.4) 74 | ffi 75 | stud (0.0.22) 76 | thread_safe (0.3.5-java) 77 | treetop (1.4.15) 78 | polyglot 79 | polyglot (>= 0.3.1) 80 | 81 | PLATFORMS 82 | java 83 | 84 | DEPENDENCIES 85 | logstash-devutils 86 | logstash-input-log4j2! 87 | 88 | BUNDLED WITH 89 | 1.10.6 90 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # logstash-log4j2 2 | 3 | Log4j2 plugin for logstash. 4 | 5 | ## Supported Log4J2 versions: 6 | Version: 2.1+ 7 | 8 | ## Get the plugin 9 | 10 | ### Logstash 1.5+ 11 | 12 | Use the install method 13 | 14 | ```$LS_HOME/bin/plugin install logstash-input-log4j2``` 15 | 16 | ### Logstash 1.4 17 | 18 | Download the latest release at: https://github.com/jurmous/logstash-log4j2/releases and unzip it. 19 | 20 | If you download the source you also need rake to run ```rake vendor``` to download the correct log4j2 jars. 21 | 22 | To run the plugin you need to start logstash with the plugin path `./bin/logstash --pluginpath PATH_TO_PLUGIN -f YOUR_CONF.conf` 23 | 24 | ## Setup log4j2 25 | Set log4j2.xml in your project 26 | ```xml 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | ``` 41 | 42 | ## Setup Logstash 43 | 44 | ``` 45 | input { 46 | log4j2 { 47 | port => 7000 48 | mode => "server" 49 | } 50 | } 51 | 52 | output { 53 | stdout { codec => rubydebug } 54 | } 55 | ``` 56 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | @files=[ 2 | {'url' => "https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/2.1/log4j-core-2.1.jar", 'sha1' => '31823dcde108f2ea4a5801d1acc77869d7696533' }, 3 | {'url' => "https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/2.1/log4j-api-2.1.jar", 'sha1' => '588c32c91544d80cc706447aa2b8037230114931'}, 4 | {'url' => "https://repo1.maven.org/maven2/com/lmax/disruptor/3.3.0/disruptor-3.3.0.jar", 'sha1' => '6925c7787741f6ac1535188ea450f04fa1246acf'} 5 | ] 6 | 7 | task :default do 8 | system("rake -T") 9 | end 10 | 11 | require "logstash/devutils/rake" 12 | -------------------------------------------------------------------------------- /lib/logstash/inputs/log4j2.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require "logstash/inputs/base" 3 | require "logstash/errors" 4 | require "logstash/environment" 5 | require "logstash/namespace" 6 | require "logstash/util/socket_peer" 7 | require "socket" 8 | require "timeout" 9 | 10 | # Read events over a TCP socket from a Log4j2 SocketAppender. 11 | # 12 | # Can either accept connections from clients or connect to a server, 13 | # depending on `mode`. Depending on which `mode` is configured, 14 | # you need a matching SocketAppender or a SocketHubAppender 15 | # on the remote side. 16 | class LogStash::Inputs::Log4j2 < LogStash::Inputs::Base 17 | 18 | config_name "log4j2" 19 | milestone 1 20 | 21 | # When mode is `server`, the address to listen on. 22 | # When mode is `client`, the address to connect to. 23 | config :host, :validate => :string, :default => "0.0.0.0" 24 | 25 | # When mode is `server`, the port to listen on. 26 | # When mode is `client`, the port to connect to. 27 | config :port, :validate => :number, :default => 4560 28 | 29 | # Read timeout in seconds. If a particular TCP connection is 30 | # idle for more than this timeout period, we will assume 31 | # it is dead and close it. 32 | # If you never want to timeout, use -1. 33 | config :data_timeout, :validate => :number, :default => 5 34 | 35 | # Mode to operate in. `server` listens for client connections, 36 | # `client` connects to a server. 37 | config :mode, :validate => ["server", "client"], :default => "server" 38 | 39 | def initialize(*args) 40 | super(*args) 41 | end # def initialize 42 | 43 | public 44 | def register 45 | require "java" 46 | require "jruby/serialization" 47 | 48 | begin 49 | vendor_dir = ::File.expand_path("../../../vendor/", ::File.dirname(__FILE__)) 50 | require File.join(vendor_dir, "log4j-api-2.1.jar") 51 | require File.join(vendor_dir, "log4j-core-2.1.jar") 52 | require File.join(vendor_dir, "disruptor-3.3.0.jar") 53 | 54 | Java::OrgApacheLoggingLog4jCoreImpl.const_get("Log4jLogEvent") 55 | rescue 56 | raise(LogStash::PluginLoadingError, "Log4j2 java library not loaded") 57 | end 58 | 59 | if server? 60 | @logger.info("Starting Log4j2 input listener", :address => "#{@host}:#{@port}") 61 | @server_socket = TCPServer.new(@host, @port) 62 | end 63 | @logger.info("Log4j input") 64 | end # def register 65 | 66 | private 67 | def pretty_print_stack_trace(proxy) 68 | indentation = "\n\t" 69 | result = "#{proxy.getName}: #{proxy.getMessage.to_s}#{indentation}#{proxy.getExtendedStackTrace.to_a.join(indentation)}" 70 | cause = proxy.getCauseProxy 71 | while cause 72 | result += "\nCaused by: #{cause.getName}: #{cause.getMessage.to_s}#{indentation}#{cause.getExtendedStackTrace.to_a.join(indentation)}" 73 | cause = cause.getCauseProxy 74 | end 75 | result 76 | end 77 | 78 | private 79 | def handle_socket(socket, output_queue) 80 | begin 81 | # JRubyObjectInputStream uses JRuby class path to find the class to de-serialize to 82 | ois = JRubyObjectInputStream.new(java.io.BufferedInputStream.new(socket.to_inputstream)) 83 | loop do 84 | # NOTE: log4j_obj is org.apache.logging.log4j.core.impl.Log4jLogEvent 85 | log4j_obj = ois.readObject 86 | event = LogStash::Event.new("message" => log4j_obj.getMessage.getFormattedMessage, LogStash::Event::TIMESTAMP => Time.at(log4j_obj.getTimeMillis/1000,log4j_obj.getTimeMillis%1000*1000).gmtime) 87 | decorate(event) 88 | event["host"] = socket.peer 89 | event["marker"] = log4j_obj.getMarker.getName if log4j_obj.getMarker 90 | event["path"] = log4j_obj.getLoggerName 91 | event["priority"] = log4j_obj.getLevel.toString 92 | event["logger_name"] = log4j_obj.getLoggerName 93 | event["thread"] = log4j_obj.getThreadName 94 | if log4j_obj.getSource() 95 | event["class"] = log4j_obj.getSource().getClassName 96 | event["file"] = log4j_obj.getSource().getFileName + ":" + log4j_obj.getSource().getLineNumber.to_s 97 | event["method"] = log4j_obj.getSource().getMethodName 98 | end 99 | # Add the context properties to '@fields' 100 | if log4j_obj.contextMap 101 | log4j_obj.contextMap.keySet.each do |key| 102 | event["cmap_"+key] = log4j_obj.contextMap.get(key) 103 | end 104 | end 105 | 106 | proxy = log4j_obj.getThrownProxy 107 | if proxy 108 | event["stack_trace"] = pretty_print_stack_trace(proxy) 109 | end 110 | 111 | event["cstack"] = log4j_obj.getContextStack.to_a if log4j_obj.getContextStack 112 | output_queue << event 113 | end # loop do 114 | rescue => e 115 | @logger.debug(e) 116 | @logger.debug("Closing connection", :client => socket.peer, 117 | :exception => e) 118 | rescue Timeout::Error 119 | @logger.debug("Closing connection after read timeout", 120 | :client => socket.peer) 121 | end # begin 122 | ensure 123 | begin 124 | socket.close 125 | rescue IOError 126 | pass 127 | end # begin 128 | end 129 | 130 | private 131 | def server? 132 | @mode == "server" 133 | end # def server? 134 | 135 | private 136 | def readline(socket) 137 | line = socket.readline 138 | end # def readline 139 | 140 | public 141 | def run(output_queue) 142 | if server? 143 | loop do 144 | # Start a new thread for each connection. 145 | Thread.start(@server_socket.accept) do |s| 146 | # TODO(sissel): put this block in its own method. 147 | 148 | # monkeypatch a 'peer' method onto the socket. 149 | s.instance_eval { class << self; include ::LogStash::Util::SocketPeer end } 150 | @logger.debug("Accepted connection", :client => s.peer, 151 | :server => "#{@host}:#{@port}") 152 | handle_socket(s, output_queue) 153 | end # Thread.start 154 | end # loop 155 | else 156 | loop do 157 | client_socket = TCPSocket.new(@host, @port) 158 | client_socket.instance_eval { class << self; include ::LogStash::Util::SocketPeer end } 159 | @logger.debug("Opened connection", :client => "#{client_socket.peer}") 160 | handle_socket(client_socket, output_queue) 161 | end # loop 162 | end 163 | end # def run 164 | end # class LogStash::Inputs::Log4j2 165 | -------------------------------------------------------------------------------- /logstash-input-log4j2.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | s.name = 'logstash-input-log4j2' 3 | s.version = '5.2' 4 | s.licenses = ['Apache License (2.0)'] 5 | s.summary = "Read events over a TCP socket from a Log4j2 SocketAppender" 6 | s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" 7 | s.authors = ["Jurriaan Mous"] 8 | s.email = 'jurmous@jurmo.us' 9 | s.homepage = "https://github.com/jurmous/logstash-log4j2" 10 | s.require_paths = ["lib"] 11 | 12 | # Files 13 | #s.files = `git ls-files`.split($\)+::Dir.glob('vendor/*') 14 | 15 | 16 | s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT'] 17 | 18 | # Tests 19 | s.test_files = s.files.grep(%r{^(test|spec|features)/}) 20 | 21 | # Special flag to let us know this is actually a logstash plugin 22 | s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" } 23 | 24 | s.platform = 'java' 25 | 26 | s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0" 27 | 28 | s.add_development_dependency 'logstash-devutils' 29 | end 30 | --------------------------------------------------------------------------------