├── .gitignore ├── .travis.yml ├── Rakefile ├── NOTICE.TXT ├── .github ├── PULL_REQUEST_TEMPLATE.md ├── ISSUE_TEMPLATE.md └── CONTRIBUTING.md ├── Gemfile ├── CONTRIBUTORS ├── logstash-input-syslog.gemspec ├── CHANGELOG.md ├── README.md ├── docs └── index.asciidoc ├── LICENSE ├── lib └── logstash │ └── inputs │ └── syslog.rb └── spec └── inputs └── syslog_spec.rb /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | Gemfile.lock 3 | .bundle 4 | vendor 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | import: 2 | - logstash-plugins/.ci:travis/travis.yml@1.x -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | @files=[] 2 | 3 | task :default do 4 | system("rake -T") 5 | end 6 | 7 | require "logstash/devutils/rake" 8 | -------------------------------------------------------------------------------- /NOTICE.TXT: -------------------------------------------------------------------------------- 1 | Elasticsearch 2 | Copyright 2012-2018 Elasticsearch 3 | 4 | This product includes software developed by The Apache Software 5 | Foundation (http://www.apache.org/). -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Thanks for contributing to Logstash! If you haven't already signed our CLA, here's a handy link: https://www.elastic.co/contributor-agreement/ 2 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | 5 | logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash" 6 | use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1" 7 | 8 | if Dir.exist?(logstash_path) && use_logstash_source 9 | gem 'logstash-core', :path => "#{logstash_path}/logstash-core" 10 | gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api" 11 | end 12 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Please post all product and debugging questions on our [forum](https://discuss.elastic.co/c/logstash). Your questions will reach our wider community members there, and if we confirm that there is a bug, then we can open a new issue here. 2 | 3 | For all general issues, please provide the following details for fast resolution: 4 | 5 | - Version: 6 | - Operating System: 7 | - Config File (if you have sensitive info, please remove it): 8 | - Sample Data: 9 | - Steps to Reproduce: 10 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | The following is a list of people who have contributed ideas, code, bug 2 | reports, or in general have helped logstash along its way. 3 | 4 | Contributors: 5 | * Adrien Anceau (nwknu) 6 | * Bernd Ahlers (bernd) 7 | * Brad Ison (bison) 8 | * Colin Surprenant (colinsurprenant) 9 | * James Turnbull (jamtur01) 10 | * Joel Merrick (joelio) 11 | * John E. Vincent (lusis) 12 | * Jordan Sissel (jordansissel) 13 | * Joshua Bussdieker (jbussdieker) 14 | * Kurt Hurtado (kurtado) 15 | * Pete Fritchman (fetep) 16 | * Pier-Hugues Pellerin (ph) 17 | * Richard Pijnenburg (electrical) 18 | * Simon Mulser (simonmulser) 19 | * Suyog Rao (suyograo) 20 | * coagentpai 21 | 22 | Note: If you've sent us patches, bug reports, or otherwise contributed to 23 | Logstash, and you aren't on the list above and want to be, please let us know 24 | and we'll make sure you're here. Contributions from folks like you are what make 25 | open source awesome. 26 | -------------------------------------------------------------------------------- /logstash-input-syslog.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | 3 | s.name = 'logstash-input-syslog' 4 | s.version = '3.7.1' 5 | s.licenses = ['Apache License (2.0)'] 6 | s.summary = "Reads syslog messages as events" 7 | s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" 8 | s.authors = ["Elastic"] 9 | s.email = 'info@elastic.co' 10 | s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" 11 | s.require_paths = ["lib"] 12 | 13 | # Files 14 | s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"] 15 | 16 | # Tests 17 | s.test_files = s.files.grep(%r{^(test|spec|features)/}) 18 | 19 | # Special flag to let us know this is actually a logstash plugin 20 | s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" } 21 | 22 | # Gem dependencies 23 | s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" 24 | s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.2' 25 | 26 | s.add_runtime_dependency 'concurrent-ruby' 27 | s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0' 28 | 29 | s.add_runtime_dependency 'logstash-codec-plain' 30 | s.add_runtime_dependency 'logstash-filter-grok', '>= 4.4.1' 31 | s.add_runtime_dependency 'logstash-filter-date' 32 | 33 | s.add_development_dependency 'logstash-devutils', '~> 2.3' 34 | s.add_development_dependency 'logstash-codec-cef' 35 | end 36 | 37 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 3.7.1 2 | - Fix issue where the priority field was not being set correctly when grok failed [#76](https://github.com/logstash-plugins/logstash-input-syslog/pull/78) 3 | 4 | ## 3.7.0 5 | - Changed the TCP reading mode to use the non-blocking method [#75](https://github.com/logstash-plugins/logstash-input-syslog/pull/75) 6 | It fixes the high CPU usage when TCP clients do not properly disconnect/send EOF. 7 | - Fixed broken tests 8 | 9 | ## 3.6.0 10 | - Add support for ECS v8 as alias to v1 implementation [#68](https://github.com/logstash-plugins/logstash-input-syslog/pull/68) 11 | 12 | ## 3.5.0 13 | - Feat: ECS compatibility support [#63](https://github.com/logstash-plugins/logstash-input-syslog/pull/63) 14 | 15 | ## 3.4.5 16 | - Added support for listening on IPv6 addresses 17 | 18 | ## 3.4.4 19 | - Refactor: avoid global side-effect + cleanup [#62](https://github.com/logstash-plugins/logstash-input-syslog/pull/62) 20 | * avoid setting `BasicSocket.do_not_reverse_lookup` as it has side effects for others 21 | 22 | ## 3.4.3 23 | - [DOC] Added expanded descriptions and requirements for facility_labels and severity_labels. [#52](https://github.com/logstash-plugins/logstash-input-syslog/pull/52) 24 | 25 | ## 3.4.2 26 | - Remove (deprecated) dependency on thread_safe gem. 27 | - CI: upgrade testing [#58](https://github.com/logstash-plugins/logstash-input-syslog/pull/58) 28 | - [DOC] Correct example for `timezone` option [#53](https://github.com/logstash-plugins/logstash-input-syslog/pull/53) 29 | 30 | ## 3.4.1 31 | - Docs: Set the default_codec doc attribute. 32 | 33 | ## 3.4.0 34 | - Allow the syslog field to be a configurable option. This is useful for when codecs change 35 | the field containing the syslog data (e.g. the CEF codec). 36 | 37 | ## 3.3.0 38 | - Make the grok pattern a configurable option 39 | 40 | ## 3.2.4 41 | - Fix issue where stopping a pipeline (e.g., while reloading configuration) with active inbound syslog connections could cause Logstash to crash 42 | 43 | ## 3.2.3 44 | - Update gemspec summary 45 | 46 | ## 3.2.2 47 | - Fix some documentation issues 48 | 49 | ## 3.2.0 50 | - Add support for proxy protocol. 51 | 52 | ## 3.1.1 53 | - Move one log message from info to debug to avoid noise 54 | 55 | ## 3.1.0 56 | - Add metrics for events, messages received, errors and connections attemps happening during execution time. 57 | 58 | ## 3.0.2 59 | - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99 60 | 61 | ## 3.0.1 62 | - Republish all the gems under jruby. 63 | ## 3.0.0 64 | - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141 65 | # 2.0.5 66 | - Temporary specs fix, see https://github.com/logstash-plugins/logstash-input-syslog/pull/25 67 | # 2.0.4 68 | - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash 69 | # 2.0.3 70 | - New dependency requirements for logstash-core for the 5.0 release 71 | ## 2.0.0 72 | - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 73 | instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895 74 | - Dependency on logstash-core update to 2.0 75 | 76 | # 1.0.1 77 | - fix deprecation warning from `concurrent-ruby` 78 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Logstash 2 | 3 | All contributions are welcome: ideas, patches, documentation, bug reports, 4 | complaints, etc! 5 | 6 | Programming is not a required skill, and there are many ways to help out! 7 | It is more important to us that you are able to contribute. 8 | 9 | That said, some basic guidelines, which you are free to ignore :) 10 | 11 | ## Want to learn? 12 | 13 | Want to lurk about and see what others are doing with Logstash? 14 | 15 | * The irc channel (#logstash on irc.freenode.org) is a good place for this 16 | * The [forum](https://discuss.elastic.co/c/logstash) is also 17 | great for learning from others. 18 | 19 | ## Got Questions? 20 | 21 | Have a problem you want Logstash to solve for you? 22 | 23 | * You can ask a question in the [forum](https://discuss.elastic.co/c/logstash) 24 | * Alternately, you are welcome to join the IRC channel #logstash on 25 | irc.freenode.org and ask for help there! 26 | 27 | ## Have an Idea or Feature Request? 28 | 29 | * File a ticket on [GitHub](https://github.com/elastic/logstash/issues). Please remember that GitHub is used only for issues and feature requests. If you have a general question, the [forum](https://discuss.elastic.co/c/logstash) or IRC would be the best place to ask. 30 | 31 | ## Something Not Working? Found a Bug? 32 | 33 | If you think you found a bug, it probably is a bug. 34 | 35 | * If it is a general Logstash or a pipeline issue, file it in [Logstash GitHub](https://github.com/elasticsearch/logstash/issues) 36 | * If it is specific to a plugin, please file it in the respective repository under [logstash-plugins](https://github.com/logstash-plugins) 37 | * or ask the [forum](https://discuss.elastic.co/c/logstash). 38 | 39 | # Contributing Documentation and Code Changes 40 | 41 | If you have a bugfix or new feature that you would like to contribute to 42 | logstash, and you think it will take more than a few minutes to produce the fix 43 | (ie; write code), it is worth discussing the change with the Logstash users and developers first! You can reach us via [GitHub](https://github.com/elastic/logstash/issues), the [forum](https://discuss.elastic.co/c/logstash), or via IRC (#logstash on freenode irc) 44 | Please note that Pull Requests without tests will not be merged. If you would like to contribute but do not have experience with writing tests, please ping us on IRC/forum or create a PR and ask our help. 45 | 46 | ## Contributing to plugins 47 | 48 | Check our [documentation](https://www.elastic.co/guide/en/logstash/current/contributing-to-logstash.html) on how to contribute to plugins or write your own! It is super easy! 49 | 50 | ## Contribution Steps 51 | 52 | 1. Test your changes! [Run](https://github.com/elastic/logstash#testing) the test suite 53 | 2. Please make sure you have signed our [Contributor License 54 | Agreement](https://www.elastic.co/contributor-agreement/). We are not 55 | asking you to assign copyright to us, but to give us the right to distribute 56 | your code without restriction. We ask this of all contributors in order to 57 | assure our users of the origin and continuing existence of the code. You 58 | only need to sign the CLA once. 59 | 3. Send a pull request! Push your changes to your fork of the repository and 60 | [submit a pull 61 | request](https://help.github.com/articles/using-pull-requests). In the pull 62 | request, describe what your changes do and mention any bugs/issues related 63 | to the pull request. 64 | 65 | 66 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Logstash Plugin 2 | 3 | [![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-input-syslog.svg)](https://travis-ci.com/logstash-plugins/logstash-input-syslog) 4 | 5 | This is a plugin for [Logstash](https://github.com/elastic/logstash). 6 | 7 | It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way. 8 | 9 | ## Documentation 10 | 11 | Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/). 12 | 13 | - For formatting code or config example, you can use the asciidoc `[source,ruby]` directive 14 | - For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide 15 | 16 | ## Need Help? 17 | 18 | Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum. 19 | 20 | ## Developing 21 | 22 | ### 1. Plugin Developement and Testing 23 | 24 | #### Code 25 | - To get started, you'll need JRuby with the Bundler gem installed. 26 | 27 | - Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example). 28 | 29 | - Install dependencies 30 | ```sh 31 | bundle install 32 | ``` 33 | 34 | #### Test 35 | 36 | - Update your dependencies 37 | 38 | ```sh 39 | bundle install 40 | ``` 41 | 42 | - Run tests 43 | 44 | ```sh 45 | bundle exec rspec 46 | ``` 47 | 48 | ### 2. Running your unpublished Plugin in Logstash 49 | 50 | #### 2.1 Run in a local Logstash clone 51 | 52 | - Edit Logstash `Gemfile` and add the local plugin path, for example: 53 | ```ruby 54 | gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome" 55 | ``` 56 | - Install plugin 57 | ```sh 58 | # Logstash 2.3 and higher 59 | bin/logstash-plugin install --no-verify 60 | 61 | # Prior to Logstash 2.3 62 | bin/plugin install --no-verify 63 | 64 | ``` 65 | - Run Logstash with your plugin 66 | ```sh 67 | bin/logstash -e 'filter {awesome {}}' 68 | ``` 69 | At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash. 70 | 71 | #### 2.2 Run in an installed Logstash 72 | 73 | You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using: 74 | 75 | - Build your plugin gem 76 | ```sh 77 | gem build logstash-filter-awesome.gemspec 78 | ``` 79 | - Install the plugin from the Logstash home 80 | ```sh 81 | # Logstash 2.3 and higher 82 | bin/logstash-plugin install --no-verify 83 | 84 | # Prior to Logstash 2.3 85 | bin/plugin install --no-verify 86 | 87 | ``` 88 | - Start Logstash and proceed to test the plugin 89 | 90 | ## Contributing 91 | 92 | All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin. 93 | 94 | Programming is not a required skill. Whatever you've seen about open source and maintainers or community members saying "send patches or die" - you will not see that here. 95 | 96 | It is more important to the community that you are able to contribute. 97 | 98 | For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file. -------------------------------------------------------------------------------- /docs/index.asciidoc: -------------------------------------------------------------------------------- 1 | :plugin: syslog 2 | :type: input 3 | :default_codec: plain 4 | 5 | /////////////////////////////////////////// 6 | START - GENERATED VARIABLES, DO NOT EDIT! 7 | /////////////////////////////////////////// 8 | :version: %VERSION% 9 | :release_date: %RELEASE_DATE% 10 | :changelog_url: %CHANGELOG_URL% 11 | :include_path: ../../../../logstash/docs/include 12 | /////////////////////////////////////////// 13 | END - GENERATED VARIABLES, DO NOT EDIT! 14 | /////////////////////////////////////////// 15 | 16 | [id="plugins-{type}s-{plugin}"] 17 | 18 | === Syslog input plugin 19 | 20 | include::{include_path}/plugin_header.asciidoc[] 21 | 22 | ==== Description 23 | 24 | Read syslog messages as events over the network. 25 | 26 | This input is a good choice if you already use syslog today. 27 | It is also a good choice if you want to receive logs from 28 | appliances and network devices where you cannot run your own 29 | log collector. 30 | 31 | Of course, 'syslog' is a very muddy term. By default, this input only 32 | supports `RFC3164` syslog with some small modifications. However, some 33 | non-standard syslog formats can be read and parsed if a functional 34 | `grok_pattern` is provided. The date format is still only allowed to be 35 | `RFC3164` style or `ISO8601`. 36 | 37 | For more information see the http://www.ietf.org/rfc/rfc3164.txt[RFC3164 page]. 38 | 39 | Note: This input will start listeners on both TCP and UDP. 40 | 41 | 42 | [id="plugins-{type}s-{plugin}-options"] 43 | ==== Syslog Input Configuration Options 44 | 45 | This plugin supports the following configuration options plus the <> described later. 46 | 47 | [cols="<,<,<",options="header",] 48 | |======================================================================= 49 | |Setting |Input type|Required 50 | | <> | <>|No 51 | | <> |<>|No 52 | | <> |<>|No 53 | | <> |<>|No 54 | | <> |<>|No 55 | | <> |<>|No 56 | | <> |<>|No 57 | | <> |<>|No 58 | | <> |<>|No 59 | | <> |<>|No 60 | | <> |<>|No 61 | |======================================================================= 62 | 63 | Also see <> for a list of options supported by all 64 | input plugins. 65 | 66 |   67 | 68 | [id="plugins-{type}s-{plugin}-ecs_compatibility"] 69 | ===== `ecs_compatibility` 70 | 71 | * Value type is <> 72 | * Supported values are: 73 | ** `disabled`: does not use ECS-compatible field names (for example, `priority` for syslog priority) 74 | ** `v1`,`v8`: uses fields that are compatible with Elastic Common Schema (for example, `[log][syslog][priority]`) 75 | * Default value depends on which version of Logstash is running: 76 | ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default 77 | ** Otherwise, the default value is `disabled`. 78 | 79 | Controls this plugin's compatibility with the 80 | {ecs-ref}[Elastic Common Schema (ECS)]. 81 | 82 | [id="plugins-{type}s-{plugin}-facility_labels"] 83 | ===== `facility_labels` 84 | 85 | * Value type is <> 86 | * Default value is `["kernel", "user-level", "mail", "system", "security/authorization", "syslogd", "line printer", "network news", "UUCP", "clock", "security/authorization", "FTP", "NTP", "log audit", "log alert", "clock", "local0", "local1", "local2", "local3", "local4", "local5", "local6", "local7"]` 87 | 88 | Labels for facility levels defined in RFC3164. 89 | 90 | You can use this option to override the integer->label mapping for syslog inputs 91 | that behave differently than the RFCs. 92 | 93 | Provide a zero-indexed array with all of your facility labels _in order_. 94 | If a log message contains a facility number with no corresponding entry, 95 | the facility_label is not added to the event. 96 | 97 | [id="plugins-{type}s-{plugin}-grok_pattern"] 98 | ===== `grok_pattern` 99 | 100 | * Value type is <> 101 | * Default value is `"<%{POSINT:priority}>%{SYSLOGLINE}"` 102 | * Default value depends on whether <> is enabled: 103 | ** ECS Compatibility disabled: `"<%{POSINT:priority}>%{SYSLOGLINE}"` 104 | ** ECS Compatibility enabled: `"<%{POSINT:[log][syslog][priority]:int}>%{SYSLOGLINE}"` 105 | 106 | The default value should read and properly parse syslog lines which are 107 | fully compliant with http://www.ietf.org/rfc/rfc3164.txt[RFC3164]. 108 | 109 | You can override this value to parse non-standard lines with a valid grok 110 | pattern which will parse the received lines. If the line is unable to 111 | be parsed, the `_grokparsefailure_sysloginput` tag will be added. 112 | 113 | The grok pattern must provide a `timestamp` field. If the `timestamp` 114 | field is omitted, or is unable to be parsed as `RFC3164` style or 115 | `ISO8601`, a `_dateparsefailure` tag will be added. 116 | 117 | 118 | [id="plugins-{type}s-{plugin}-host"] 119 | ===== `host` 120 | 121 | * Value type is <> 122 | * Default value is `"0.0.0.0"` 123 | 124 | The address to listen on. 125 | 126 | [id="plugins-{type}s-{plugin}-locale"] 127 | ===== `locale` 128 | 129 | * Value type is <> 130 | * There is no default value for this setting. 131 | 132 | Specify a locale to be used for date parsing using either IETF-BCP47 or POSIX language tag. 133 | Simple examples are `en`,`en-US` for BCP47 or `en_US` for POSIX. 134 | If not specified, the platform default will be used. 135 | 136 | The locale is mostly necessary to be set for parsing month names (pattern with MMM) and 137 | weekday names (pattern with EEE). 138 | 139 | 140 | [id="plugins-{type}s-{plugin}-port"] 141 | ===== `port` 142 | 143 | * Value type is <> 144 | * Default value is `514` 145 | 146 | The port to listen on. Remember that ports less than 1024 (privileged 147 | ports) may require root to use. 148 | 149 | [id="plugins-{type}s-{plugin}-proxy_protocol"] 150 | ===== `proxy_protocol` 151 | 152 | * Value type is <> 153 | * Default value is `false` 154 | 155 | Proxy protocol support, only v1 is supported at this time 156 | http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt 157 | 158 | [id="plugins-{type}s-{plugin}-severity_labels"] 159 | ===== `severity_labels` 160 | 161 | * Value type is <> 162 | * Default value is `["Emergency", "Alert", "Critical", "Error", "Warning", "Notice", "Informational", "Debug"]` 163 | 164 | Labels for severity levels defined in RFC3164. 165 | 166 | Provide a zero-indexed array with all of your severity labels _in order_. 167 | If a log message contains a severity label with no corresponding entry, 168 | the severity_label is not added to the event. 169 | 170 | [id="plugins-{type}s-{plugin}-syslog_field"] 171 | ===== `syslog_field` 172 | 173 | * Value type is <> 174 | * Default value is `"message"` 175 | 176 | Codecs process the data before the rest of the data is parsed. Some codecs, 177 | like CEF, put the syslog data into another field after pre-processing the 178 | data. Use this option in conjunction with the `grok_pattern` configuration 179 | to allow the syslog input plugin to fully parse the syslog data in this case. 180 | 181 | [source,sh] 182 | ------- 183 | input { 184 | syslog { 185 | port => 12345 186 | codec => cef 187 | syslog_field => "syslog" 188 | grok_pattern => "<%{POSINT:priority}>%{SYSLOGTIMESTAMP:timestamp} CUSTOM GROK HERE" 189 | } 190 | } 191 | ------- 192 | 193 | [id="plugins-{type}s-{plugin}-timezone"] 194 | ===== `timezone` 195 | 196 | * Value type is <> 197 | * There is no default value for this setting. 198 | 199 | Specify a time zone canonical ID to be used for date parsing. 200 | The valid IDs are listed on the [Joda.org available time zones page](http://joda-time.sourceforge.net/timezones.html). 201 | This is useful in case the time zone cannot be extracted from the value, 202 | and is not the platform default. 203 | If this is not specified the platform default will be used. 204 | Canonical ID is good as it takes care of daylight saving time for you. 205 | For example, `America/Los_Angeles` or `Europe/Paris` are valid IDs. 206 | 207 | [id="plugins-{type}s-{plugin}-use_labels"] 208 | ===== `use_labels` 209 | 210 | * Value type is <> 211 | * Default value is `true` 212 | 213 | Use label parsing for severity and facility levels. 214 | 215 | 216 | 217 | [id="plugins-{type}s-{plugin}-common-options"] 218 | include::{include_path}/{type}.asciidoc[] 219 | 220 | :default_codec!: 221 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2020 Elastic and contributors 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /lib/logstash/inputs/syslog.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require "date" 3 | require "socket" 4 | require "concurrent/array" 5 | require "logstash/filters/grok" 6 | require "logstash/filters/date" 7 | require "logstash/inputs/base" 8 | require "logstash/namespace" 9 | require 'logstash/plugin_mixins/ecs_compatibility_support' 10 | require "stud/interval" 11 | 12 | # Read syslog messages as events over the network. 13 | # 14 | # This input is a good choice if you already use syslog today. 15 | # It is also a good choice if you want to receive logs from 16 | # appliances and network devices where you cannot run your own 17 | # log collector. 18 | # 19 | # Of course, 'syslog' is a very muddy term. This input only supports `RFC3164` 20 | # syslog with some small modifications. The date format is allowed to be 21 | # `RFC3164` style or `ISO8601`. Otherwise the rest of `RFC3164` must be obeyed. 22 | # If you do not use `RFC3164`, do not use this input. 23 | # 24 | # For more information see the http://www.ietf.org/rfc/rfc3164.txt[RFC3164 page]. 25 | # 26 | # Note: This input will start listeners on both TCP and UDP. 27 | # 28 | class LogStash::Inputs::Syslog < LogStash::Inputs::Base 29 | include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1) 30 | 31 | config_name "syslog" 32 | 33 | default :codec, "plain" 34 | 35 | # The address to listen on. 36 | config :host, :validate => :string, :default => "0.0.0.0" 37 | 38 | # The port to listen on. Remember that ports less than 1024 (privileged 39 | # ports) may require root to use. 40 | config :port, :validate => :number, :default => 514 41 | 42 | # Use custom post-codec processing field (e.g. syslog, after cef codec 43 | # processing) instead of the default `message` field 44 | config :syslog_field, :validate => :string, :default => "message" 45 | 46 | # Set custom grok pattern to parse the syslog, in case the format differs 47 | # from the defined standard. This is common in security and other appliances 48 | config :grok_pattern, :validate => :string 49 | 50 | # Proxy protocol support, only v1 is supported at this time 51 | # http://www.haproxy.org/download/1.5/doc/proxy-protocol.txt 52 | config :proxy_protocol, :validate => :boolean, :default => false 53 | 54 | # Use label parsing for severity and facility levels. 55 | config :use_labels, :validate => :boolean, :default => true 56 | 57 | # Labels for facility levels. These are defined in RFC3164. 58 | config :facility_labels, :validate => :array, :default => [ "kernel", "user-level", "mail", "system", "security/authorization", "syslogd", "line printer", "network news", "UUCP", "clock", "security/authorization", "FTP", "NTP", "log audit", "log alert", "clock", "local0", "local1", "local2", "local3", "local4", "local5", "local6", "local7" ] 59 | 60 | # Labels for severity levels. These are defined in RFC3164. 61 | config :severity_labels, :validate => :array, :default => [ "Emergency" , "Alert", "Critical", "Error", "Warning", "Notice", "Informational", "Debug" ] 62 | 63 | # Specify a time zone canonical ID to be used for date parsing. 64 | # The valid IDs are listed on the [Joda.org available time zones page](http://joda-time.sourceforge.net/timezones.html). 65 | # This is useful in case the time zone cannot be extracted from the value, and is not the platform default. 66 | # If this is not specified the platform default will be used. 67 | # Canonical ID is good as it takes care of daylight saving time for you 68 | # For example, `America/Los_Angeles` or `Europe/France` are valid IDs. 69 | config :timezone, :validate => :string 70 | 71 | # Specify a locale to be used for date parsing using either IETF-BCP47 or POSIX language tag. 72 | # Simple examples are `en`,`en-US` for BCP47 or `en_US` for POSIX. 73 | # If not specified, the platform default will be used. 74 | # 75 | # The locale is mostly necessary to be set for parsing month names (pattern with MMM) and 76 | # weekday names (pattern with EEE). 77 | # 78 | config :locale, :validate => :string 79 | 80 | # ECS only option to configure [service][type] value in produced events. 81 | # 82 | # NOTE: for now, purposefully un-documented as there are other [service] fields we could support, 83 | # assuming users would want that (they have specific use-case for LS as syslog server). 84 | config :service_type, :validate => :string, :default => 'system' 85 | 86 | GROK_FAILURE_TAG = "_grokparsefailure_sysloginput" 87 | 88 | def initialize(*params) 89 | super 90 | 91 | @priority_key = ecs_select[disabled:'priority', v1:'[log][syslog][priority]'] 92 | @facility_key = ecs_select[disabled:'facility', v1:'[log][syslog][facility][code]'] 93 | @severity_key = ecs_select[disabled:'severity', v1:'[log][syslog][severity][code]'] 94 | 95 | @facility_label_key = ecs_select[disabled:'facility_label', v1:'[log][syslog][facility][name]'] 96 | @severity_label_key = ecs_select[disabled:'severity_label', v1:'[log][syslog][severity][name]'] 97 | 98 | @host_key = ecs_select[disabled:'host', v1:'[host][ip]'] 99 | 100 | @grok_pattern ||= ecs_select[ 101 | disabled:"<%{POSINT:#{@priority_key}}>%{SYSLOGLINE}", 102 | v1:"<%{POSINT:#{@priority_key}:int}>%{SYSLOGLINE}" 103 | ] 104 | 105 | @grok_filter = LogStash::Filters::Grok.new( 106 | "overwrite" => @syslog_field, 107 | "match" => { @syslog_field => @grok_pattern }, 108 | "tag_on_failure" => [GROK_FAILURE_TAG], 109 | "ecs_compatibility" => ecs_compatibility # use ecs-compliant patterns 110 | ) 111 | 112 | @grok_filter_exec = ecs_select[ 113 | disabled: -> (event) { @grok_filter.filter(event) }, 114 | v1: -> (event) { 115 | event.set('[event][original]', event.get(@syslog_field)) 116 | @grok_filter.filter(event) 117 | set_service_fields(event) 118 | } 119 | ] 120 | 121 | @date_filter = LogStash::Filters::Date.new( 122 | "match" => [ "timestamp", "MMM dd HH:mm:ss", "MMM d HH:mm:ss", "MMM d HH:mm:ss", "ISO8601"], 123 | "locale" => @locale, 124 | "timezone" => @timezone, 125 | ) 126 | 127 | @date_filter_exec = ecs_select[ 128 | disabled: -> (event) { 129 | # in legacy (non-ecs) mode we used to match (SYSLOGBASE2) timestamp into two fields 130 | event.set("timestamp", event.get("timestamp8601")) if event.include?("timestamp8601") 131 | @date_filter.filter(event) 132 | }, 133 | v1: -> (event) { 134 | @date_filter.filter(event) 135 | event.remove('timestamp') 136 | } 137 | ] 138 | end 139 | 140 | def register 141 | @metric_errors = metric.namespace(:errors) 142 | 143 | @grok_filter.register 144 | @date_filter.register 145 | 146 | @tcp_sockets = Concurrent::Array.new 147 | @tcp = @udp = nil 148 | end # def register 149 | 150 | private 151 | 152 | def run(output_queue) 153 | udp_thr = Thread.new(output_queue) do |output_queue| 154 | server(:udp, output_queue) 155 | end 156 | 157 | tcp_thr = Thread.new(output_queue) do |output_queue| 158 | server(:tcp, output_queue) 159 | end 160 | 161 | # If we exit and we're the only input, the agent will think no inputs 162 | # are running and initiate a shutdown. 163 | udp_thr.join 164 | tcp_thr.join 165 | end # def run 166 | public :run 167 | 168 | # server call the specified protocol listener and basically restarts on 169 | # any listener uncatched exception 170 | # 171 | # @param protocol [Symbol] either :udp or :tcp 172 | # @param output_queue [Queue] the pipeline input to filters queue 173 | def server(protocol, output_queue) 174 | self.send("#{protocol}_listener", output_queue) 175 | rescue => e 176 | if !stop? 177 | @logger.warn("syslog listener died", :protocol => protocol, :address => "#{@host}:#{@port}", :exception => e, :backtrace => e.backtrace) 178 | @metric_errors.increment(:listener) 179 | Stud.stoppable_sleep(5) { stop? } 180 | retry 181 | end 182 | end 183 | 184 | # udp_listener creates the udp socket and continously read from it. 185 | # upon exception the socket will be closed and the exception bubbled 186 | # in the server which will restart the listener 187 | def udp_listener(output_queue) 188 | @logger.info("Starting syslog udp listener", :address => "#{@host}:#{@port}") 189 | 190 | @udp.close if @udp 191 | @udp = UDPSocket.new (IPAddr.new(@host).ipv6? rescue nil) ? Socket::AF_INET6 : Socket::AF_INET 192 | @udp.do_not_reverse_lookup = true 193 | @udp.bind(@host, @port) 194 | 195 | while !stop? 196 | payload, client = @udp.recvfrom(65507) 197 | metric.increment(:messages_received) 198 | decode(client[3], output_queue, payload) 199 | end 200 | ensure 201 | close_udp 202 | end # def udp_listener 203 | 204 | # tcp_listener accepts tcp connections and creates a new tcp_receiver thread 205 | # for each accepted socket. 206 | # upon exception all tcp sockets will be closed and the exception bubbled 207 | # in the server which will restart the listener. 208 | def tcp_listener(output_queue) 209 | @logger.info("Starting syslog tcp listener", :address => "#{@host}:#{@port}") 210 | @tcp = TCPServer.new(@host, @port) 211 | @tcp.do_not_reverse_lookup = true 212 | 213 | while !stop? 214 | socket = @tcp.accept 215 | @tcp_sockets << socket 216 | metric.increment(:connections) 217 | 218 | Thread.new(output_queue, socket) do |output_queue, socket| 219 | tcp_receiver(output_queue, socket) 220 | end 221 | end 222 | ensure 223 | close_tcp 224 | end # def tcp_listener 225 | 226 | def tcp_read_lines(socket) 227 | buffer = String.new 228 | loop do 229 | begin 230 | buffer << socket.read_nonblock(1024) 231 | while (newline = buffer.index("\n")) 232 | yield buffer.slice!(0..newline) 233 | end 234 | rescue IO::WaitReadable 235 | IO.select([socket], nil) 236 | retry 237 | end 238 | end 239 | end 240 | 241 | # tcp_receiver is executed in a thread, any uncatched exception will be bubbled up to the 242 | # tcp server thread and all tcp connections will be closed and the listener restarted. 243 | def tcp_receiver(output_queue, socket) 244 | peer_addr = socket.peeraddr 245 | ip, port = peer_addr[3], peer_addr[1] 246 | 247 | @logger.info("new connection", :client => "#{ip}:#{port}") 248 | LogStash::Util::set_thread_name("input|syslog|tcp|#{ip}:#{port}}") 249 | 250 | first_read = true 251 | 252 | tcp_read_lines(socket) do |line| 253 | metric.increment(:messages_received) 254 | if @proxy_protocol && first_read 255 | first_read = false 256 | pp_info = line.split(/\s/) 257 | # PROXY proto clientip proxyip clientport proxyport 258 | if pp_info[0] != "PROXY" 259 | @logger.error("invalid proxy protocol header label", header: line) 260 | raise IOError 261 | else 262 | @logger.debug("proxy protocol detected", header: line) 263 | ip = pp_info[2] 264 | port = pp_info[3] 265 | next 266 | end 267 | end 268 | decode(ip, output_queue, line) 269 | end 270 | rescue Errno::ECONNRESET 271 | # swallow connection reset exceptions to avoid bubling up the tcp_listener & server 272 | logger.info("connection reset", :client => "#{ip}:#{port}") 273 | rescue Errno::EBADF, EOFError 274 | # swallow connection closed exceptions to avoid bubling up the tcp_listener & server 275 | logger.info("connection closed", :client => "#{ip}:#{port}") 276 | rescue IOError => e 277 | # swallow connection closed exceptions to avoid bubling up the tcp_listener & server 278 | raise(e) unless socket.closed? && e.message.to_s.include?("closed") 279 | logger.info("connection error:", :exception => e.class, :message => e.message) 280 | ensure 281 | @tcp_sockets.delete(socket) 282 | socket.close rescue log_and_squash(:close_tcp_receiver_socket) 283 | end 284 | 285 | def decode(ip, output_queue, data) 286 | @codec.decode(data) do |event| 287 | decorate(event) 288 | event.set(@host_key, ip) 289 | syslog_relay(event) 290 | output_queue << event 291 | metric.increment(:events) 292 | end 293 | rescue => e 294 | # swallow and log all decoding exceptions, these will never be socket related 295 | @logger.error("Error decoding data", :data => data.inspect, :exception => e.class, :message => e.message, :backtrace => e.backtrace) 296 | @metric_errors.increment(:decoding) 297 | end 298 | 299 | # @see LogStash::Plugin#close 300 | def stop 301 | close_udp 302 | close_tcp 303 | end 304 | public :stop 305 | 306 | def close_udp 307 | if @udp 308 | @udp.close_read rescue log_and_squash(:close_udp_read) 309 | @udp.close_write rescue log_and_squash(:close_udp_write) 310 | end 311 | @udp = nil 312 | end 313 | 314 | # Helper for inline rescues, which logs the exception at "DEBUG" level and returns nil. 315 | # 316 | # Instead of: 317 | # ~~~ ruby 318 | #. foo rescue nil 319 | # ~~~ 320 | # Do: 321 | # ~~~ ruby 322 | #. foo rescue log_and_squash(:foo) 323 | # ~~~ 324 | def log_and_squash(label) 325 | $! && logger.debug("#{label} failed:", :exception => $!.class, :message => $!.message) 326 | nil 327 | end 328 | 329 | def close_tcp 330 | # If we somehow have this left open, close it. 331 | @tcp_sockets.each do |socket| 332 | socket.close rescue log_and_squash(:close_tcp_socket) 333 | end 334 | @tcp.close if @tcp rescue log_and_squash(:close_tcp) 335 | @tcp = nil 336 | end 337 | 338 | # Following RFC3164 where sane, we'll try to parse a received message 339 | # as if you were relaying a syslog message to it. 340 | # If the message cannot be recognized (see @grok_filter), we'll 341 | # treat it like the whole event["message"] is correct and try to fill 342 | # the missing pieces (host, priority, etc) 343 | def syslog_relay(event) 344 | @grok_filter_exec.(event) 345 | 346 | if event.get("tags").nil? || !event.get("tags").include?(GROK_FAILURE_TAG) 347 | # Per RFC3164, priority = (facility * 8) + severity 348 | # = (facility << 3) & (severity) 349 | priority = if event.include?(@priority_key) 350 | event.get(@priority_key).to_i rescue 13 351 | else 352 | 13 353 | end 354 | set_priority event, priority 355 | 356 | @date_filter_exec.(event) 357 | 358 | else 359 | @logger.debug? && @logger.debug("un-matched syslog message", :message => event.get("message")) 360 | 361 | # RFC3164 says unknown messages get pri=13 362 | set_priority event, 13 363 | metric.increment(:unknown_messages) 364 | end 365 | 366 | # Apply severity and facility metadata if use_labels => true 367 | set_labels(event) if @use_labels 368 | end # def syslog_relay 369 | public :syslog_relay 370 | 371 | def set_priority(event, priority) 372 | severity = priority & 7 # 7 is 111 (3 bits) 373 | facility = priority >> 3 374 | event.set(@priority_key, priority) 375 | event.set(@severity_key, severity) 376 | event.set(@facility_key, facility) 377 | end 378 | 379 | def set_labels(event) 380 | facility_number = event.get(@facility_key) 381 | severity_number = event.get(@severity_key) 382 | 383 | facility_label = @facility_labels[facility_number] 384 | event.set(@facility_label_key, facility_label) if facility_label 385 | 386 | severity_label = @severity_labels[severity_number] 387 | event.set(@severity_label_key, severity_label) if severity_label 388 | end 389 | 390 | def set_service_fields(event) 391 | service_type = @service_type 392 | if service_type && !service_type.empty? 393 | event.set('[service][type]', service_type) unless event.include?('[service][type]') 394 | end 395 | end 396 | 397 | end # class LogStash::Inputs::Syslog 398 | -------------------------------------------------------------------------------- /spec/inputs/syslog_spec.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require "logstash/devutils/rspec/spec_helper" 3 | require "logstash/devutils/rspec/shared_examples" 4 | 5 | require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper' 6 | 7 | # running the grok code outside a logstash package means 8 | # LOGSTASH_HOME will not be defined, so let's set it here 9 | # before requiring the grok filter 10 | unless LogStash::Environment.const_defined?(:LOGSTASH_HOME) 11 | LogStash::Environment::LOGSTASH_HOME = File.expand_path("../../", __FILE__) 12 | end 13 | 14 | # temporary fix to have the spec pass for an urgen mass-publish requirement. 15 | # cut & pasted from the same tmp fix in the grok spec 16 | # see https://github.com/logstash-plugins/logstash-filter-grok/issues/72 17 | # this needs to be refactored and properly fixed 18 | module LogStash::Environment 19 | # also :pattern_path method must exist so we define it too 20 | unless self.method_defined?(:pattern_path) 21 | def pattern_path(path) 22 | ::File.join(LOGSTASH_HOME, "patterns", path) 23 | end 24 | end 25 | end 26 | 27 | require "logstash/inputs/syslog" 28 | require "logstash/codecs/cef" 29 | require "logstash/event" 30 | require "stud/try" 31 | require "socket" 32 | 33 | describe LogStash::Inputs::Syslog do 34 | SYSLOG_LINE = "<164>Oct 26 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434 dst outside:192.168.0.1/53 by access-group \"acl_drac\" [0x0, 0x0]" 35 | 36 | context 'ECS common behavior', :ecs_compatibility_support do 37 | ecs_compatibility_matrix(:disabled, :v1, :v8 => :v1) do |ecs_select| 38 | let(:priority_key) { ecs_select[disabled:'priority', v1:'[log][syslog][priority]'] } 39 | let(:facility_key) { ecs_select[disabled:'facility', v1:'[log][syslog][facility][code]'] } 40 | let(:severity_key) { ecs_select[disabled:'severity', v1:'[log][syslog][severity][code]'] } 41 | let(:host_key) { ecs_select[disabled:'host', v1:'[host][ip]'] } 42 | 43 | before(:each) do 44 | allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility) 45 | end 46 | 47 | it "should properly handle priority, severity and facilities" do 48 | skip_if_stack_known_issue 49 | port = 5511 50 | event_count = 10 51 | conf = <<-CONFIG 52 | input { 53 | syslog { 54 | type => "blah" 55 | port => #{port} 56 | } 57 | } 58 | CONFIG 59 | 60 | events = input(conf) do |pipeline, queue| 61 | socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) } 62 | event_count.times do |i| 63 | socket.puts(SYSLOG_LINE) 64 | end 65 | socket.close 66 | 67 | event_count.times.collect { queue.pop } 68 | end 69 | 70 | expect( events.length ).to eql event_count 71 | events.each do |event| 72 | expect( event.get(priority_key) ).to eql 164 73 | expect( event.get(severity_key) ).to eql 4 74 | expect( event.get(facility_key) ).to eql 20 75 | end 76 | end 77 | 78 | it "should properly PROXY protocol v1" do 79 | skip_if_stack_known_issue 80 | port = 5511 81 | event_count = 10 82 | conf = <<-CONFIG 83 | input { 84 | syslog { 85 | type => "blah" 86 | port => #{port} 87 | proxy_protocol => true 88 | } 89 | } 90 | CONFIG 91 | 92 | events = input(conf) do |pipeline, queue| 93 | socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) } 94 | socket.puts("PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678\r\n") 95 | socket.flush 96 | event_count.times do |i| 97 | socket.puts(SYSLOG_LINE) 98 | end 99 | socket.close 100 | 101 | event_count.times.collect { queue.pop } 102 | end 103 | 104 | expect( events.length ).to eql event_count 105 | events.each do |event| 106 | expect( event.get(priority_key) ).to eql 164 107 | expect( event.get(severity_key) ).to eql 4 108 | expect( event.get(facility_key) ).to eql 20 109 | expect( event.get(host_key) ).to eql "1.2.3.4" 110 | end 111 | end 112 | 113 | context 'grok' do 114 | it "should add unique tag when grok parsing fails with live syslog input" do 115 | skip_if_stack_known_issue 116 | port = 5511 117 | event_count = 10 118 | conf = <<-CONFIG 119 | input { 120 | syslog { 121 | type => "blah" 122 | port => #{port} 123 | } 124 | } 125 | CONFIG 126 | events = input(conf) do |pipeline, queue| 127 | socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) } 128 | event_count.times do |i| 129 | socket.puts("message which causes the a grok parse failure") 130 | end 131 | socket.close 132 | event_count.times.collect { queue.pop } 133 | end 134 | expect( events.length ).to eql event_count 135 | event_count.times do |i| 136 | expect( events[i].get("tags") ).to eql ["_grokparsefailure_sysloginput"] 137 | end 138 | end 139 | 140 | it "should add unique tag when grok parsing fails" do 141 | input = LogStash::Inputs::Syslog.new({}) 142 | input.register 143 | 144 | # event which is not syslog should have a new tag 145 | event = LogStash::Event.new({ "message" => "hello world, this is not syslog RFC3164" }) 146 | input.syslog_relay(event) 147 | expect( event.get("tags") ).to eql ["_grokparsefailure_sysloginput"] 148 | expect( event.get(priority_key) ).to eql 13 149 | 150 | syslog_event = LogStash::Event.new({ "message" => "<164>Oct 26 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434" }) 151 | input.syslog_relay(syslog_event) 152 | expect( syslog_event.get(priority_key) ).to eql 164 153 | expect( syslog_event.get(severity_key) ).to eql 4 154 | expect( syslog_event.get("tags") ).to be nil 155 | 156 | input.close 157 | end 158 | 159 | it "should properly handle a custom grok_pattern" do 160 | port = 5511 161 | event_count = 1 162 | custom_grok = "<%{POSINT:#{priority_key}}>%{SYSLOGTIMESTAMP:timestamp} atypical %{GREEDYDATA:message}" 163 | message_field = "This part constitutes the message field" 164 | timestamp = "Oct 26 15:19:25" 165 | custom_line = "<164>#{timestamp} atypical #{message_field}" 166 | 167 | conf = <<-CONFIG 168 | input { 169 | syslog { 170 | type => "blah" 171 | port => #{port} 172 | grok_pattern => "#{custom_grok}" 173 | } 174 | } 175 | CONFIG 176 | 177 | events = input(conf) do |pipeline, queue| 178 | socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) } 179 | event_count.times do |i| 180 | socket.puts(custom_line) 181 | end 182 | socket.close 183 | 184 | event_count.times.collect { queue.pop } 185 | end 186 | 187 | expect( events.length ).to eql event_count 188 | events.each do |event| 189 | expect( event.get(priority_key) ).to eql 164 190 | expect( event.get(severity_key) ).to eql 4 191 | expect( event.get(facility_key) ).to eql 20 192 | expect( event.get("message") ).to eql "#{message_field}\n" 193 | expect( event.get('timestamp') ).to eql timestamp if ecs_compatibility == :disabled 194 | expect( event.include?('timestamp') ).to be false if ecs_compatibility != :disabled 195 | end 196 | end 197 | 198 | it "should properly handle the cef codec with a custom grok_pattern" do 199 | port = 5511 200 | event_count = 1 201 | 202 | custom_grok = "<%{POSINT:#{priority_key}}>%{TIMESTAMP_ISO8601:timestamp} atypical %{GREEDYDATA:syslog_message}" 203 | timestamp = "2018-02-07T12:40:00.000Z" 204 | cef_message = "Description Omitted" 205 | syslog_message = "foo bar" 206 | syslog_message_envelope = "<134>#{timestamp} atypical #{syslog_message}" 207 | custom_line = "CEF:0|Company Name|Application Name|Application Version Number|632|Syslog Configuration Updated|3|src=192.168.0.1 suser=user@example.com target=TARGET msg=#{cef_message} syslog=#{syslog_message_envelope} KeyValueOne=kv1 KeyValueTwo=12345 " 208 | 209 | conf = <<-CONFIG 210 | input { 211 | syslog { 212 | port => #{port} 213 | syslog_field => "syslog" 214 | grok_pattern => "#{custom_grok}" 215 | codec => cef { ecs_compatibility => #{ ecs_compatibility } } 216 | } 217 | } 218 | CONFIG 219 | 220 | events = input(conf) do |pipeline, queue| 221 | socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) } 222 | event_count.times do |i| 223 | socket.puts(custom_line) 224 | end 225 | socket.close 226 | 227 | event_count.times.collect { queue.pop } 228 | end 229 | 230 | expect( events.length ).to eql event_count 231 | events.each do |event| 232 | expect( event.get(priority_key) ).to eql 134 233 | expect( event.get(severity_key) ).to eql 6 234 | expect( event.get(facility_key) ).to eql 16 235 | expect( event.get("message") ).to eql cef_message 236 | expect( event.get("syslog_message") ).to eql syslog_message 237 | expect( event.get('timestamp') ).to eql timestamp if ecs_compatibility == :disabled 238 | expect( event.include?('timestamp') ).to be false if ecs_compatibility != :disabled 239 | end 240 | end 241 | end 242 | 243 | context 'timestamp' do 244 | it "should properly handle locale and timezone" do 245 | port = 5511 246 | event_count = 10 247 | 248 | conf = <<-CONFIG 249 | input { 250 | syslog { 251 | type => "blah" 252 | port => #{port} 253 | locale => "en" 254 | timezone => "UTC" 255 | } 256 | } 257 | CONFIG 258 | 259 | events = input(conf) do |pipeline, queue| 260 | socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) } 261 | event_count.times do |i| 262 | socket.puts(SYSLOG_LINE) 263 | end 264 | socket.close 265 | 266 | event_count.times.collect { queue.pop } 267 | end 268 | 269 | expect( events.length ).to eql event_count 270 | events.each do |event| 271 | expect( event.get("@timestamp") ).to be_a_logstash_timestamp_equivalent_to("#{Time.now.year}-10-26T15:19:25Z") 272 | end 273 | end 274 | 275 | it "should properly handle no locale and no timezone" do 276 | port = 5511 277 | 278 | conf = <<-CONFIG 279 | input { 280 | syslog { 281 | type => "blah" 282 | port => #{port} 283 | } 284 | } 285 | CONFIG 286 | 287 | event = input(conf) do |pipeline, queue| 288 | socket = Stud.try(5.times) { TCPSocket.new("127.0.0.1", port) } 289 | socket.puts(SYSLOG_LINE) 290 | socket.close 291 | 292 | queue.pop 293 | end 294 | 295 | # chances platform timezone is not UTC, so parse without offset to create expectation 296 | equivalent_time = Time.parse("#{Time.now.year}-10-26T15:19:25") 297 | expect( event.get("@timestamp") ).to be_a_logstash_timestamp_equivalent_to(equivalent_time) 298 | end 299 | 300 | it "should support non UTC timezone" do 301 | input = LogStash::Inputs::Syslog.new({"timezone" => "-05:00"}) 302 | input.register 303 | 304 | # event which is not syslog should have a new tag 305 | 306 | syslog_event = LogStash::Event.new({ "message" => "<164>Oct 26 15:19:25 1.2.3.4 %ASA-4-106023: Deny udp src DRAC:10.1.2.3/43434" }) 307 | input.syslog_relay(syslog_event) 308 | 309 | expect( syslog_event.get("@timestamp") ).to be_a_logstash_timestamp_equivalent_to("#{Time.now.year}-10-26T20:19:25Z") 310 | 311 | input.close 312 | end 313 | end 314 | end 315 | end 316 | 317 | context 'ECS :v1 behavior', :ecs_compatibility_support do 318 | 319 | ecs_compatibility_matrix(:v1) do 320 | 321 | before(:each) do 322 | allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility) 323 | end 324 | 325 | let(:event) do 326 | LogStash::Event.new("message" => "<164>Oct 26 15:19:25 1.2.3.4 a sample message") 327 | end 328 | 329 | subject { LogStash::Inputs::Syslog.new } 330 | 331 | before { subject.register } 332 | after { subject.close } 333 | 334 | it "should not have a timestamp field" do 335 | subject.syslog_relay(event) 336 | 337 | expect( event.to_hash.keys ).to_not include 'timestamp' 338 | end 339 | 340 | it "overwrites message" do 341 | subject.syslog_relay(event) 342 | 343 | expect( event.get('message') ).to eql 'a sample message' 344 | end 345 | 346 | it "keep original log message" do 347 | subject.syslog_relay(event) 348 | 349 | expect( event.get('[event][original]') ).to eql '<164>Oct 26 15:19:25 1.2.3.4 a sample message' 350 | end 351 | 352 | it "sets syslog priority and severity" do 353 | subject.syslog_relay(event) 354 | 355 | expect( event.get('log') ).to include 'syslog' => hash_including('priority' => 164) 356 | expect( event.get('log') ).to include 'syslog' => hash_including('severity' => { 'code' => 4, 'name' => 'Warning' }) 357 | end 358 | 359 | it "sets service type" do 360 | subject.syslog_relay(event) 361 | 362 | expect( event.get('service') ).to include 'type' => 'system' 363 | end 364 | 365 | let(:queue) { Queue.new } 366 | 367 | let(:socket) do 368 | server = double('tcp-server') 369 | allow( subject ).to receive(:tcp_read_lines).and_yield("<133>Mar 11 08:44:43 precision kernel: [765135.424096] mce: CPU6: Package temperature/speed normal\n") 370 | allow( server ).to receive(:close) 371 | server 372 | end 373 | 374 | it "sets host IP" do 375 | expect( socket ).to receive(:peeraddr).and_return(["AF_INET", 514, "192.168.0.10", "192.168.0.10"]) 376 | subject.send :tcp_receiver, queue, socket 377 | 378 | expect( queue.size ).to eql 1 379 | event = queue.pop 380 | expect( event.get('host') ).to eql 'hostname' => 'precision', 'ip' => '192.168.0.10' 381 | end 382 | end 383 | end 384 | 385 | context 'tcp receiver' do 386 | subject(:plugin) { LogStash::Inputs::Syslog.new } 387 | before { plugin.register } 388 | after { plugin.close } 389 | 390 | let(:queue) { Queue.new } 391 | let(:socket) do 392 | socket = double('tcp-socket') 393 | expect( socket ).to receive(:peeraddr).and_return(["AF_INET", 514, "192.168.0.10", "192.168.0.10"]) 394 | socket 395 | end 396 | 397 | it 'should close connection when client sends EOF' do 398 | expect( socket ).to receive(:read_nonblock).and_raise(EOFError) 399 | expect( socket ).to receive(:close) 400 | allow( plugin.logger ).to receive(:info) 401 | 402 | plugin.send :tcp_receiver, queue, socket 403 | 404 | expect( plugin.logger ).to have_received(:info).with(/connection closed/, anything) 405 | expect( queue.size ).to eql 0 406 | end 407 | 408 | it 'should properly read partially received messages' do 409 | expect( socket ).to receive(:close) 410 | allow( plugin.codec ).to receive(:decode).and_call_original 411 | 412 | messages = ["<133>Mar 11 08:44:43 localhost message 2\n", "message 1\n", "<133>Mar 11 08:44:43 localhost ", ] 413 | allow( socket ).to receive(:read_nonblock).at_least(messages.size).times do 414 | msg = messages.pop 415 | raise EOFError unless msg 416 | msg 417 | end 418 | 419 | plugin.send :tcp_receiver, queue, socket 420 | 421 | expect( queue.size ).to eql 2 422 | expect( plugin.codec ).to have_received(:decode).with("<133>Mar 11 08:44:43 localhost message 1\n") 423 | expect( plugin.codec ).to have_received(:decode).with("<133>Mar 11 08:44:43 localhost message 2\n") 424 | end 425 | end 426 | 427 | 428 | it_behaves_like 'an interruptible input plugin' do 429 | let(:config) { { "port" => 5511 } } 430 | end 431 | 432 | private 433 | 434 | def skip_if_stack_known_issue 435 | skip 'elastic/logstash#11196 known LS 7.5 issue' if ENV['ELASTIC_STACK_VERSION'] && JRUBY_VERSION.eql?('9.2.8.0') 436 | end 437 | end 438 | --------------------------------------------------------------------------------