├── .gitignore ├── Dockerfile ├── LICENSE ├── README.md ├── example ├── config │ ├── conf.d │ │ ├── 00_syslog.filter.conf │ │ └── 01_applications.filter.conf │ └── patterns │ │ ├── dovecot.pattern │ │ └── syslog.pattern └── test │ ├── filters │ ├── misc │ │ └── empty.json │ └── syslog │ │ └── syslog_base.json │ └── patterns │ ├── dovecot │ └── dovecot_login.json │ └── misc.json ├── logstash-tester.sh ├── run_example.sh └── test ├── run-tests.sh └── spec ├── filter_spec.rb ├── patterns_spec.rb └── spec_helper.rb /.gitignore: -------------------------------------------------------------------------------- 1 | # Sample log files go in this directory 2 | samples/* 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM logstash:2.3.1 2 | 3 | RUN logstash-plugin install --development 4 | 5 | ARG LST 6 | ARG FILTER_CONFIG 7 | ARG PATTERN_CONFIG 8 | ARG FILTER_TESTS 9 | ARG PATTERN_TESTS 10 | 11 | ADD $PATTERN_CONFIG /etc/logstash/patterns 12 | ADD $LST/test /test 13 | ADD $FILTER_CONFIG /test/spec/filter_config 14 | ADD $FILTER_TESTS /test/spec/filter_data 15 | ADD $PATTERN_TESTS /test/spec/pattern_data 16 | 17 | ENTRYPOINT ["/test/run-tests.sh"] 18 | # ENTRYPOINT ["bash"] 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Rodolfo Ripado 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Logstash Tester [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)](/LICENSE) 2 | 3 | ## TL;DR 4 | 5 | Logstash Tester is a tool to write and run unit tests against your Logstash config files and/or your custom patterns. 6 | It uses RSpec and Logstash running in a Docker container 7 | 8 | Test it, it works: ```./run_example.sh``` (You must have a running Docker environment). 9 | 10 | Type ```./logstash-tester.sh -h``` to see the available options. 11 | 12 | ## Long version 13 | 14 | When your logstash config starts getting really long, and you start loosing control of 15 | all the cases covered by your custom Grok patterns, you know you're entering **Logstash 16 | Config Hell**. 17 | 18 | *Logstash Tester* helps you (hopefully) to grow your logstash config files, AND keep your 19 | sanity, by unit testing everything. 20 | 21 | #### HOWTO use it 22 | 23 | You should have a working Docker environment for *Logstash Tester* to work. 24 | 25 | **1. The Data Directory** 26 | 27 | You should have a directory (the data dir) structured in the following way: 28 | - */config/conf.d* 29 | Contains the logstash filter configurations 30 | - */config/patterns* 31 | Contains your custom patterns 32 | - */test/filters* 33 | Your filtering test case files 34 | - */test/patterns* 35 | Custom patterns test case files 36 | 37 | **2. Test case files** 38 | 39 | Test cases are written in JSON. When logstash-tester runs, it looks for test cases in every file matching ```/test/patterns/**/*.json``` and ```/test/filters/**/*.json```. 40 | 41 | Test case syntax is pretty straigtforward. 42 | 43 | Here's a filter testcase taken from the examples : 44 | 45 | ```json 46 | { 47 | "name": "CUSTOM Syslog message", 48 | "fields": { 49 | "type": "syslog" 50 | }, 51 | "ignore": ["@version", "@timestamp"], 52 | "cases": [{ 53 | "in": "<22>Feb 2 09:47:12 mail dovecot: pop3-login: Login: user=, method=PLAIN, rip=192.148.8.32, lip=192.148.34.126", 54 | "out": { 55 | "type": "syslog", 56 | "message": "pop3-login: Login: user=, method=PLAIN, rip=192.148.8.32, lip=192.148.34.126", 57 | "syslog_pri": "22", 58 | "syslog_timestamp": "Feb 2 09:47:12", 59 | "application": "dovecot", 60 | "application_host": "mail", 61 | "protocol": "pop3", 62 | "action": "login", 63 | "method": "plain", 64 | "src": "192.148.8.32", 65 | "dest": "192.148.34.126", 66 | "user": "somename@somedomain.fr" 67 | } 68 | }] 69 | } 70 | ``` 71 | 72 | - *fields*: the values are are expected to be appended to the message object by the input plugin. 73 | - *ignore*: a list of keys that will be ignored in the logstash output 74 | - *cases*: the list of test cases each with an "in" field containing the message to pass through the filtering pipeline and a "out" object field with the exact expected output. 75 | 76 | Pattern test cases are described pretty much in the same way : 77 | 78 | ```json 79 | { 80 | "name": "Dovecot Login variations", 81 | "pattern": "DOVECOT_LOGIN", 82 | "cases": [ 83 | { 84 | "in": "pop3-login: Login: user=, method=PLAIN, rip=192.148.8.32, lip=192.148.34.126", 85 | "out": { 86 | "protocol": "pop3", 87 | "action": "Login", 88 | "user": "somename@somedomain.fr", 89 | "method": "PLAIN", 90 | "remote_ip": "192.148.8.32", 91 | "local_ip": "192.148.34.126" 92 | } 93 | } 94 | ] 95 | } 96 | ``` 97 | 98 | where : 99 | - *pattern*: is the custom grok pattern to be tested 100 | - *cases*: is the list of test each with an "in" field containing the string to grok parse and a "out" object field with the exact expected output from the grok filter. 101 | 102 | Since an example is (sometimes) worth a thousand words, check out the 'example' directory to get the idea of how things are organized. 103 | 104 | **Other useful info** 105 | - The Logstash filter config files should follow a naming convention 106 | (```[some-custom-label].filter.conf```). 107 | They'll be loaded in alphabetical order. 108 | 109 | - Logstash-tester assumes the patternsdir setting in grok filters is set to ```/etc/logstash/patterns```. 110 | 111 | **Examples** 112 | 113 | Using the "example" data directory: 114 | 115 | - ```./logstash-tester.sh -d example``` 116 | 117 | runs every pattern and filter test case in every json test file. 118 | 119 | - ```./logstash-tester.sh -d example filters``` 120 | 121 | runs filter test cases ignoring pattern tests 122 | 123 | - ```./logstash-tester.sh -d example -p syslog filters``` 124 | 125 | runs only filter test files in 'example/test/filter/syslog' 126 | 127 | Type ```./logstash-tester.sh -h``` to know more about the command line options. 128 | 129 | #### Some other stuff 130 | 131 | I'm not a rubyist, the ruby code in the test suites was hacked together from bits 132 | and pieces found here and there. If you're a rubyist, please clean up my mess 133 | and do me a pull request :-). If you're not a rubyistm neither, don't worry, it works. 134 | 135 | #### Credits 136 | 137 | *Logstash Tester* drew its inspiration from two interesting projects: 138 | - [RSpec logstash filter](https://github.com/tcnksm/rspec-logstash-filter) 139 | - [Logstash filter verifier](https://github.com/magnusbaeck/logstash-filter-verifier) 140 | 141 | 142 | -------------------------------------------------------------------------------- /example/config/conf.d/00_syslog.filter.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | 3 | # Drop empty lines 4 | if [message] == "" { 5 | drop {} 6 | } 7 | 8 | # Handle Syslog Wrapping 9 | # 'message' field will contain only the program message part 10 | 11 | if [type] == "syslog" { 12 | grok { 13 | patterns_dir => [ "/etc/logstash/patterns" ] 14 | match => { "message" => [ "%{SYSLOG_CUSTOM}" ] } 15 | } 16 | 17 | mutate { 18 | replace => { "message" => "%{syslog_message}" } 19 | remove_field => [ "syslog_message" ] 20 | rename => { "syslog_program" => "application" } 21 | rename => { "syslog_hostname" => "application_host" } 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /example/config/conf.d/01_applications.filter.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | 3 | # Application specific filtering 4 | 5 | # DOVECOT 6 | if [application] == "dovecot" { 7 | grok { 8 | patterns_dir => ["/etc/logstash/patterns"] 9 | match => { "message" => ["%{DOVECOT_LOGIN}"] } 10 | } 11 | 12 | mutate { 13 | rename => { "remote_ip" => "src" } 14 | rename => { "local_ip" => "dest" } 15 | rename => { "user_ip" => "src" } 16 | lowercase => [ "action", "method", "protocol" ] 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /example/config/patterns/dovecot.pattern: -------------------------------------------------------------------------------- 1 | ## Dovecot Patterns 2 | 3 | DOVECOT_PROTOCOL pop3|imap|POP3|IMAP 4 | DOVECOT_ACTION Login|Disconnected 5 | DOVECOT_METHOD [a-zA-Z]+ 6 | DOVECOT_USER [^>]* 7 | 8 | DOVECOT_LOGIN %{DOVECOT_PROTOCOL:protocol}-login: %{DOVECOT_ACTION:action}( \(%{DATA:action_info}\))?: user=<%{DATA:user}>, method=%{DOVECOT_METHOD:method}, rip=%{IP:remote_ip}, lip=%{IP:local_ip} 9 | 10 | -------------------------------------------------------------------------------- /example/config/patterns/syslog.pattern: -------------------------------------------------------------------------------- 1 | # Syslog 2 | SYSLOG_CUSTOM <%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT}\])?: %{GREEDYDATA:syslog_message} 3 | -------------------------------------------------------------------------------- /example/test/filters/misc/empty.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Misc cases", 3 | "fields": { 4 | "type": "syslog" 5 | }, 6 | "ignore": ["@version", "@timestamp"], 7 | "cases": [{ 8 | "#": "empty line send to logstash", 9 | "in": "", 10 | "out": {} 11 | }] 12 | } 13 | 14 | -------------------------------------------------------------------------------- /example/test/filters/syslog/syslog_base.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CUSTOM Syslog message", 3 | "fields": { 4 | "type": "syslog" 5 | }, 6 | "ignore": ["@version", "@timestamp"], 7 | "cases": [{ 8 | "in": "<22>Feb 2 09:47:12 mail dovecot: pop3-login: Login: user=, method=PLAIN, rip=192.148.8.32, lip=192.148.34.126", 9 | "out": { 10 | "type": "syslog", 11 | "message": "pop3-login: Login: user=, method=PLAIN, rip=192.148.8.32, lip=192.148.34.126", 12 | "syslog_pri": "22", 13 | "syslog_timestamp": "Feb 2 09:47:12", 14 | "application": "dovecot", 15 | "application_host": "mail", 16 | "protocol": "pop3", 17 | "action": "login", 18 | "method": "plain", 19 | "src": "192.148.8.32", 20 | "dest": "192.148.34.126", 21 | "user": "somename@somedomain.fr" 22 | } 23 | }] 24 | } 25 | 26 | -------------------------------------------------------------------------------- /example/test/patterns/dovecot/dovecot_login.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Dovecot Login variations", 3 | "pattern": "DOVECOT_LOGIN", 4 | "cases": [ 5 | { 6 | "in": "pop3-login: Login: user=, method=PLAIN, rip=192.148.8.32, lip=192.148.34.126", 7 | "out": { 8 | "protocol": "pop3", 9 | "action": "Login", 10 | "user": "somename@somedomain.fr", 11 | "method": "PLAIN", 12 | "remote_ip": "192.148.8.32", 13 | "local_ip": "192.148.34.126" 14 | } 15 | },{ 16 | "in": "imap-login: Login: user=, method=PLAIN, rip=192.148.8.32, lip=192.148.34.126, TLS", 17 | "out": { 18 | "protocol": "imap", 19 | "action": "Login", 20 | "user": "somename@somedomain.fr", 21 | "method": "PLAIN", 22 | "remote_ip": "192.148.8.32", 23 | "local_ip": "192.148.34.126" 24 | } 25 | },{ 26 | "in": "pop3-login: Disconnected (auth failed, 1 attempts): user=, method=PLAIN, rip=192.148.8.32, lip=192.148.34.126", 27 | "out": { 28 | "protocol": "pop3", 29 | "action": "Disconnected", 30 | "action_info": "auth failed, 1 attempts", 31 | "user": "somename@somedomain.fr", 32 | "method": "PLAIN", 33 | "remote_ip": "192.148.8.32", 34 | "local_ip": "192.148.34.126" 35 | } 36 | } 37 | ] 38 | } 39 | -------------------------------------------------------------------------------- /example/test/patterns/misc.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Misc examples", 3 | "pattern": "SYSLOG_CUSTOM", 4 | "cases": [ 5 | { 6 | "in": "<53>Feb 2 09:47:12 www apache2: some apache message", 7 | "out": { 8 | "syslog_pri": "53", 9 | "syslog_timestamp": "Feb 2 09:47:12", 10 | "syslog_hostname": "www", 11 | "syslog_program": "apache2", 12 | "syslog_message": "some apache message" 13 | } 14 | }, 15 | { 16 | "#": "This is an example of negative testing: this message should fail parsing", 17 | "in": "Feb 2 09:47:12 www apache2: invalid syslog message", 18 | "out": { 19 | "tags": ["_grokparsefailure"] 20 | } 21 | } 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /logstash-tester.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | usage() { 4 | echo " 5 | Logstash Tester - Unit-testing for Logstash configuration fields 6 | 7 | Usage: 8 | ./logstash-tester.sh [-chp] -d path [test_target] 9 | 10 | - 'path' is the base directory for your config files and test cases. 11 | - 'test_target' takes one of three possible values: 12 | 'patterns', 'filters', 'all'. 13 | It tells logstash-tester to runs pattern tests only, 14 | filter tests only or both, respectively. The default is 'all'. 15 | See examples for ... hum ... examples. 16 | 17 | Options: 18 | -d 19 | Root directory for all your logstash config and test files. 20 | It is not optional and it should have a specific structure. 21 | See documentation for details or the 'example' directory in the 22 | repository root. 23 | -c 24 | Don't check the syntax of logstash configuration before running tests. 25 | The default is to execute 'logstash --configtest -f ' 26 | before running the tests. 27 | -p 28 | The filter tests subdirectory, inside the main test case directory. 29 | This allows you to run a subset of tests. 30 | -h 31 | This text. 32 | 33 | Examples 34 | ./logstash-tester.sh -d example 35 | The simplest command line form. Run all tests, root dir for config and 36 | test files is 'example'. 37 | ./logstash-tester.sh -d example -p syslog filters 38 | Run the subset of filter tests located in the 'syslog' directory 39 | (./test/filters/syslog). 40 | 41 | More info on the project repository: 42 | https://github.com/gaspaio/logstash-tester 43 | " 44 | 45 | } 46 | 47 | error() { 48 | echo "$* See help (-h) for details." 49 | exit 1 50 | } 51 | 52 | run_docker() { 53 | action=$1 54 | configtest=$2 55 | 56 | if ! hash docker 2> /dev/null; then 57 | error "Can't find the Docker executable. Did you install it?" 58 | fi 59 | 60 | rootdir=`dirname $0` 61 | 62 | echo "====> Build docker image for test" 63 | docker build -t gaspaio/logstash-tester \ 64 | --build-arg LST=$rootdir \ 65 | --build-arg FILTER_CONFIG=$3 \ 66 | --build-arg PATTERN_CONFIG=$4 \ 67 | --build-arg FILTER_TESTS=$5 \ 68 | --build-arg PATTERN_TESTS=$6 \ 69 | -f $rootdir/Dockerfile . 70 | 71 | echo "====> Run test in docker container" 72 | docker run --rm -it gaspaio/logstash-tester $action $configtest 73 | } 74 | 75 | # Default values 76 | action=all 77 | configtest=y 78 | filter_test_path= 79 | datadir= 80 | while getopts ":d:p:ch" opt; do 81 | case $opt in 82 | d) 83 | if [[ -d $OPTARG ]]; then 84 | datadir=$OPTARG 85 | else 86 | error "'$OPTARG' is not a valid directory." 87 | fi 88 | ;; 89 | c) 90 | configtest=n 91 | ;; 92 | p) 93 | filter_test_path=$OPTARG 94 | ;; 95 | h) 96 | usage 97 | exit 0 98 | ;; 99 | :) 100 | error "Option -$OPTARG requires an argument." 101 | ;; 102 | \?) 103 | error "Invalid option -$OPTARG." 104 | ;; 105 | esac 106 | done 107 | 108 | # Handle remaining positional arguments 109 | shift $((OPTIND-1)) 110 | 111 | if [[ -z $@ ]]; then 112 | action=all 113 | elif [[ $@ != 'all' && $@ != 'filters' && $@ != 'patterns' ]]; then 114 | error "'$@' is not a valid action." 115 | else 116 | action=$@ 117 | fi 118 | 119 | # Handle compulsory arguments 120 | if [[ -z $datadir ]]; then 121 | error "You must define a root dir for your config and test files." 122 | fi 123 | 124 | # Validate directories 125 | docker_filter_config=$datadir/config/conf.d 126 | if [[ ! -d $docker_filter_config ]]; then 127 | error "The filter config directory '$docker_filter_config' does not exist." 128 | fi 129 | 130 | docker_pattern_config=$datadir/config/patterns 131 | if [[ ! -d $docker_pattern_config ]]; then 132 | error "The patterns directory '$docker_pattern_config' does not exist." 133 | fi 134 | 135 | docker_filter_test=$datadir/test/filters 136 | if [[ ! -z $filter_test_path ]]; then 137 | docker_filter_test=$docker_filter_test/$filter_test_path 138 | fi 139 | if [[ ! -d $docker_filter_test ]]; then 140 | error "The filter tests directory '$docker_filter_test' does not exist." 141 | fi 142 | 143 | docker_pattern_test=$datadir/test/patterns 144 | if [[ ! -d $docker_pattern_test ]]; then 145 | error "The patterns tests directory '$docker_pattern_test' does not exist." 146 | fi 147 | 148 | run_docker $action $configtest $docker_filter_config $docker_pattern_config $docker_filter_test $docker_pattern_test 149 | 150 | -------------------------------------------------------------------------------- /run_example.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "Running the example dir." 3 | echo "Logstash config is stored at example/config, test cases at example/test" 4 | ./logstash-tester.sh -d example $@ 5 | -------------------------------------------------------------------------------- /test/run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [[ $1 == "all" || $1 == "patterns" ]]; then 3 | echo "### RUN PATTERN TESTS #####################" 4 | rspec -f p /test/spec/patterns_spec.rb 5 | fi 6 | 7 | if [[ $1 == "all" || $1 == "filters" ]]; then 8 | echo "### RUN FILTER Tests ####################" 9 | if [[ $2 == "y" ]]; then 10 | logstash --configtest -f /test/spec/filter_config 11 | fi 12 | 13 | rspec -f p /test/spec/filter_spec.rb 14 | fi 15 | 16 | -------------------------------------------------------------------------------- /test/spec/filter_spec.rb: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | require "logstash/devutils/rspec/spec_helper" 3 | require "rspec/expectations" 4 | require 'json' 5 | 6 | # Load the test cases 7 | filter_data = Dir[File.join(File.dirname(__FILE__), 'filter_data/**/*.json')] 8 | 9 | # Load the logstash filter config files 10 | files = Dir[File.join(File.dirname(__FILE__), 'filter_config/*.filter.conf')] 11 | @@configuration = String.new 12 | files.sort.each do |file| 13 | @@configuration << File.read(file) 14 | end 15 | 16 | 17 | def run_case(tcase, fields, ignore, data_file, i) 18 | input = fields 19 | input['message'] = tcase['in'] 20 | 21 | msg_header = "[#{File.basename(data_file)}##{i}]" 22 | 23 | sample(input) do 24 | expected = tcase['out'] 25 | expected_fields = expected.keys 26 | 27 | # Handle no results (for example, when a line is voluntarily dropped) 28 | lsresult = results.any? ? results[0] : {} 29 | result_fields = lsresult.to_hash.keys.select { |f| not ignore.include?(f) } 30 | 31 | # TODO test for grokparsefailures 32 | 33 | # Test for presence of expected fields 34 | missing = expected_fields.select { |f| not result_fields.include?(f) } 35 | msg = "\n#{msg_header} Fields missing in logstash output: #{missing}\nComplete logstash output: #{lsresult.to_hash}\n--" 36 | expect(missing).to be_empty, msg 37 | 38 | # Test for absence of unknown fields 39 | extra = result_fields.select { |f| not expected_fields.include?(f) } 40 | msg = "\n#{msg_header} Unexpected fields in logstash output: #{extra}\nComplete logstash output: #{lsresult.to_hash}\n--" 41 | expect(extra).to be_empty, msg 42 | 43 | # Test individual field values 44 | expected.each do |name,value| 45 | msg = "\n#{msg_header} Field value mismatch: '#{name}'\nExpected: #{value} (#{value.class})\nGot: #{lsresult[name]} (#{lsresult[name].class})\n\n--" 46 | expect(lsresult[name].to_s).to eq(value.to_s), msg 47 | end 48 | end 49 | end 50 | 51 | filter_data.each do |data_file| 52 | # Count test cases in this file 53 | test_case = JSON.parse(File.read(data_file)) 54 | 55 | (0..(test_case['cases'].length-1)).each do |i| 56 | describe "#{File.basename(data_file)}##{i}" do 57 | config(@@configuration) 58 | test_case = JSON.parse(File.read(data_file)) 59 | run_case(test_case['cases'][i], test_case['fields'], test_case['ignore'], data_file, i) 60 | end 61 | end 62 | end 63 | 64 | -------------------------------------------------------------------------------- /test/spec/patterns_spec.rb: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | require_relative "./spec_helper" 3 | require "json" 4 | 5 | # Load pattern test cases 6 | pattern_data = Dir[File.join(File.dirname(__FILE__), 'pattern_data/**/*.json')] 7 | 8 | pattern_data.each do |data_file| 9 | # Load test case data from file 10 | @@test_case = JSON.parse(File.read(data_file)) 11 | 12 | describe "#{@@test_case['name']} (#{@@test_case['pattern']}, #{File.basename(data_file)})" do 13 | @@test_case['cases'].each_with_index do |item,i| 14 | name = "##{i} - #{item["in"][0..25]}..." 15 | expected_fields = item["out"].keys 16 | pattern = @@test_case['pattern'] 17 | 18 | # Expected fields are present, have expected value, and no other fields are present 19 | it "'#{name}' should be correct" do 20 | match_res = grok_match(pattern, item['in']) 21 | # Ignore logstash added fields. These are always present. 22 | result = match_res.select{ |x| !['@version', '@timestamp', 'message'].include?(x) } 23 | 24 | # test for missing fields in match output 25 | missing = expected_fields.select { |f| not result.keys.include?(f) } 26 | msg = "\nFields missing in pattern output: #{missing}\nComplete grok output: #{result}\n\n--" 27 | expect(missing).to be_empty, msg 28 | 29 | # test for unexpected fields in match output 30 | extra = result.keys.select { |f| not expected_fields.include?(f) } 31 | msg = "\nUnexpected fields in pattern output: #{extra}\nComplete grok output: #{result}\n\n--" 32 | expect(extra).to be_empty, msg 33 | 34 | # test for field values 35 | item['out'].each do |name,value| 36 | msg = "\nField mismatch: '#{name}'\nExpected: #{value}\nGot: #{match_res[name]}\nComplete grok output: #{result}\n\n--" 37 | expect(result[name]).to eq(value), msg 38 | end 39 | end 40 | end 41 | end 42 | end 43 | 44 | -------------------------------------------------------------------------------- /test/spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # Forked from: https://github.com/logstash-plugins/logstash-patterns-core 2 | 3 | require "logstash/devutils/rspec/spec_helper" 4 | require 'rspec/expectations' 5 | 6 | # running the grok code outside a logstash package means 7 | # LOGSTASH_HOME will not be defined, so let's set it here 8 | # before requiring the grok filter 9 | unless LogStash::Environment.const_defined?(:LOGSTASH_HOME) 10 | LogStash::Environment::LOGSTASH_HOME = File.expand_path("../", __FILE__) 11 | end 12 | 13 | require "logstash/filters/grok" 14 | 15 | module GrokHelpers 16 | def grok_match(label, message) 17 | grok = build_grok(label) 18 | event = build_event(message) 19 | grok.filter(event) 20 | event.to_hash 21 | end 22 | 23 | def build_grok(label) 24 | grok = LogStash::Filters::Grok.new("match" => ["message", "%{#{label}}"]) 25 | # Manually set patterns_dir so that grok finds them when we're testing patterns 26 | grok.patterns_dir = ["/etc/logstash/patterns"] 27 | grok.register 28 | grok 29 | end 30 | 31 | def build_event(message) 32 | LogStash::Event.new("message" => message) 33 | end 34 | end 35 | 36 | RSpec.configure do |c| 37 | c.include GrokHelpers 38 | end 39 | 40 | RSpec::Matchers.define :pass do |expected| 41 | match do |actual| 42 | !actual.include?("tags") 43 | end 44 | end 45 | 46 | RSpec::Matchers.define :match do |value| 47 | match do |grok| 48 | grok = build_grok(grok) 49 | event = build_event(value) 50 | grok.filter(event) 51 | !event.include?("tags") 52 | end 53 | end 54 | 55 | --------------------------------------------------------------------------------