├── README.md ├── conf.d ├── collectd-filter.conf ├── collectd-input.conf └── sysdig-shipper.conf ├── elasticsearch-template-es2x.json ├── elasticsearch-template-es5x.json ├── elasticsearch-template-es6x.json ├── elasticsearch-template-es7x.json ├── filebeat.yml ├── indexer.d └── indexer.conf ├── patterns ├── amavis ├── apache ├── fail2ban ├── postfix ├── saslauthd ├── sshd ├── sysdig └── zimbra └── shipper.d └── shipper.conf /README.md: -------------------------------------------------------------------------------- 1 | # logstash 2 | ## my logsash config 3 | 4 | ELK (Elasticsearch + Logstash + Kibana) is fun! 5 | 6 | Logstash is super flexible, most operations can be. 7 | 8 | ## memo 9 | 10 | Start separately Java process, shipper indexer. 11 | (divided into two by copying the startup script that is distributed in the package version) 12 | 13 | ## reference 14 | 15 | postfix grok patterns : 16 | * https://github.com/whyscream/postfix-grok-patterns 17 | * https://gist.github.com/poolski/9911628 18 | * https://gist.github.com/jamtur01/4385667 19 | * https://gist.github.com/randywallace/6983588 20 | 21 | sshd grok patterns : 22 | * https://github.com/autosportlabs/docker-logstash/blob/master/src/conf/520-mogrify-sshd.conf 23 | 24 | Lightweight log shipper : logstash-forwarder (aka lumberjack) 25 | * https://github.com/elasticsearch/logstash-forwarder 26 | * https://www.digitalocean.com/community/tutorial_series/centralized-logging-with-logstash-and-kibana-on-ubuntu-14-04 27 | * https://www.digitalocean.com/community/tutorials/adding-logstash-filters-to-improve-centralized-logging 28 | 29 | grok filter ruby : 30 | * https://groups.google.com/forum/#!topic/logstash-users/iEYRv7bCqdM 31 | * http://stackoverflow.com/questions/20512416/adding-tags-to-logstash-events-based-on-the-md5-of-the-filename 32 | 33 | kibana geoip BetterMap : 34 | * https://beingasysadmin.wordpress.com/2014/04/07/near-realtime-dashboard-with-kibana-and-elasticsearch/ 35 | * http://dev.maxmind.com/geoip/legacy/geolite/ 36 | 37 | grok apache User-Agent : 38 | * http://untergeek.com/2013/09/11/getting-apache-to-output-json-for-logstash-1-2-x/ 39 | * https://github.com/ua-parser/uap-core/blob/master/regexes.yaml 40 | 41 | Integrating DataDog 42 | * http://ifdattic.com/integrating-datadog-and-logstash-on-aws-ec2/ 43 | 44 | zimbra mailbox.log & zimbra.log (amavis) 45 | * http://blog.itlinux.cl/blog/2015/05/25/buscando-mensajes-de-correo-con-kibana/ 46 | * https://github.com/ITLinuxCL/zimbra_logstash 47 | * http://antisp.in/2014/04/01/useful-logstash-grok-patterns/ 48 | * https://github.com/Autobase/Zimbra/blob/4bf3dc250c68a38e38286bdd972c8d5469d40e34/ZimbraCommon/src/java/com/zimbra/common/util/ZimbraLog.java 49 | * https://wiki.zimbra.com/wiki/Centralized_Logs_-_Elasticsearch,_Logstash_and_Kibana 50 | * https://blog.zimbra.com/2007/05/mailboxlog-the-king-of-zimbra-log-files/ 51 | * https://www.zimbra.com/docs/os/5.0.19/administration_guide/9_Monitoring.11.1.html 52 | -------------------------------------------------------------------------------- /conf.d/collectd-filter.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | # TEST implementation of parse for collectd 3 | if [type] == "collectd" { 4 | if [plugin] { 5 | mutate { 6 | rename => { "plugin" => "collectd_plugin" } 7 | } 8 | } 9 | if [plugin_instance] { 10 | mutate { 11 | rename => { "plugin_instance" => "collectd_plugin_instance" } 12 | } 13 | } 14 | if [type_instance] { 15 | mutate { 16 | rename => { "type_instance" => "collectd_type_instance" } 17 | } 18 | } 19 | if [value] { 20 | mutate { 21 | rename => { "value" => "collectd_value" } 22 | } 23 | mutate { 24 | convert => { "collectd_value" => "float" } 25 | } 26 | } 27 | if [collectd_plugin] == "interface" { 28 | mutate { 29 | add_field => { 30 | "collectd_value_instance" => "rx" 31 | "collectd_value" => "%{rx}" 32 | } 33 | } 34 | mutate { 35 | convert => { 36 | "tx" => "float" 37 | "collectd_value" => "float" 38 | } 39 | } 40 | # force clone for kibana3 41 | clone { 42 | clones => [ "tx" ] 43 | } 44 | ##### BUG EXISTS : AFTER clone 'if [type] == "foo"' NOT WORKING : ruby code is working ##### 45 | ruby { 46 | code => " 47 | if event.get('type') == 'tx' 48 | event.set('collectd_value_instance', 'tx') 49 | event.set('collectd_value', event.get('tx')) 50 | end 51 | " 52 | } 53 | mutate { 54 | replace => { "_type" => "collectd" } 55 | replace => { "type" => "collectd" } 56 | remove_field => [ "rx", "tx" ] 57 | } 58 | } 59 | if [collectd_plugin] == "disk" { 60 | mutate { 61 | add_field => { 62 | "collectd_value_instance" => "read" 63 | "collectd_value" => "%{read}" 64 | } 65 | } 66 | mutate { 67 | convert => { 68 | "write" => "float" 69 | "collectd_value" => "float" 70 | } 71 | } 72 | # force clone for kibana3 73 | clone { 74 | clones => [ "write" ] 75 | } 76 | ##### BUG EXISTS : AFTER clone 'if [type] == "foo"' NOT WORKING : ruby code is working ##### 77 | ruby { 78 | code => " 79 | if event.get('type') == 'write' 80 | event.set('collectd_value_instance','write') 81 | event.set('collectd_value', event.get('write')) 82 | end 83 | " 84 | } 85 | mutate { 86 | replace => { "_type" => "collectd" } 87 | replace => { "type" => "collectd" } 88 | remove_field => [ "read", "write" ] 89 | } 90 | } 91 | if [collectd_plugin] == "df" { 92 | mutate { 93 | add_field => { 94 | "collectd_value_instance" => "free" 95 | "collectd_value" => "%{free}" 96 | } 97 | } 98 | mutate { 99 | convert => { 100 | "used" => "float" 101 | "collectd_value" => "float" 102 | } 103 | } 104 | # force clone for kibana3 105 | clone { 106 | clones => [ "used" ] 107 | } 108 | ##### BUG EXISTS : AFTER clone 'if [type] == "foo"' NOT WORKING : ruby code is working ##### 109 | ruby { 110 | code => " 111 | if event.get('type') == 'used' 112 | event.set('collectd_value_instance', 'used') 113 | event.set('collectd_value', event.get('used')) 114 | end 115 | " 116 | } 117 | mutate { 118 | replace => { "_type" => "collectd" } 119 | replace => { "type" => "collectd" } 120 | remove_field => [ "used", "free" ] 121 | } 122 | } 123 | if [collectd_plugin] == "load" { 124 | mutate { 125 | add_field => { 126 | "collectd_value_instance" => "shortterm" 127 | "collectd_value" => "%{shortterm}" 128 | } 129 | } 130 | mutate { 131 | convert => { 132 | "longterm" => "float" 133 | "midterm" => "float" 134 | "collectd_value" => "float" 135 | } 136 | } 137 | # force clone for kibana3 138 | clone { 139 | clones => [ "longterm", "midterm" ] 140 | } 141 | ##### BUG EXISTS : AFTER clone 'if [type] == "foo"' NOT WORKING : ruby code is working ##### 142 | ruby { 143 | code => " 144 | if event.get('type') != 'collectd' 145 | event.set('collectd_value_instance', event.get('type')) 146 | event.set('collectd_value', event.get(event.get('type'))) 147 | end 148 | " 149 | } 150 | mutate { 151 | replace => { "_type" => "collectd" } 152 | replace => { "type" => "collectd" } 153 | remove_field => [ "longterm", "midterm", "shortterm" ] 154 | } 155 | } 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /conf.d/collectd-input.conf: -------------------------------------------------------------------------------- 1 | input { 2 | udp { 3 | port => 25826 4 | type => "collectd" 5 | buffer_size => 1452 6 | codec => collectd { 7 | authfile => "/etc/logstash/collectd.auth" 8 | security_level => "Encrypt" 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /conf.d/sysdig-shipper.conf: -------------------------------------------------------------------------------- 1 | input { 2 | stdin { 3 | type => "sysdig" 4 | } 5 | } 6 | 7 | output { 8 | redis { 9 | host => "127.0.0.1" 10 | data_type => "list" 11 | key => "logstash" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /elasticsearch-template-es2x.json: -------------------------------------------------------------------------------- 1 | { 2 | "template" : "logstash-*", 3 | "settings" : { 4 | "number_of_replicas" : 0, 5 | "number_of_shards" : 1, 6 | "index.refresh_interval" : "5s" 7 | }, 8 | "mappings" : { 9 | "_default_" : { 10 | "_all" : {"enabled" : true, "omit_norms" : true}, 11 | "dynamic_templates" : [ { 12 | "message_field" : { 13 | "match" : "message", 14 | "match_mapping_type" : "string", 15 | "mapping" : { 16 | "type" : "string", "index" : "analyzed", "omit_norms" : true, 17 | "fielddata" : { "format" : "disabled" } 18 | } 19 | } 20 | }, { 21 | "string_fields" : { 22 | "match" : "*", 23 | "match_mapping_type" : "string", 24 | "mapping" : { 25 | "type" : "string", "index" : "not_analyzed", "doc_values" : true 26 | } 27 | } 28 | }, { 29 | "float_fields" : { 30 | "match" : "*", 31 | "match_mapping_type" : "float", 32 | "mapping" : { "type" : "float", "doc_values" : true } 33 | } 34 | }, { 35 | "double_fields" : { 36 | "match" : "*", 37 | "match_mapping_type" : "double", 38 | "mapping" : { "type" : "double", "doc_values" : true } 39 | } 40 | }, { 41 | "byte_fields" : { 42 | "match" : "*", 43 | "match_mapping_type" : "byte", 44 | "mapping" : { "type" : "byte", "doc_values" : true } 45 | } 46 | }, { 47 | "short_fields" : { 48 | "match" : "*", 49 | "match_mapping_type" : "short", 50 | "mapping" : { "type" : "short", "doc_values" : true } 51 | } 52 | }, { 53 | "integer_fields" : { 54 | "match" : "*", 55 | "match_mapping_type" : "integer", 56 | "mapping" : { "type" : "integer", "doc_values" : true } 57 | } 58 | }, { 59 | "long_fields" : { 60 | "match" : "*", 61 | "match_mapping_type" : "long", 62 | "mapping" : { "type" : "long", "doc_values" : true } 63 | } 64 | }, { 65 | "date_fields" : { 66 | "match" : "*", 67 | "match_mapping_type" : "date", 68 | "mapping" : { "type" : "date", "doc_values" : true } 69 | } 70 | }, { 71 | "geo_point_fields" : { 72 | "match" : "*", 73 | "match_mapping_type" : "geo_point", 74 | "mapping" : { "type" : "geo_point", "doc_values" : true } 75 | } 76 | } ], 77 | "properties" : { 78 | "@timestamp": { "type": "date", "doc_values" : true }, 79 | "@version": { "type": "string", "index": "not_analyzed", "doc_values" : true }, 80 | "apache_agent": { 81 | "type": "string", "index": "analyzed", "omit_norms" : true, 82 | "fielddata" : { "format" : "disabled" } 83 | }, 84 | "apache_request": { 85 | "type": "string", "index": "analyzed", "omit_norms" : true, 86 | "fielddata" : { "format" : "disabled" } 87 | }, 88 | "apache_referrer": { 89 | "type": "string", "index": "analyzed", "omit_norms" : true, 90 | "fielddata" : { "format" : "disabled" } 91 | }, 92 | "apache_error_message": { 93 | "type": "string", "index": "analyzed", "omit_norms" : true, 94 | "fielddata" : { "format" : "disabled" } 95 | }, 96 | "syslog_message": { 97 | "type": "string", "index": "analyzed", "omit_norms" : true, 98 | "fielddata" : { "format" : "disabled" } 99 | }, 100 | "dpkg_message": { 101 | "type": "string", "index": "analyzed", "omit_norms" : true, 102 | "fielddata" : { "format" : "disabled" } 103 | }, 104 | "amavis_source_ip": { "type": "ip", "doc_values" : true }, 105 | "amavis_relay_ip": { "type": "ip", "doc_values" : true }, 106 | "amavis_origin_ip": { "type": "ip", "doc_values" : true }, 107 | "clamd_source_ip": { "type": "ip", "doc_values" : true }, 108 | "clamd_relay_ip": { "type": "ip", "doc_values" : true }, 109 | "clamd_origin_ip": { "type": "ip", "doc_values" : true }, 110 | "postfix_relay_ip": { "type": "ip", "doc_values" : true }, 111 | "postfix_server_ip": { "type": "ip", "doc_values" : true }, 112 | "postfix_client_ip": { "type": "ip", "doc_values" : true }, 113 | "postfix_dnsbl_result": { "type": "ip", "doc_values" : true }, 114 | "sshd_listen_ip": { "type": "ip", "doc_values" : true }, 115 | "sshd_client_ip": { "type": "ip", "doc_values" : true }, 116 | "zimbra_account_ip": { "type": "ip", "doc_values" : true }, 117 | "zimbra_account_oip": { "type": "ip", "doc_values" : true }, 118 | "fail2ban_source_ip": { "type": "ip", "doc_values" : true }, 119 | "offset": { "type": "long", "doc_values" : true }, 120 | "geoip" : { 121 | "type" : "object", 122 | "dynamic": true, 123 | "properties" : { 124 | "ip": { "type": "ip", "doc_values" : true }, 125 | "location" : { "type" : "geo_point", "doc_values" : true }, 126 | "latitude" : { "type" : "float", "doc_values" : true }, 127 | "longitude" : { "type" : "float", "doc_values" : true } 128 | } 129 | }, 130 | "postfix_geoip" : { 131 | "type" : "object", 132 | "dynamic": true, 133 | "properties" : { 134 | "ip": { "type": "ip", "doc_values" : true }, 135 | "location" : { "type" : "geo_point", "doc_values" : true }, 136 | "latitude" : { "type" : "float", "doc_values" : true }, 137 | "longitude" : { "type" : "float", "doc_values" : true } 138 | } 139 | }, 140 | "sshd_geoip" : { 141 | "type" : "object", 142 | "dynamic": true, 143 | "properties" : { 144 | "ip": { "type": "ip", "doc_values" : true }, 145 | "location" : { "type" : "geo_point", "doc_values" : true }, 146 | "latitude" : { "type" : "float", "doc_values" : true }, 147 | "longitude" : { "type" : "float", "doc_values" : true } 148 | } 149 | }, 150 | "apache_geoip" : { 151 | "type" : "object", 152 | "dynamic": true, 153 | "properties" : { 154 | "ip": { "type": "ip", "doc_values" : true }, 155 | "location" : { "type" : "geo_point", "doc_values" : true }, 156 | "latitude" : { "type" : "float", "doc_values" : true }, 157 | "longitude" : { "type" : "float", "doc_values" : true } 158 | } 159 | }, 160 | "apache_error_geoip" : { 161 | "type" : "object", 162 | "dynamic": true, 163 | "properties" : { 164 | "ip": { "type": "ip", "doc_values" : true }, 165 | "location" : { "type" : "geo_point", "doc_values" : true }, 166 | "latitude" : { "type" : "float", "doc_values" : true }, 167 | "longitude" : { "type" : "float", "doc_values" : true } 168 | } 169 | }, 170 | "zimbra_geoip" : { 171 | "type" : "object", 172 | "dynamic": true, 173 | "properties" : { 174 | "ip": { "type": "ip", "doc_values" : true }, 175 | "location" : { "type" : "geo_point", "doc_values" : true }, 176 | "latitude" : { "type" : "float", "doc_values" : true }, 177 | "longitude" : { "type" : "float", "doc_values" : true } 178 | } 179 | }, 180 | "zimbra_origin_geoip" : { 181 | "type" : "object", 182 | "dynamic": true, 183 | "properties" : { 184 | "ip": { "type": "ip", "doc_values" : true }, 185 | "location" : { "type" : "geo_point", "doc_values" : true }, 186 | "latitude" : { "type" : "float", "doc_values" : true }, 187 | "longitude" : { "type" : "float", "doc_values" : true } 188 | } 189 | }, 190 | "amavis_geoip" : { 191 | "type" : "object", 192 | "dynamic": true, 193 | "properties" : { 194 | "ip": { "type": "ip", "doc_values" : true }, 195 | "location" : { "type" : "geo_point", "doc_values" : true }, 196 | "latitude" : { "type" : "float", "doc_values" : true }, 197 | "longitude" : { "type" : "float", "doc_values" : true } 198 | } 199 | }, 200 | "fail2ban_geoip" : { 201 | "type" : "object", 202 | "dynamic": true, 203 | "properties" : { 204 | "ip": { "type": "ip", "doc_values" : true }, 205 | "location" : { "type" : "geo_point", "doc_values" : true }, 206 | "latitude" : { "type" : "float", "doc_values" : true }, 207 | "longitude" : { "type" : "float", "doc_values" : true } 208 | } 209 | } 210 | } 211 | } 212 | } 213 | } 214 | -------------------------------------------------------------------------------- /elasticsearch-template-es5x.json: -------------------------------------------------------------------------------- 1 | { 2 | "template" : "logstash-*", 3 | "version" : 50001, 4 | "settings" : { 5 | "number_of_replicas" : 0, 6 | "number_of_shards" : 1, 7 | "index.refresh_interval" : "5s" 8 | }, 9 | "mappings" : { 10 | "_default_" : { 11 | "_all" : {"enabled" : true, "norms" : false}, 12 | "dynamic_templates" : [ { 13 | "message_field" : { 14 | "path_match" : "message", 15 | "match_mapping_type" : "string", 16 | "mapping" : { 17 | "type" : "text", 18 | "norms" : false 19 | } 20 | } 21 | }, { 22 | "string_fields" : { 23 | "match" : "*", 24 | "match_mapping_type" : "string", 25 | "mapping" : { 26 | "type" : "text", "norms" : false, 27 | "fields" : { 28 | "keyword" : { "type": "keyword" } 29 | } 30 | } 31 | } 32 | } ], 33 | "properties" : { 34 | "@timestamp": { "type": "date", "include_in_all": false }, 35 | "@version": { "type": "keyword", "include_in_all": false }, 36 | "apache_agent": { "type": "text", "norms" : false }, 37 | "apache_request": { "type": "text", "norms" : false }, 38 | "apache_referrer": { "type": "text", "norms" : false }, 39 | "apache_error_message": { "type": "text", "norms" : false }, 40 | "syslog_message": { "type": "text", "norms" : false }, 41 | "dpkg_message": { "type": "text", "norms" : false }, 42 | "amavis_source_ip": { "type": "ip" }, 43 | "amavis_relay_ip": { "type": "ip" }, 44 | "amavis_origin_ip": { "type": "ip" }, 45 | "clamd_source_ip": { "type": "ip" }, 46 | "clamd_relay_ip": { "type": "ip" }, 47 | "clamd_origin_ip": { "type": "ip" }, 48 | "postfix_relay_ip": { "type": "ip" }, 49 | "postfix_server_ip": { "type": "ip" }, 50 | "postfix_client_ip": { "type": "ip" }, 51 | "postfix_dnsbl_result": { "type": "ip" }, 52 | "sshd_listen_ip": { "type": "ip" }, 53 | "sshd_client_ip": { "type": "ip" }, 54 | "zimbra_account_ip": { "type": "ip" }, 55 | "zimbra_account_oip": { "type": "ip" }, 56 | "fail2ban_source_ip": { "type": "ip" }, 57 | "offset": { "type": "long" }, 58 | "geoip" : { 59 | "dynamic": true, 60 | "properties" : { 61 | "ip": { "type": "ip" }, 62 | "location" : { "type" : "geo_point" }, 63 | "latitude" : { "type" : "half_float" }, 64 | "longitude" : { "type" : "half_float" } 65 | } 66 | }, 67 | "postfix_geoip" : { 68 | "dynamic": true, 69 | "properties" : { 70 | "ip": { "type": "ip" }, 71 | "location" : { "type" : "geo_point" }, 72 | "latitude" : { "type" : "half_float" }, 73 | "longitude" : { "type" : "half_float" } 74 | } 75 | }, 76 | "sshd_geoip" : { 77 | "dynamic": true, 78 | "properties" : { 79 | "ip": { "type": "ip" }, 80 | "location" : { "type" : "geo_point" }, 81 | "latitude" : { "type" : "half_float" }, 82 | "longitude" : { "type" : "half_float" } 83 | } 84 | }, 85 | "apache_geoip" : { 86 | "dynamic": true, 87 | "properties" : { 88 | "ip": { "type": "ip" }, 89 | "location" : { "type" : "geo_point" }, 90 | "latitude" : { "type" : "half_float" }, 91 | "longitude" : { "type" : "half_float" } 92 | } 93 | }, 94 | "apache_error_geoip" : { 95 | "dynamic": true, 96 | "properties" : { 97 | "ip": { "type": "ip" }, 98 | "location" : { "type" : "geo_point" }, 99 | "latitude" : { "type" : "half_float" }, 100 | "longitude" : { "type" : "half_float" } 101 | } 102 | }, 103 | "zimbra_geoip" : { 104 | "dynamic": true, 105 | "properties" : { 106 | "ip": { "type": "ip" }, 107 | "location" : { "type" : "geo_point" }, 108 | "latitude" : { "type" : "half_float" }, 109 | "longitude" : { "type" : "half_float" } 110 | } 111 | }, 112 | "zimbra_origin_geoip" : { 113 | "dynamic": true, 114 | "properties" : { 115 | "ip": { "type": "ip" }, 116 | "location" : { "type" : "geo_point" }, 117 | "latitude" : { "type" : "half_float" }, 118 | "longitude" : { "type" : "half_float" } 119 | } 120 | }, 121 | "amavis_geoip" : { 122 | "dynamic": true, 123 | "properties" : { 124 | "ip": { "type": "ip" }, 125 | "location" : { "type" : "geo_point" }, 126 | "latitude" : { "type" : "half_float" }, 127 | "longitude" : { "type" : "half_float" } 128 | } 129 | }, 130 | "fail2ban_geoip" : { 131 | "dynamic": true, 132 | "properties" : { 133 | "ip": { "type": "ip" }, 134 | "location" : { "type" : "geo_point" }, 135 | "latitude" : { "type" : "half_float" }, 136 | "longitude" : { "type" : "half_float" } 137 | } 138 | } 139 | } 140 | } 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /elasticsearch-template-es6x.json: -------------------------------------------------------------------------------- 1 | { 2 | "template" : "logstash-*", 3 | "version" : 60001, 4 | "settings" : { 5 | "number_of_replicas" : 0, 6 | "number_of_shards" : 1, 7 | "index.refresh_interval" : "5s" 8 | }, 9 | "mappings" : { 10 | "_default_" : { 11 | "dynamic_templates" : [ { 12 | "message_field" : { 13 | "path_match" : "message", 14 | "match_mapping_type" : "string", 15 | "mapping" : { 16 | "type" : "text", 17 | "norms" : false 18 | } 19 | } 20 | }, { 21 | "string_fields" : { 22 | "match" : "*", 23 | "match_mapping_type" : "string", 24 | "mapping" : { 25 | "type" : "text", "norms" : false, 26 | "fields" : { 27 | "keyword" : { "type": "keyword", "ignore_above": 256 } 28 | } 29 | } 30 | } 31 | } ], 32 | "properties" : { 33 | "@timestamp": { "type": "date" }, 34 | "@version": { "type": "keyword" }, 35 | "apache_agent": { "type": "text", "norms" : false }, 36 | "apache_request": { "type": "text", "norms" : false }, 37 | "apache_referrer": { "type": "text", "norms" : false }, 38 | "apache_error_message": { "type": "text", "norms" : false }, 39 | "syslog_message": { "type": "text", "norms" : false }, 40 | "dpkg_message": { "type": "text", "norms" : false }, 41 | "amavis_source_ip": { "type": "ip" }, 42 | "amavis_relay_ip": { "type": "ip" }, 43 | "amavis_origin_ip": { "type": "ip" }, 44 | "clamd_source_ip": { "type": "ip" }, 45 | "clamd_relay_ip": { "type": "ip" }, 46 | "clamd_origin_ip": { "type": "ip" }, 47 | "postfix_relay_ip": { "type": "ip" }, 48 | "postfix_server_ip": { "type": "ip" }, 49 | "postfix_client_ip": { "type": "ip" }, 50 | "postfix_dnsbl_result": { "type": "ip" }, 51 | "sshd_listen_ip": { "type": "ip" }, 52 | "sshd_client_ip": { "type": "ip" }, 53 | "zimbra_account_ip": { "type": "ip" }, 54 | "zimbra_account_oip": { "type": "ip" }, 55 | "fail2ban_source_ip": { "type": "ip" }, 56 | "offset": { "type": "long" }, 57 | "geoip" : { 58 | "dynamic": true, 59 | "properties" : { 60 | "ip": { "type": "ip" }, 61 | "location" : { "type" : "geo_point" }, 62 | "latitude" : { "type" : "half_float" }, 63 | "longitude" : { "type" : "half_float" } 64 | } 65 | }, 66 | "postfix_geoip" : { 67 | "dynamic": true, 68 | "properties" : { 69 | "ip": { "type": "ip" }, 70 | "location" : { "type" : "geo_point" }, 71 | "latitude" : { "type" : "half_float" }, 72 | "longitude" : { "type" : "half_float" } 73 | } 74 | }, 75 | "sshd_geoip" : { 76 | "dynamic": true, 77 | "properties" : { 78 | "ip": { "type": "ip" }, 79 | "location" : { "type" : "geo_point" }, 80 | "latitude" : { "type" : "half_float" }, 81 | "longitude" : { "type" : "half_float" } 82 | } 83 | }, 84 | "apache_geoip" : { 85 | "dynamic": true, 86 | "properties" : { 87 | "ip": { "type": "ip" }, 88 | "location" : { "type" : "geo_point" }, 89 | "latitude" : { "type" : "half_float" }, 90 | "longitude" : { "type" : "half_float" } 91 | } 92 | }, 93 | "apache_error_geoip" : { 94 | "dynamic": true, 95 | "properties" : { 96 | "ip": { "type": "ip" }, 97 | "location" : { "type" : "geo_point" }, 98 | "latitude" : { "type" : "half_float" }, 99 | "longitude" : { "type" : "half_float" } 100 | } 101 | }, 102 | "zimbra_geoip" : { 103 | "dynamic": true, 104 | "properties" : { 105 | "ip": { "type": "ip" }, 106 | "location" : { "type" : "geo_point" }, 107 | "latitude" : { "type" : "half_float" }, 108 | "longitude" : { "type" : "half_float" } 109 | } 110 | }, 111 | "zimbra_origin_geoip" : { 112 | "dynamic": true, 113 | "properties" : { 114 | "ip": { "type": "ip" }, 115 | "location" : { "type" : "geo_point" }, 116 | "latitude" : { "type" : "half_float" }, 117 | "longitude" : { "type" : "half_float" } 118 | } 119 | }, 120 | "amavis_geoip" : { 121 | "dynamic": true, 122 | "properties" : { 123 | "ip": { "type": "ip" }, 124 | "location" : { "type" : "geo_point" }, 125 | "latitude" : { "type" : "half_float" }, 126 | "longitude" : { "type" : "half_float" } 127 | } 128 | }, 129 | "fail2ban_geoip" : { 130 | "dynamic": true, 131 | "properties" : { 132 | "ip": { "type": "ip" }, 133 | "location" : { "type" : "geo_point" }, 134 | "latitude" : { "type" : "half_float" }, 135 | "longitude" : { "type" : "half_float" } 136 | } 137 | } 138 | } 139 | } 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /elasticsearch-template-es7x.json: -------------------------------------------------------------------------------- 1 | { 2 | "template" : "logstash-*", 3 | "version" : 60002, 4 | "settings" : { 5 | "number_of_replicas" : 0, 6 | "number_of_shards" : 1, 7 | "index.refresh_interval" : "5s" 8 | }, 9 | "mappings" : { 10 | "dynamic_templates" : [ { 11 | "message_field" : { 12 | "path_match" : "message", 13 | "match_mapping_type" : "string", 14 | "mapping" : { 15 | "type" : "text", 16 | "norms" : false 17 | } 18 | } 19 | }, { 20 | "string_fields" : { 21 | "match" : "*", 22 | "match_mapping_type" : "string", 23 | "mapping" : { 24 | "type" : "text", "norms" : false, 25 | "fields" : { 26 | "keyword" : { "type": "keyword", "ignore_above": 256 } 27 | } 28 | } 29 | } 30 | } ], 31 | "properties" : { 32 | "@timestamp": { "type": "date" }, 33 | "@version": { "type": "keyword" }, 34 | "apache_agent": { "type": "text", "norms" : false }, 35 | "apache_request": { "type": "text", "norms" : false }, 36 | "apache_referrer": { "type": "text", "norms" : false }, 37 | "apache_error_message": { "type": "text", "norms" : false }, 38 | "apache_client_ip": { "type": "ip" }, 39 | "apache_client_ipv6": { "type": "ip" }, 40 | "apache_error_client_ip": { "type": "ip" }, 41 | "apache_error_client_ipv6": { "type": "ip" }, 42 | "syslog_message": { "type": "text", "norms" : false }, 43 | "dpkg_message": { "type": "text", "norms" : false }, 44 | "amavis_source_ip": { "type": "ip" }, 45 | "amavis_relay_ip": { "type": "ip" }, 46 | "amavis_origin_ip": { "type": "ip" }, 47 | "clamd_source_ip": { "type": "ip" }, 48 | "clamd_relay_ip": { "type": "ip" }, 49 | "clamd_origin_ip": { "type": "ip" }, 50 | "postfix_relay_ip": { "type": "ip" }, 51 | "postfix_server_ip": { "type": "ip" }, 52 | "postfix_client_ip": { "type": "ip" }, 53 | "postfix_dnsbl_result": { "type": "ip" }, 54 | "sshd_listen_ip": { "type": "ip" }, 55 | "sshd_client_ip": { "type": "ip" }, 56 | "sshd_client_ipv6": { "type": "ip" }, 57 | "zimbra_account_ip": { "type": "ip" }, 58 | "zimbra_account_oip": { "type": "ip" }, 59 | "fail2ban_source_ip": { "type": "ip" }, 60 | "offset": { "type": "long" }, 61 | "geoip" : { 62 | "dynamic": true, 63 | "properties" : { 64 | "ip": { "type": "ip" }, 65 | "location" : { "type" : "geo_point" }, 66 | "latitude" : { "type" : "half_float" }, 67 | "longitude" : { "type" : "half_float" }, 68 | "postal_code" : { "type" : "keyword" } 69 | } 70 | }, 71 | "postfix_geoip" : { 72 | "dynamic": true, 73 | "properties" : { 74 | "ip": { "type": "ip" }, 75 | "location" : { "type" : "geo_point" }, 76 | "latitude" : { "type" : "half_float" }, 77 | "longitude" : { "type" : "half_float" }, 78 | "postal_code" : { "type" : "keyword" } 79 | } 80 | }, 81 | "postfix_bgp" : { 82 | "dynamic": true, 83 | "properties" : { 84 | "ip": { "type": "ip" } 85 | } 86 | }, 87 | "sshd_geoip" : { 88 | "dynamic": true, 89 | "properties" : { 90 | "ip": { "type": "ip" }, 91 | "location" : { "type" : "geo_point" }, 92 | "latitude" : { "type" : "half_float" }, 93 | "longitude" : { "type" : "half_float" }, 94 | "postal_code" : { "type" : "keyword" } 95 | } 96 | }, 97 | "sshd_bgp" : { 98 | "dynamic": true, 99 | "properties" : { 100 | "ip": { "type": "ip" } 101 | } 102 | }, 103 | "apache_geoip" : { 104 | "dynamic": true, 105 | "properties" : { 106 | "ip": { "type": "ip" }, 107 | "location" : { "type" : "geo_point" }, 108 | "latitude" : { "type" : "half_float" }, 109 | "longitude" : { "type" : "half_float" }, 110 | "postal_code" : { "type" : "keyword" } 111 | } 112 | }, 113 | "apache_bgp" : { 114 | "dynamic": true, 115 | "properties" : { 116 | "ip": { "type": "ip" } 117 | } 118 | }, 119 | "apache_error_geoip" : { 120 | "dynamic": true, 121 | "properties" : { 122 | "ip": { "type": "ip" }, 123 | "location" : { "type" : "geo_point" }, 124 | "latitude" : { "type" : "half_float" }, 125 | "longitude" : { "type" : "half_float" }, 126 | "postal_code" : { "type" : "keyword" } 127 | } 128 | }, 129 | "apache_error_bgp" : { 130 | "dynamic": true, 131 | "properties" : { 132 | "ip": { "type": "ip" } 133 | } 134 | }, 135 | "zimbra_geoip" : { 136 | "dynamic": true, 137 | "properties" : { 138 | "ip": { "type": "ip" }, 139 | "location" : { "type" : "geo_point" }, 140 | "latitude" : { "type" : "half_float" }, 141 | "longitude" : { "type" : "half_float" }, 142 | "postal_code" : { "type" : "keyword" } 143 | } 144 | }, 145 | "zimbra_bgp" : { 146 | "dynamic": true, 147 | "properties" : { 148 | "ip": { "type": "ip" } 149 | } 150 | }, 151 | "zimbra_origin_geoip" : { 152 | "dynamic": true, 153 | "properties" : { 154 | "ip": { "type": "ip" }, 155 | "location" : { "type" : "geo_point" }, 156 | "latitude" : { "type" : "half_float" }, 157 | "longitude" : { "type" : "half_float" }, 158 | "postal_code" : { "type" : "keyword" } 159 | } 160 | }, 161 | "zimbra_origin_bgp" : { 162 | "dynamic": true, 163 | "properties" : { 164 | "ip": { "type": "ip" } 165 | } 166 | }, 167 | "amavis_geoip" : { 168 | "dynamic": true, 169 | "properties" : { 170 | "ip": { "type": "ip" }, 171 | "location" : { "type" : "geo_point" }, 172 | "latitude" : { "type" : "half_float" }, 173 | "longitude" : { "type" : "half_float" }, 174 | "postal_code" : { "type" : "keyword" } 175 | } 176 | }, 177 | "amavis_bgp" : { 178 | "dynamic": true, 179 | "properties" : { 180 | "ip": { "type": "ip" } 181 | } 182 | }, 183 | "fail2ban_geoip" : { 184 | "dynamic": true, 185 | "properties" : { 186 | "ip": { "type": "ip" }, 187 | "location" : { "type" : "geo_point" }, 188 | "latitude" : { "type" : "half_float" }, 189 | "longitude" : { "type" : "half_float" }, 190 | "postal_code" : { "type" : "keyword" } 191 | } 192 | }, 193 | "fail2ban_bgp" : { 194 | "dynamic": true, 195 | "properties" : { 196 | "ip": { "type": "ip" } 197 | } 198 | } 199 | } 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /filebeat.yml: -------------------------------------------------------------------------------- 1 | ######################## Filebeat Configuration ############################ 2 | 3 | #=========================== Filebeat prospectors ============================= 4 | 5 | filebeat.inputs: 6 | 7 | - type: log 8 | paths: 9 | - /var/log/syslog 10 | - /var/log/mail.log 11 | - /var/log/auth.log 12 | fields: 13 | log_type: syslog 14 | fields_under_root: true 15 | 16 | - type: log 17 | paths: 18 | /var/log/apache2/access.log 19 | fields: 20 | log_type: apache 21 | fields_under_root: true 22 | 23 | - type: log 24 | paths: 25 | /var/log/apache2/other_vhosts_access.log 26 | fields: 27 | log_type: apache-other-vhost 28 | fields_under_root: true 29 | 30 | - type: log 31 | paths: 32 | /var/log/apache2/error.log 33 | fields: 34 | log_type: apache-error 35 | fields_under_root: true 36 | 37 | - type: log 38 | paths: 39 | /var/log/varnish/varnishncsa.log 40 | fields: 41 | log_type: varnish 42 | fields_under_root: true 43 | 44 | - type: log 45 | paths: 46 | /var/log/dpkg.log 47 | fields: 48 | log_type: dpkg 49 | fields_under_root: true 50 | 51 | - type: log 52 | paths: 53 | /var/log/fail2ban.log 54 | fields: 55 | log_type: fail2ban 56 | fields_under_root: true 57 | 58 | #========================= Filebeat global options ============================ 59 | 60 | #filebeat.registry_file: /var/lib/filebeat/registry 61 | 62 | #================================ Outputs ====================================== 63 | 64 | #----------------------------- Logstash output --------------------------------- 65 | output.logstash: 66 | enabled: true 67 | # The Logstash hosts 68 | #hosts: ["localhost:5044"] 69 | hosts: ["172.16.4.21:5001"] 70 | 71 | # Optional SSL. By default is off. 72 | # List of root certificates for HTTPS server verifications 73 | ssl.certificate_authorities: ["/etc/logstash/logstash-forwarder.crt"] 74 | 75 | # Certificate for TLS client authentication 76 | #ssl.certificate: "/etc/logstash/logstash-forwarder.crt" 77 | 78 | # Client Certificate Key 79 | #ssl.key: "/etc/logstash/logstash-forwarder.key" 80 | 81 | # Configure SSL verification mode. If `none` is configured, all server hosts 82 | # and certificates will be accepted. In this mode, SSL based connections are 83 | # susceptible to man-in-the-middle attacks. Use only for testing. Default is 84 | # `full`. 85 | #ssl.verification_mode: full 86 | ssl.verification_mode: none 87 | 88 | #================================ Logging ====================================== 89 | # There are three options for the log output: syslog, file, stderr. 90 | # Under Windows systems, the log files are per default sent to the file output, 91 | # under all other system per default to syslog. 92 | 93 | # Sets log level. The default log level is info. 94 | # Available log levels are: critical, error, warning, info, debug 95 | #logging.level: info 96 | logging.level: info 97 | 98 | # Enable debug output for selected components. To enable all selectors use ["*"] 99 | # Other available selectors are "beat", "publish", "service" 100 | # Multiple selectors can be chained. 101 | #logging.selectors: [ ] 102 | 103 | # Send all logging output to syslog. The default is false. 104 | #logging.to_syslog: true 105 | logging.to_syslog: false 106 | 107 | # If enabled, filebeat periodically logs its internal metrics that have changed 108 | # in the last period. For each metric that changed, the delta from the value at 109 | # the beginning of the period is logged. Also, the total values for 110 | # all non-zero internal metrics are logged on shutdown. The default is true. 111 | #logging.metrics.enabled: true 112 | 113 | # The period after which to log the internal metrics. The default is 30s. 114 | #logging.metrics.period: 30s 115 | 116 | # Logging to rotating files files. Set logging.to_files to false to disable logging to 117 | # files. 118 | logging.to_files: true 119 | logging.files: 120 | # Configure the path where the logs are written. The default is the logs directory 121 | # under the home path (the binary location). 122 | #path: /var/log/filebeat 123 | path: /var/log/filebeat 124 | 125 | # The name of the files where the logs are written to. 126 | #name: filebeat 127 | name: filebeat.log 128 | 129 | # Configure log file size limit. If limit is reached, log file will be 130 | # automatically rotated 131 | #rotateeverybytes: 10485760 # = 10MB 132 | rotateeverybytes: 10485760 # = 10MB 133 | 134 | # Number of rotated log files to keep. Oldest files will be deleted first. 135 | #keepfiles: 7 136 | keepfiles: 7 137 | 138 | # The permissions mask to apply when rotating log files. The default value is 0600. 139 | # Must be a valid Unix-style file permissions mask expressed in octal notation. 140 | #permissions: 0600 141 | 142 | # Set to true to log messages in json format. 143 | #logging.json: false 144 | -------------------------------------------------------------------------------- /indexer.d/indexer.conf: -------------------------------------------------------------------------------- 1 | input { 2 | redis { 3 | host => "127.0.0.1" 4 | data_type => "list" 5 | key => "logstash" 6 | codec => json 7 | } 8 | } 9 | 10 | filter { 11 | if [log_type] == "syslog" { 12 | ### "$RepeatedMsgReduction off" /etc/rsyslog.conf 13 | #if [message] =~ /last message repeated [0-9]+ times/ { 14 | # drop { } 15 | #} 16 | ## enable high precision timestamps 17 | # comment out $ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat 18 | grok { 19 | match => { "message" => "(?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:syslog_timestamp}) %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" } 20 | add_field => { 21 | "syslog_received_at" => "%{@timestamp}" 22 | "syslog_received_from" => "%{[host][name]}" 23 | } 24 | } 25 | syslog_pri { 26 | } 27 | date { 28 | match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ] 29 | timezone => "Asia/Tokyo" 30 | locale => "en" 31 | } 32 | mutate { 33 | replace => { "syslog_timestamp" => "%{@timestamp}" } 34 | } 35 | # for check grok data type conversion bug??? 36 | mutate { 37 | convert => { 38 | "syslog_pid" => "integer" 39 | } 40 | } 41 | 42 | # postfix details 43 | if [syslog_program] =~ /^postfix.*\/smtpd$/ { 44 | grok { 45 | patterns_dir => "/etc/logstash/patterns" 46 | match => { "syslog_message" => "%{POSTFIX_SMTPD}" } 47 | } 48 | } else if [syslog_program] == "postfix/smtp" 49 | or [syslog_program] == "postfix/lmtp" 50 | or [syslog_program] == "postfix/local" { 51 | grok { 52 | patterns_dir => "/etc/logstash/patterns" 53 | match => { "syslog_message" => "%{POSTFIX_SMTP}" } 54 | } 55 | } else if [syslog_program] == "postfix/bounce" { 56 | grok { 57 | patterns_dir => "/etc/logstash/patterns" 58 | match => { "syslog_message" => "%{POSTFIX_BOUNCE}" } 59 | } 60 | } else if [syslog_program] == "postfix/qmgr" { 61 | grok { 62 | patterns_dir => "/etc/logstash/patterns" 63 | match => { "syslog_message" => "%{POSTFIX_QMGR}" } 64 | } 65 | } else if [syslog_program] == "postfix/anvil" { 66 | grok { 67 | patterns_dir => "/etc/logstash/patterns" 68 | match => { "syslog_message" => "%{POSTFIX_ANVIL}" } 69 | } 70 | } else if [syslog_program] == "postfix/cleanup" { 71 | grok { 72 | patterns_dir => "/etc/logstash/patterns" 73 | match => { "syslog_message" => "%{POSTFIX_CLEANUP}" } 74 | } 75 | } else if [syslog_program] == "postfix/dnsblog" { 76 | grok { 77 | patterns_dir => "/etc/logstash/patterns" 78 | match => { "syslog_message" => "%{POSTFIX_DNSBLOG}" } 79 | } 80 | } else if [syslog_program] == "postfix/master" { 81 | grok { 82 | patterns_dir => "/etc/logstash/patterns" 83 | match => { "syslog_message" => "%{POSTFIX_MASTER}" } 84 | } 85 | } else if [syslog_program] == "postfix/pickup" { 86 | grok { 87 | patterns_dir => "/etc/logstash/patterns" 88 | match => { "syslog_message" => "%{POSTFIX_PICKUP}" } 89 | } 90 | } else if [syslog_program] == "postfix/pipe" { 91 | grok { 92 | patterns_dir => "/etc/logstash/patterns" 93 | match => { "syslog_message" => "%{POSTFIX_PIPE}" } 94 | } 95 | } else if [syslog_program] == "postfix/postdrop" { 96 | grok { 97 | patterns_dir => "/etc/logstash/patterns" 98 | match => { "syslog_message" => "%{POSTFIX_POSTDROP}" } 99 | } 100 | } else if [syslog_program] == "postfix/postscreen" { 101 | grok { 102 | patterns_dir => "/etc/logstash/patterns" 103 | match => { "syslog_message" => "%{POSTFIX_POSTSCREEN}" } 104 | } 105 | } else if [syslog_program] == "postfix/sendmail" { 106 | grok { 107 | patterns_dir => "/etc/logstash/patterns" 108 | match => { "syslog_message" => "%{POSTFIX_SENDMAIL}" } 109 | } 110 | } else if [syslog_program] == "postfix/tlsmgr" { 111 | grok { 112 | patterns_dir => "/etc/logstash/patterns" 113 | match => { "syslog_message" => "%{POSTFIX_TLSMGR}" } 114 | } 115 | } else if [syslog_program] == "postfix/tlsproxy" { 116 | grok { 117 | patterns_dir => "/etc/logstash/patterns" 118 | match => { "syslog_message" => "%{POSTFIX_TLSPROXY}" } 119 | } 120 | } else if [syslog_program] == "postfix/trivial-rewrite" { 121 | grok { 122 | patterns_dir => "/etc/logstash/patterns" 123 | match => { "syslog_message" => "%{POSTFIX_TRIVIAL_REWRITE}" } 124 | } 125 | } 126 | 127 | if [syslog_program] =~ /^postfix.*\// { 128 | mutate { 129 | replace => { "log_type" => "postfix" } 130 | } 131 | # process key-value data is it exists 132 | if [postfix_keyvalue_data] { 133 | kv { 134 | source => "postfix_keyvalue_data" 135 | include_keys => [ "act", "addr", "class", "client", "conn_use", "delay", "delays", "delim", "depth", "dest", "dest_label", "dest_prop", "domain", "dsn", "dsn_orig", "dsn_ret", "encoding", "endp", "endp_label", "endp_prop", "envid", "family", "fd", "features", "filter", "flags", "from", "func", "helo", "host", "id", "ident", "key", "map", "message-id", "name", "notif", "nrcpt", "num", "off", "orig_id", "orig_to", "pattern", "port", "probed", "prop", "proto", "queue", "rcpt", "realm", "reply", "resent-message-id", "relay", "result", "ret", "score", "sender", "server_flags", "service", "size", "sock", "stat", "status", "stream", "subject", "table", "text", "to", "transp", "uid", "updated", "used", "val", "value", "verify", "weight", "why" ] 136 | remove_char_value => "<>," 137 | #trim_value => "<>," 138 | #trim => "<>," 139 | prefix => "postfix_" 140 | remove_field => [ "postfix_keyvalue_data" ] 141 | } 142 | # some post processing of key-value data 143 | if [postfix_client] { 144 | grok { 145 | patterns_dir => "/etc/logstash/patterns" 146 | match => { "postfix_client" => "%{POSTFIX_CLIENT_INFO}" } 147 | tag_on_failure => [ "_grok_kv_postfix_client_nomatch" ] 148 | remove_field => [ "postfix_client" ] 149 | } 150 | } 151 | if [postfix_relay] { 152 | grok { 153 | patterns_dir => "/etc/logstash/patterns" 154 | match => { "postfix_relay" => "%{POSTFIX_RELAY_INFO}" } 155 | tag_on_failure => [ "_grok_kv_postfix_relay_nomatch" ] 156 | remove_field => [ "postfix_relay" ] 157 | } 158 | } 159 | if [postfix_delays] { 160 | grok { 161 | patterns_dir => "/etc/logstash/patterns" 162 | match => { "postfix_delays" => "%{POSTFIX_DELAYS}" } 163 | tag_on_failure => [ "_grok_kv_postfix_delays_nomatch" ] 164 | remove_field => [ "postfix_delays" ] 165 | } 166 | } 167 | if [postfix_from] == "" { 168 | mutate { 169 | replace => { "postfix_from" => "NULL" } 170 | } 171 | } 172 | } 173 | # Do some key-value data type conversions & check grok data type conversion bug??? 174 | mutate { 175 | convert => { 176 | "postfix_nrcpt" => "integer" 177 | "postfix_size" => "integer" 178 | "postfix_uid" => "integer" 179 | "postfix_client_port" => "integer" 180 | "postfix_relay_port" => "integer" 181 | "postfix_server_port" => "integer" 182 | "postfix_status_code" => "integer" 183 | "postfix_postscreen_dnsbl_rank" => "integer" 184 | "postfix_postscreen_cache_retained" => "integer" 185 | "postfix_postscreen_cache_dropped" => "integer" 186 | "postfix_postscreen_violation_time" => "integer" 187 | "postfix_anvil_conn_rate" => "integer" 188 | "postfix_anvil_cache_size" => "integer" 189 | "postfix_anvil_conn_count" => "integer" 190 | "postfix_termination_signal" => "integer" 191 | "postfix_scache_hits" => "integer" 192 | "postfix_scache_miss" => "integer" 193 | "postfix_scache_success" => "integer" 194 | "postfix_scache_domains" => "integer" 195 | "postfix_scache_addresses" => "integer" 196 | "postfix_scache_connection" => "integer" 197 | "postfix_delay" => "float" 198 | "postfix_delay_before_qmgr" => "float" 199 | "postfix_delay_in_qmgr" => "float" 200 | "postfix_delay_conn_setup" => "float" 201 | "postfix_delay_transmission" => "float" 202 | } 203 | } 204 | #if [postfix_client_ip] and [postfix_client_ip] != "unknown" and [postfix_client_ip] !~ /^127\.0\./ { 205 | if [postfix_client_ip] and [postfix_client_ip] != "unknown" { 206 | # testing private plugin : https://github.com/nxhack/logstash-filter-geoasn 207 | #geoasn { 208 | # database => "/etc/logstash/geoip/GeoIPASNum.dat" 209 | # source => "postfix_client_ip" 210 | # target => "postfix_bgp" 211 | #} 212 | # version 4.2.1 213 | geoip { 214 | database => "/etc/logstash/geoip/GeoLite2-ASN.mmdb" 215 | source => "postfix_client_ip" 216 | target => "postfix_bgp" 217 | } 218 | geoip { 219 | #database => "/etc/logstash/geoip/GeoLiteCity.dat" 220 | database => "/etc/logstash/geoip/GeoLite2-City.mmdb" 221 | source => "postfix_client_ip" 222 | target => "postfix_geoip" 223 | } 224 | # AWS us-east-1 : Virginia 225 | if [postfix_client_ip] =~ /^10\./ { 226 | mutate { replace => { "[postfix_geoip][timezone]" => "America/New_York" } } 227 | mutate { replace => { "[postfix_geoip][country_name]" => "United States" } } 228 | mutate { replace => { "[postfix_geoip][country_code2]" => "US" } } 229 | mutate { replace => { "[postfix_geoip][country_code3]" => "USA" } } 230 | mutate { replace => { "[postfix_geoip][area_code]" => "703" } } 231 | mutate { convert => { "[postfix_geoip][area_code]" => "integer" } } 232 | mutate { replace => { "[postfix_geoip][city_name]" => "Ashburn" } } 233 | mutate { replace => { "[postfix_geoip][continent_code]" => "NA" } } 234 | mutate { replace => { "[postfix_geoip][dma_code]" => "511" } } 235 | mutate { convert => { "[postfix_geoip][dma_code]" => "integer" } } 236 | mutate { replace => { "[postfix_geoip][ip]" => "%{postfix_client_ip}" } } 237 | mutate { replace => { "[postfix_geoip][postal_code]" => "20147" } } 238 | mutate { replace => { "[postfix_geoip][real_region_name]" => "Virginia" } } 239 | mutate { replace => { "[postfix_geoip][region_name]" => "VA" } } 240 | mutate { replace => { "[postfix_geoip][latitude]" => "39.044" } } 241 | mutate { convert => { "[postfix_geoip][latitude]" => "float" } } 242 | mutate { replace => { "[postfix_geoip][longitude]" => "-77.4875" } } 243 | mutate { convert => { "[postfix_geoip][longitude]" => "float" } } 244 | mutate { replace => { "[postfix_geoip][location]" => "-77.4875, 39.044" } } 245 | } 246 | } 247 | # in case of unknown[unknown] issue 248 | if [postfix_client_ip] == "unknown" { 249 | mutate { 250 | remove_field => [ "postfix_client_ip" ] 251 | } 252 | } 253 | } 254 | 255 | if [syslog_program] == "sshd" { 256 | grok { 257 | patterns_dir => "/etc/logstash/patterns" 258 | match => { "syslog_message" => "%{SSHD_LOG}" } 259 | } 260 | # for check grok data type conversion bug??? 261 | mutate { 262 | convert => { 263 | "sshd_signal" => "integer" 264 | "sshd_port" => "integer" 265 | "sshd_listen_port" => "integer" 266 | } 267 | } 268 | if [sshd_client_ip] and [sshd_client_ip] != "unknown" { 269 | # testing private plugin : https://github.com/nxhack/logstash-filter-geoasn 270 | #geoasn { 271 | # database => "/etc/logstash/geoip/GeoIPASNum.dat" 272 | # source => "sshd_client_ip" 273 | # target => "sshd_bgp" 274 | #} 275 | geoip { 276 | database => "/etc/logstash/geoip/GeoLite2-ASN.mmdb" 277 | source => "sshd_client_ip" 278 | target => "sshd_bgp" 279 | } 280 | geoip { 281 | #database => "/etc/logstash/geoip/GeoLiteCity.dat" 282 | database => "/etc/logstash/geoip/GeoLite2-City.mmdb" 283 | source => "sshd_client_ip" 284 | target => "sshd_geoip" 285 | } 286 | # AWS us-east-1 : Virginia 287 | if [sshd_client_ip] =~ /^10\./ { 288 | mutate { replace => { "[sshd_geoip][timezone]" => "America/New_York" } } 289 | mutate { replace => { "[sshd_geoip][country_name]" => "United States" } } 290 | mutate { replace => { "[sshd_geoip][country_code2]" => "US" } } 291 | mutate { replace => { "[sshd_geoip][country_code3]" => "USA" } } 292 | mutate { replace => { "[sshd_geoip][area_code]" => "703" } } 293 | mutate { convert => { "[sshd_geoip][area_code]" => "integer" } } 294 | mutate { replace => { "[sshd_geoip][city_name]" => "Ashburn" } } 295 | mutate { replace => { "[sshd_geoip][continent_code]" => "NA" } } 296 | mutate { replace => { "[sshd_geoip][dma_code]" => "511" } } 297 | mutate { convert => { "[sshd_geoip][dma_code]" => "integer" } } 298 | mutate { replace => { "[sshd_geoip][ip]" => "%{sshd_client_ip}" } } 299 | mutate { replace => { "[sshd_geoip][postal_code]" => "20147" } } 300 | mutate { replace => { "[sshd_geoip][real_region_name]" => "Virginia" } } 301 | mutate { replace => { "[sshd_geoip][region_name]" => "VA" } } 302 | mutate { replace => { "[sshd_geoip][latitude]" => "39.044" } } 303 | mutate { convert => { "[sshd_geoip][latitude]" => "float" } } 304 | mutate { replace => { "[sshd_geoip][longitude]" => "-77.4875" } } 305 | mutate { convert => { "[sshd_geoip][longitude]" => "float" } } 306 | mutate { replace => { "[sshd_geoip][location]" => "-77.4875, 39.044" } } 307 | } 308 | if ![sshd_client_hostname] { 309 | mutate { 310 | add_field => { "sshd_client_hostname" => "%{sshd_client_ip}" } 311 | } 312 | dns { 313 | reverse => [ "sshd_client_hostname" ] 314 | action => "replace" 315 | hit_cache_size => 8000 316 | hit_cache_ttl => 300 317 | failed_cache_size => 1000 318 | failed_cache_ttl => 300 319 | } 320 | if [sshd_client_ip] =~ /:/ { 321 | mutate { 322 | add_field => { "sshd_client_ipv6" => "%{sshd_client_ip}" } 323 | } 324 | } 325 | } 326 | } 327 | } 328 | 329 | # auth.log 330 | if [syslog_program] == "saslauthd" { 331 | grok { 332 | patterns_dir => "/etc/logstash/patterns" 333 | match => { "syslog_message" => "%{SASLAUTHD}" } 334 | } 335 | } 336 | } 337 | 338 | if [log_type] == "apache" or [log_type] == "apache-other-vhost" or [log_type] == "varnish" { 339 | if [log_type] == "apache-other-vhost" { 340 | grok { 341 | patterns_dir => "/etc/logstash/patterns" 342 | match => { "message" => "%{APACHE_OTHER_VHOST_EXT}" } 343 | } 344 | } else { 345 | grok { 346 | patterns_dir => "/etc/logstash/patterns" 347 | match => { "message" => "%{APACHE_EXT_COMBINED}" } 348 | } 349 | } 350 | date { 351 | match => [ "apache_timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ] 352 | timezone => "Asia/Tokyo" 353 | locale => "en" 354 | } 355 | mutate { 356 | replace => { "apache_timestamp" => "%{@timestamp}" } 357 | } 358 | # Do some key-value data type conversions 359 | mutate { 360 | convert => { 361 | "apache_response" => "integer" 362 | "apache_bytes" => "integer" 363 | "apache_responsetime" => "integer" 364 | "apache_vhost_port" => "integer" 365 | } 366 | } 367 | if [apache_client_ip] and [apache_client_ip] != "unknown" { 368 | # testing private plugin : https://github.com/nxhack/logstash-filter-geoasn 369 | #geoasn { 370 | # database => "/etc/logstash/geoip/GeoIPASNum.dat" 371 | # source => "apache_client_ip" 372 | # target => "apache_bgp" 373 | #} 374 | geoip { 375 | database => "/etc/logstash/geoip/GeoLite2-ASN.mmdb" 376 | source => "apache_client_ip" 377 | target => "apache_bgp" 378 | } 379 | geoip { 380 | #database => "/etc/logstash/geoip/GeoLiteCity.dat" 381 | database => "/etc/logstash/geoip/GeoLite2-City.mmdb" 382 | source => "apache_client_ip" 383 | target => "apache_geoip" 384 | } 385 | # AWS us-east-1 : Virginia 386 | if [apache_client_ip] =~ /^10\./ { 387 | mutate { replace => { "[apache_geoip][timezone]" => "America/New_York" } } 388 | mutate { replace => { "[apache_geoip][country_name]" => "United States" } } 389 | mutate { replace => { "[apache_geoip][country_code2]" => "US" } } 390 | mutate { replace => { "[apache_geoip][country_code3]" => "USA" } } 391 | mutate { replace => { "[apache_geoip][area_code]" => "703" } } 392 | mutate { convert => { "[apache_geoip][area_code]" => "integer" } } 393 | mutate { replace => { "[apache_geoip][city_name]" => "Ashburn" } } 394 | mutate { replace => { "[apache_geoip][continent_code]" => "NA" } } 395 | mutate { replace => { "[apache_geoip][dma_code]" => "511" } } 396 | mutate { convert => { "[apache_geoip][dma_code]" => "integer" } } 397 | mutate { replace => { "[apache_geoip][ip]" => "%{apache_client_ip}" } } 398 | mutate { replace => { "[apache_geoip][postal_code]" => "20147" } } 399 | mutate { replace => { "[apache_geoip][real_region_name]" => "Virginia" } } 400 | mutate { replace => { "[apache_geoip][region_name]" => "VA" } } 401 | mutate { replace => { "[apache_geoip][latitude]" => "39.044" } } 402 | mutate { convert => { "[apache_geoip][latitude]" => "float" } } 403 | mutate { replace => { "[apache_geoip][longitude]" => "-77.4875" } } 404 | mutate { convert => { "[apache_geoip][longitude]" => "float" } } 405 | mutate { replace => { "[apache_geoip][location]" => "-77.4875, 39.044" } } 406 | } 407 | mutate { 408 | add_field => { "apache_client_hostname" => "%{apache_client_ip}" } 409 | } 410 | dns { 411 | reverse => [ "apache_client_hostname" ] 412 | action => "replace" 413 | hit_cache_size => 8000 414 | hit_cache_ttl => 300 415 | failed_cache_size => 1000 416 | failed_cache_ttl => 300 417 | } 418 | if [apache_client_ip] =~ /:/ { 419 | mutate { 420 | add_field => { "apache_client_ipv6" => "%{apache_client_ip}" } 421 | } 422 | } 423 | } 424 | if [apache_agent] { 425 | useragent { 426 | source => "apache_agent" 427 | target => "apache_useragent" 428 | regexes => "/etc/logstash/regexes.yaml" 429 | } 430 | ruby { 431 | code => " 432 | bot_hash = { 433 | 'libwww\-perl' => 'libwww-perl', 434 | 'Wget' => 'Wget', 435 | '^\"\-\"$' => 'MINUS', 436 | 'Feedbin' => 'Feedbin', 437 | 'Googlebot' => 'Googlebot', 438 | 'AhrefsBot' => 'AhrefsBot', 439 | 'bingbot' => 'bingbot', 440 | 'FreshReader' => 'FreshReader', 441 | '360Spider' => '360Spider', 442 | 'Windows\-RSS\-Platform' => 'Windows-RSS-Platform', 443 | 'Baiduspider' => 'Baiduspider', 444 | 'BaiduSpider' => 'Baiduspider', 445 | 'Feedeen' => 'Feedeen', 446 | 'Tiny\ Tiny\ RSS' => 'Tiny_Tiny_RSS', 447 | 'Feedly' => 'Feedly', 448 | 'WordPress' => 'WordPress', 449 | 'linkdexbot' => 'linkdexbot', 450 | 'Yahoo' => 'Yahoo', 451 | 'SemrushBot' => 'SemrushBot', 452 | 'Bumble\ Bee' => 'Bumble_Bee', 453 | 'Mail\.RU_Bot' => 'Mail.RU_Bot', 454 | 'BLEXBot' => 'BLEXBot', 455 | 'everrss' => 'everrss', 456 | 'MJ12bot' => 'MJ12bot', 457 | 'Media\ Center\ PC' => 'Media_Center_PC', 458 | 'TurnitinBot' => 'TurnitinBot', 459 | 'livedoor\ FeedFetcher' => 'livedoor_FeedFetcher', 460 | 'DotBot' => 'DotBot', 461 | 'Superfeedr\ bot' => 'Superfeedr_bot', 462 | '^\"robots\"$' => 'robots', 463 | 'Digg\ Feed\ Fetcher' => 'Digg_Feed_Fetcher', 464 | 'python\-requests' => 'python-requests', 465 | 'Y!J\-' => 'Yahoo!_Japan_Bot', 466 | 'Google\ favicon' => 'Google_favicon', 467 | 'NerdyBot' => 'NerdyBot', 468 | 'spbot' => 'spbot', 469 | 'Yandex' => 'Yandex', 470 | 'magpie\-crawler' => 'magpie-crawler', 471 | 'meanpathbot' => 'meanpathbot', 472 | 'com\.apple\.Safari\.SearchHelper' => 'com.apple.Safari.SearchHelper', 473 | 'AlexaToolbar' => 'AlexaToolbar', 474 | '^\"iPhone\"$' => 'iPhone_Fake_google', 475 | 'SurveyBot' => 'SurveyBot', 476 | 'com\.apple\.WebKit\.WebContent' => 'com.apple.WebKit.WebContent', 477 | 'curl' => 'curl', 478 | 'Nutch' => 'Nutch', 479 | 'Faraday' => 'Faraday', 480 | 'archive\.org_bot' => 'archive.org_bot', 481 | 'BingPreview' => 'BingPreview', 482 | 'Steeler' => 'Steeler', 483 | 'Python\-urllib' => 'Python-urllib', 484 | 'Python\-httplib' => 'Python-httplib', 485 | 'CCBot' => 'CCBot', 486 | 'Mechanize' => 'Mechanize', 487 | 'Sage\+\+' => 'Sage++', 488 | 'user_agent' => 'Program_Miss', 489 | 'special_archiver' => 'special_archiver', 490 | 'Cloud\ mapping\ experiment' => 'Cloud_mapping_experiment', 491 | 'Exabot' => 'Exabot', 492 | 'SiteExplorer' => 'SiteExplorer', 493 | 'Twitterbot' => 'Twitterbot', 494 | 'Lipperhey\-Kaus\-Australis' => 'Lipperhey-Kaus-Australis', 495 | 'x00_\-gawa\.sa\.pilipinas' => 'x00_-gawa.sa.pilipinas', 496 | 'DuckDuckGo\-Favicons\-Bot' => 'DuckDuckGo-Favicons-Bot', 497 | 'ips\-agent' => 'ips-agent', 498 | 'RSSingBot' => 'RSSingBot', 499 | 'Daum\/' => 'Daum', 500 | 'Daumoa' => 'Daumoa', 501 | 'Synapse' => 'Synapse', 502 | 'ia_archiver' => 'ia_archiver', 503 | 'aiHitBot' => 'aiHitBot', 504 | 'BDCbot' => 'BDCbot', 505 | 'CRAZYWEBCRAWLER' => 'CRAZYWEBCRAWLER', 506 | 'FlipboardRSS' => 'FlipboardRSS', 507 | 'HTTP_Request2' => 'HTTP_Request2', 508 | '^\"\"$' => 'NULL', 509 | '^\"Mozilla\"$' => 'Mozilla_Fake_google', 510 | '^\"Mozilla/\d\.\d\"$' => 'Mozilla_Fake_CN', 511 | '^\"\"Mozilla' => 'Mis_Mozilla', 512 | '^\"\\\"Mozilla' => 'Mis2_Mozilla', 513 | '^\"=Mozilla' => 'Miss_Mozilla', 514 | '^\"Hatena' => 'Hatena', 515 | 'Java' => 'Java', 516 | 'CakePHP' => 'CakePHP', 517 | 'Apache\-HttpClient' => 'Apache-HttpClient', 518 | 'lwp\-trivial' => 'lwp-trivial', 519 | '^\"PHP' => 'PHP', 520 | '^\"gecko' => 'gecko', 521 | '^\"redback' => 'redback', 522 | 'WinHttp\.WinHttpRequest' => 'WinHttp.WinHttpRequest', 523 | 'Jakarta\ Commons\-HttpClient' => 'Jakarta_Commons-HttpClient', 524 | 'Google\-Site\-Verification' => 'Google-Site-Verification', 525 | 'Google\ Favicon' => 'Google_Favicon', 526 | 'GoogleImageProxy' => 'GoogleImageProxy', 527 | 'google\.com\/\+\/web\/snippet\/' => 'GooglePlusBot', 528 | 'AppEngine\-Google' => 'AppEngine-Google', 529 | 'AdsBot\-Google' => 'AdsBot-Google', 530 | 'Slackbot\-LinkExpanding' => 'Slackbot-LinkExpanding', 531 | 'MS\ Search\ \d\.\d\ Robot' => 'MS_Search_Robot', 532 | 'Nmap\ Scripting\ Engine' => 'Nmap_Scripting_Engine', 533 | '007ac9\ Crawler' => '007ac9_Crawler', 534 | 'Accoona\-AI\-Agent' => 'Accoona-AI-Agent', 535 | 'Acidbot3' => 'Acidbot3', 536 | 'AdnormCrawler' => 'AdnormCrawler', 537 | 'Ahrefs\-Bot' => 'Ahrefs-Bot', 538 | 'Applebot' => 'Applebot', 539 | 'BCKLINKS' => 'BCKLINKS', 540 | 'BOT\ for\ JCE' => 'BOT_for_JCE', 541 | 'BPImageWalker' => 'BPImageWalker', 542 | 'BellPagesCA' => 'BellPagesCA', 543 | 'BlackCrawler' => 'BlackCrawler', 544 | 'Blackboard\ Safeassign' => 'Blackboard_Safeassign', 545 | 'Blekkobot' => 'Blekkobot', 546 | 'BoardReader\ Blog\ Indexer' => 'BoardReader_Blog_Indexer', 547 | 'BusinessBot' => 'BusinessBot', 548 | 'Buzzbot' => 'Buzzbot', 549 | 'CMS\ Crawler' => 'CMS_Crawler', 550 | 'CatchBot' => 'CatchBot', 551 | 'CheckMarkNetwork' => 'CheckMarkNetwork', 552 | 'Chilkat' => 'Chilkat', 553 | 'Claritybot' => 'Claritybot', 554 | 'Cliqzbot' => 'Cliqzbot', 555 | 'ConBot' => 'ConBot', 556 | 'DataparkSearch' => 'DataparkSearch', 557 | 'Dataprovider' => 'Dataprovider', 558 | 'Daum\ 4' => 'Daum', 559 | 'DialogSearch\.com\ Bot' => 'DialogSearch.com_Bot', 560 | 'Digincore\ bot' => 'Digincore_bot', 561 | 'Digincore\ crawler\ bot' => 'Digincore_crawler_bot', 562 | 'DomainAppender' => 'DomainAppender', 563 | 'DomainCrawler' => 'DomainCrawler', 564 | 'DomainMacroCrawler' => 'DomainMacroCrawler', 565 | 'DomainSigmaCrawler' => 'DomainSigmaCrawler', 566 | 'ENVCheck' => 'ENVCheck_envc.dip.jp', 567 | 'Embedly' => 'Embedly', 568 | 'Experibot\-v3' => 'Experibot-v3', 569 | 'Experibot_v1' => 'Experibot_v1', 570 | 'Exploratodo' => 'Exploratodo', 571 | 'ExploreDoc\ download\ bot' => 'ExploreDoc_download_bot', 572 | 'ExtLinksBot' => 'ExtLinksBot', 573 | 'FAST\-WebCrawler' => 'FAST-WebCrawler', 574 | 'Feedspot' => 'Feedspot', 575 | 'Findxbot' => 'Findxbot', 576 | 'FlipboardProxy' => 'FlipboardProxy', 577 | 'GOFORITBOT' => 'GOFORITBOT', 578 | 'GalaxyBot' => 'GalaxyBot', 579 | 'GarlikCrawler' => 'GarlikCrawler', 580 | 'GetURLInfo' => 'GetURLInfo', 581 | 'GigablastOpenSource' => 'GigablastOpenSource', 582 | 'Gigabot' => 'Gigabot', 583 | 'GimmeUSAbot' => 'GimmeUSAbot', 584 | 'Girafabot' => 'Girafabot', 585 | 'Gluten\ Free\ Crawler' => 'Gluten_Free_Crawler', 586 | 'Go\ \d\.\d\ package\ http' => 'Go_package_http', 587 | 'GurujiBot' => 'GurujiBot', 588 | 'HRCrawlerBot' => 'HRCrawlerBot', 589 | 'Hatena\ Pagetitle\ Agent' => 'Hatena_Pagetitle_Agent', 590 | 'Hatena\ Star\ UserAgent' => 'Hatena_Star_UserAgent', 591 | 'HeartRails_Capture' => 'HeartRails_Capture', 592 | 'Hivemind' => 'Hivemind', 593 | 'HubSpot\ Links\ Crawler' => 'HubSpot_Links_Crawler', 594 | 'HyperCrawl' => 'HyperCrawl', 595 | 'IABTechLab' => 'IABTechLab', 596 | 'ICC\-Crawler' => 'ICC-Crawler', 597 | 'IRLbot' => 'IRLbot', 598 | 'ImplisenseBot' => 'ImplisenseBot', 599 | 'IndeedBot' => 'IndeedBot', 600 | 'Indy\ Library' => 'Indy_Library', 601 | 'InetURL' => 'InetURL', 602 | 'Internet\-wide\-scan\-to\-be\-removed\-from\-this\-list\-email\-info\-at\-binaryedge\.io' => 'binaryedge', 603 | 'IssueCrawler' => 'IssueCrawler', 604 | 'Jorgee' => 'Jorgee', 605 | 'KickFire' => 'KickFire', 606 | 'KoepaBot' => 'KoepaBot', 607 | 'Kyoto\-Crawler' => 'Kyoto-Crawler', 608 | 'Kyoto\-Tohoku\-Crawler' => 'Kyoto-Tohoku-Crawler', 609 | 'LSSRocketCrawler' => 'LSSRocketCrawler', 610 | 'Linguee\ Bot' => 'Linguee_Bot', 611 | 'LinkWalker' => 'LinkWalker', 612 | 'Link\ Valet\ Online' => 'Link_Valet_Online', 613 | 'LinkpadBot' => 'LinkpadBot', 614 | 'LinksManager\.com_bot' => 'LinksManager.com_bot', 615 | 'MSIECrawler' => 'MSIECrawler', 616 | 'MauiBot' => 'MauiBot', 617 | 'MegaIndex\.ru' => 'MegaIndex.ru', 618 | 'Microsearch\.ru\ Bot' => 'Microsearch.ru_Bot', 619 | 'Microsoft\ URL\ Control' => 'Microsoft_URL_Control', 620 | 'Microsoft\-WebDAV\-MiniRedir' => 'Microsoft-WebDAV-MiniRedir', 621 | 'MixrankBot' => 'MixrankBot', 622 | 'Mnogosearch' => 'Mnogosearch', 623 | 'MojeekBot' => 'MojeekBot', 624 | 'Mojoo\ Robot' => 'Mojoo_Robot', 625 | 'Morfeus\ Fucking\ Scanner' => 'Morfeus_Fucking_Scanner', 626 | 'Morfeus\ strikes\ again' => 'Morfeus_strikes_again', 627 | 'NG\-Search' => 'NG-Search', 628 | 'NetResearchServer' => 'NetResearchServer', 629 | 'NetcraftSurveyAgent' => 'NetcraftSurveyAgent', 630 | 'Netcraft\ Web\ Server\ Survey' => 'Netcraft_Web_Server_Survey', 631 | 'Netseer\ crawler' => 'NetSeer_crawler', 632 | 'NewsGator' => 'NewsGator', 633 | 'Newsify' => 'Newsify', 634 | 'Nimbostratus\-Bot' => 'Nimbostratus-Bot', 635 | 'OmniExplorer_Bot' => 'OmniExplorer_Bot', 636 | 'OptimizationCrawler' => 'OptimizationCrawler', 637 | 'PageAnalyzer' => 'PageAnalyzer', 638 | 'PagesInventory' => 'PagesInventory', 639 | 'PaperLiBot' => 'PaperLiBot', 640 | 'PiplBot' => 'PiplBot', 641 | 'Plukkie' => 'Plukkie', 642 | 'PostPost' => 'PostPost', 643 | 'PrivacyAwareBot' => 'PrivacyAwareBot', 644 | 'Project\ 25499' => 'Project_25499', 645 | 'Qwantify' => 'Qwantify', 646 | 'R6_CommentReader' => 'R6_CommentReader', 647 | 'R6_FeedFetcher' => 'R6_FeedFetcher', 648 | 'RankActiveLinkBot' => 'RankActiveLinkBot', 649 | 'Readability' => 'Readability', 650 | 'Research\ scan' => 'Research_scan', 651 | 'Riddler' => 'Riddler', 652 | 'SBL\-BOT' => 'SBL-BOT', 653 | 'SEMrushBot' => 'SEMrushBot', 654 | 'SEOkicks\-Robot' => 'SEOkicks-Robot', 655 | 'SMTBot' => 'SMTBot', 656 | 'SOLOFIELD' => 'SOLOFIELD', 657 | 'SafeDNSBot' => 'SafeDNSBot', 658 | 'SalesIntelligent' => 'SalesIntelligent', 659 | 'Scanbot' => 'Scanbot', 660 | 'Scanning\ for\ research' => 'researchscan', 661 | 'Scopia\ Crawler' => 'Scopia_Crawler', 662 | 'ScoutJet' => 'ScoutJet', 663 | 'Scrapy' => 'Scrapy', 664 | 'Screaming\ Frog\ SEO\ Spider' => 'Screaming_Frog_SEO_Spider', 665 | 'ScreenerBot' => 'ScreenerBot', 666 | 'ScumScanner' => 'ScumScanner', 667 | 'SemrushBot' => 'SemrushBot', 668 | 'SeznamBot' => 'SeznamBot', 669 | 'Shim\-Crawler' => 'Shim-Crawler', 670 | 'ShortLinkTranslate' => 'ShortLinkTranslate', 671 | 'SlideGur' => 'SlideGur', 672 | 'SocialRankIOBot' => 'SocialRankIOBot', 673 | 'Sogou\ Pic\ Spider' => 'Sogou_Pic_Spider', 674 | 'Sogou\ web\ spider' => 'Sogou_web_spider', 675 | 'Sonic\/1' => 'YamanaLab-Robot', 676 | 'Speedy\ Spider' => 'Speedy_Spider', 677 | 'SpiderLing' => 'SpiderLing', 678 | 'SputnikBot' => 'SputnikBot', 679 | 'Stratagems\ Kumo' => 'Stratagems_Kumo', 680 | 'SurdotlyBot' => 'SurdotlyBot', 681 | 'Telesphoreo' => 'Telesphoreo', 682 | 'Telesphorep' => 'Telesphorep', 683 | 'TestCrawler' => 'TestCrawler', 684 | 'TheSuBot' => 'TheSuBot', 685 | 'Thumbnail\.CZ\ robot' => 'Thumbnail.CZ_robot', 686 | 'TopServer\ PHP' => 'TopServer_PHP', 687 | 'Toweyabot' => 'Toweyabot', 688 | 'UXCrawlerBot' => 'UXCrawlerBot', 689 | 'Unknown\/X-Agent' => 'Unknown_X-Agent', 690 | 'Uptimebot' => 'Uptimebot', 691 | 'Vagabondo' => 'Vagabondo', 692 | 'Virusdie\ crawler' => 'Virusdie_crawler', 693 | 'W3C_Validator' => 'W3C_Validator', 694 | 'WBSearchBot' => 'WBSearchBot', 695 | 'WWWC' => 'WWWC', 696 | 'WWW\-Mechanize' => 'WWW-Mechanize', 697 | 'WbSrch' => 'WbSrch', 698 | 'WeCrawlForThePeace' => 'WeCrawlForThePeace', 699 | 'WebCapture' => 'WebCapture', 700 | 'WebFuck' => 'WebFuck', 701 | 'WebIndex' => 'WebIndex', 702 | 'WebTarantula\.com' => 'WebTarantula.com', 703 | 'Webcrawler' => 'Webcrawler', 704 | 'Who\.is\ Bot' => 'Who.is_Bot', 705 | 'Wotbox' => 'Wotbox', 706 | 'Xenu\ Link\ Sleuth' => 'Xenu_Link_Sleuth', 707 | 'XoviBot' => 'XoviBot', 708 | 'YisouSpider' => 'YisouSpider', 709 | 'Zend_Http_Client' => 'Zend_Http_Client', 710 | 'ZmEu' => 'ZmEu', 711 | 'Zollard' => 'Linux.Trojan.Zollard', 712 | 'Zombiebot' => 'Zombiebot', 713 | 'ZyBorg' => 'ZyBorg', 714 | '^\"GoogleBot\ 1\.0\"$' => 'GoogleBot_Fake', 715 | '^\"GoogleBot\ 1\.1\"$' => 'GoogleBot_Fake', 716 | '^\"null\"$' => 'null', 717 | 'alsRobot2' => 'alsRobot2', 718 | 'apache\ 0day\ by' => 'apache_0day_by_@hxmonsegur', 719 | 'boitho\.com\-dc' => 'boitho.com-dc', 720 | 'bot\-pge\.chlooe\.com' => 'bot-pge.chlooe.com', 721 | 'centurybot9' => 'centurybot9', 722 | 'crawler4j' => 'crawler4j', 723 | 'facebookexternalhit' => 'facebookexternalhit', 724 | 'finbot' => 'finbot', 725 | 'findlinks' => 'findlinks', 726 | 'heritrix' => 'heritrix', 727 | 'hl_ftien_spider_v1' => 'hl_ftien_spider_v1', 728 | 'honzilla\ download\ bot' => 'honzilla_download_bot', 729 | 'iZSearch\.com' => 'iZSearch.com', 730 | 'iaskspider' => 'iaskspider', 731 | 'ichiro' => 'goo', 732 | 'inoreader\.com' => 'inoreader.com', 733 | 'izsearch\.com' => 'izsearch.com', 734 | 'linkapediabot' => 'linkapediabot', 735 | 'lmspider' => 'lmspider', 736 | 'ltbot' => 'ltbot', 737 | 'ltx71' => 'ltx71', 738 | 'mabontland\.com' => 'mabontland', 739 | 'masscan' => 'masscan', 740 | 'memoryBot' => 'memoryBot', 741 | 'mfibot' => 'mfibot', 742 | 'microdata\ crawler' => 'microdata_crawler', 743 | 'mobile\ goo' => 'mobile_goo', 744 | 'moukrest\.ru' => 'moukrest.ru', 745 | 'msnbot' => 'msnbot', 746 | 'munRobot' => 'munRobot', 747 | 'mxbot' => 'mxbot', 748 | 'netEstate\ NE\ Crawler' => 'netEstate_NE_Crawler', 749 | 'nlpproject\.info\ research' => 'nlpproject.info_research', 750 | 'oBot' => 'oBot', 751 | 'ocrawler' => 'ocrawler', 752 | 'panscient\.com' => 'panscient.com', 753 | 'probethenet\.com\ scanner' => 'probethenet.com_scanner', 754 | 'proximic' => 'proximic', 755 | 'psbot' => 'psbot', 756 | 'roboto' => 'roboto', 757 | 'rogerbot' => 'rogerbot', 758 | 'scrapy\-redis' => 'scrapy-redis', 759 | 'scrapyproject' => 'scrapyproject', 760 | 'scrutiny' => 'scrutiny', 761 | 'semanticbot' => 'semanticbot', 762 | 'seoscanners' => 'seoscanners', 763 | 'slidezz\ download\ bot' => 'slidezz_download_bot', 764 | 'spiderman' => 'spiderman', 765 | 'spyonweb' => 'spyonweb', 766 | 'voltron' => 'voltron', 767 | 'woobot' => 'woobot', 768 | 'wp\-poster' => 'wp-poster', 769 | 'yacybot' => 'yacybot', 770 | 'yoozBot' => 'yoozBot', 771 | 'zgrab' => 'zgrab', 772 | 'zspider' => 'zspider' 773 | } 774 | bot_hash.each do |bot_re, bot_name| 775 | if event.get('apache_agent') =~ /#{bot_re}/ 776 | event.set('apache_robot', bot_name) 777 | break 778 | end 779 | end 780 | " 781 | } 782 | } 783 | if [apache_request] { 784 | ruby { 785 | code => "event.set('apache_extension', File.extname(event.get('apache_request').split('?')[0]).delete('.'))" 786 | } 787 | if [apache_extension] == "" or [apache_extension] !~ /^[a-zA-Z0-9]{1,4}$/ { 788 | mutate { 789 | remove_field => [ "apache_extension" ] 790 | } 791 | } 792 | } 793 | } 794 | 795 | if [log_type] == "apache-error" { 796 | grok { 797 | patterns_dir => "/etc/logstash/patterns" 798 | match => { "message" => "%{APACHE_ERROR}" } 799 | } 800 | date { 801 | match => [ "apache_error_timestamp", "EEE MMM dd HH:mm:ss yyyy", "EEE MMM dd HH:mm:ss.SSSSSS yyyy" ] 802 | timezone => "Asia/Tokyo" 803 | locale => "en" 804 | } 805 | mutate { 806 | replace => { "apache_error_timestamp" => "%{@timestamp}" } 807 | } 808 | # Do some key-value data type conversions 809 | mutate { 810 | convert => { 811 | "apache_error_pid" => "integer" 812 | "apache_error_tid" => "integer" 813 | "apache_error_client_port" => "integer" 814 | } 815 | } 816 | if [apache_error_client_ip] and [apache_error_client_ip] != "unknown" { 817 | # testing private plugin : https://github.com/nxhack/logstash-filter-geoasn 818 | #geoasn { 819 | # database => "/etc/logstash/geoip/GeoIPASNum.dat" 820 | # source => "apache_error_client_ip" 821 | # target => "apache_error_bgp" 822 | #} 823 | geoip { 824 | database => "/etc/logstash/geoip/GeoLite2-ASN.mmdb" 825 | source => "apache_error_client_ip" 826 | target => "apache_error_bgp" 827 | } 828 | geoip { 829 | #database => "/etc/logstash/geoip/GeoLiteCity.dat" 830 | database => "/etc/logstash/geoip/GeoLite2-City.mmdb" 831 | source => "apache_error_client_ip" 832 | target => "apache_error_geoip" 833 | } 834 | # AWS us-east-1 : Virginia 835 | if [apache_error_client_ip] =~ /^10\./ { 836 | mutate { replace => { "[apache_error_geoip][timezone]" => "America/New_York" } } 837 | mutate { replace => { "[apache_error_geoip][country_name]" => "United States" } } 838 | mutate { replace => { "[apache_error_geoip][country_code2]" => "US" } } 839 | mutate { replace => { "[apache_error_geoip][country_code3]" => "USA" } } 840 | mutate { replace => { "[apache_error_geoip][area_code]" => "703" } } 841 | mutate { convert => { "[apache_error_geoip][area_code]" => "integer" } } 842 | mutate { replace => { "[apache_error_geoip][city_name]" => "Ashburn" } } 843 | mutate { replace => { "[apache_error_geoip][continent_code]" => "NA" } } 844 | mutate { replace => { "[apache_error_geoip][dma_code]" => "511" } } 845 | mutate { convert => { "[apache_error_geoip][dma_code]" => "integer" } } 846 | mutate { replace => { "[apache_error_geoip][ip]" => "%{apache_error_client_ip}" } } 847 | mutate { replace => { "[apache_error_geoip][postal_code]" => "20147" } } 848 | mutate { replace => { "[apache_error_geoip][real_region_name]" => "Virginia" } } 849 | mutate { replace => { "[apache_error_geoip][region_name]" => "VA" } } 850 | mutate { replace => { "[apache_error_geoip][latitude]" => "39.044" } } 851 | mutate { convert => { "[apache_error_geoip][latitude]" => "float" } } 852 | mutate { replace => { "[apache_error_geoip][longitude]" => "-77.4875" } } 853 | mutate { convert => { "[apache_error_geoip][longitude]" => "float" } } 854 | mutate { replace => { "[apache_error_geoip][location]" => "-77.4875, 39.044" } } 855 | } 856 | mutate { 857 | add_field => { "apache_error_client_hostname" => "%{apache_error_client_ip}" } 858 | } 859 | dns { 860 | reverse => [ "apache_error_client_hostname" ] 861 | action => "replace" 862 | hit_cache_size => 8000 863 | hit_cache_ttl => 300 864 | failed_cache_size => 1000 865 | failed_cache_ttl => 300 866 | } 867 | if [apache_error_client_ip] =~ /:/ { 868 | mutate { 869 | add_field => { "apache_error_client_ipv6" => "%{apache_error_client_ip}" } 870 | } 871 | } 872 | } 873 | } 874 | 875 | if [log_type] == "dpkg" { 876 | grok { 877 | match => { "message" => "%{TIMESTAMP_ISO8601:dpkg_timestamp} %{GREEDYDATA:dpkg_message}" } 878 | } 879 | date { 880 | match => [ "dpkg_timestamp", "YYYY-MM-dd HH:mm:ss", "ISO8601" ] 881 | timezone => "Asia/Tokyo" 882 | locale => "en" 883 | } 884 | mutate { 885 | replace => { "dpkg_timestamp" => "%{@timestamp}" } 886 | } 887 | } 888 | 889 | # zimbra mailbox.log 890 | if [log_type] == "zimbra" { 891 | grok { 892 | patterns_dir => "/etc/logstash/patterns" 893 | match => { "message" => "%{ZIMBRA_MAILBOX_LOG}" } 894 | } 895 | date { 896 | match => [ "zimbra_timestamp", "YYYY-MM-dd HH:mm:ss,SSS", "ISO8601" ] 897 | timezone => "Asia/Tokyo" 898 | locale => "en" 899 | } 900 | mutate { 901 | replace => { "zimbra_timestamp" => "%{@timestamp}" } 902 | } 903 | # for check grok data type conversion bug??? 904 | mutate { 905 | convert => { 906 | "zimbra_thread_number" => "integer" 907 | } 908 | } 909 | if [zimbra_account] { 910 | kv { 911 | source => "zimbra_account" 912 | include_keys => [ "ip", "oip", "id", "name", "aid", "aname", "cid", "mid", "ua", "via", "msgid", "item", "ds", "port" ] 913 | field_split => ";" 914 | prefix => "zimbra_account_" 915 | remove_field => [ "zimbra_account" ] 916 | } 917 | mutate { 918 | convert => { 919 | "zimbra_account_id" => "integer" 920 | "zimbra_account_aid" => "integer" 921 | "zimbra_account_cid" => "integer" 922 | "zimbra_account_mid" => "integer" 923 | "zimbra_account_port" => "integer" 924 | } 925 | } 926 | } 927 | if [zimbra_account_ip] { 928 | # testing private plugin : https://github.com/nxhack/logstash-filter-geoasn 929 | #geoasn { 930 | # database => "/etc/logstash/geoip/GeoIPASNum.dat" 931 | # source => "zimbra_account_ip" 932 | # target => "zimbra_bgp" 933 | #} 934 | geoip { 935 | database => "/etc/logstash/geoip/GeoLite2-ASN.mmdb" 936 | source => "zimbra_account_ip" 937 | target => "zimbra_bgp" 938 | } 939 | geoip { 940 | #database => "/etc/logstash/geoip/GeoLiteCity.dat" 941 | database => "/etc/logstash/geoip/GeoLite2-City.mmdb" 942 | source => "zimbra_account_ip" 943 | target => "zimbra_geoip" 944 | } 945 | # AWS us-east-1 : Virginia 946 | if [zimbra_account_ip] =~ /^10\./ { 947 | mutate { replace => { "[zimbra_geoip][timezone]" => "America/New_York" } } 948 | mutate { replace => { "[zimbra_geoip][country_name]" => "United States" } } 949 | mutate { replace => { "[zimbra_geoip][country_code2]" => "US" } } 950 | mutate { replace => { "[zimbra_geoip][country_code3]" => "USA" } } 951 | mutate { replace => { "[zimbra_geoip][area_code]" => "703" } } 952 | mutate { convert => { "[zimbra_geoip][area_code]" => "integer" } } 953 | mutate { replace => { "[zimbra_geoip][city_name]" => "Ashburn" } } 954 | mutate { replace => { "[zimbra_geoip][continent_code]" => "NA" } } 955 | mutate { replace => { "[zimbra_geoip][dma_code]" => "511" } } 956 | mutate { convert => { "[zimbra_geoip][dma_code]" => "integer" } } 957 | mutate { replace => { "[zimbra_geoip][ip]" => "%{zimbra_account_ip}" } } 958 | mutate { replace => { "[zimbra_geoip][postal_code]" => "20147" } } 959 | mutate { replace => { "[zimbra_geoip][real_region_name]" => "Virginia" } } 960 | mutate { replace => { "[zimbra_geoip][region_name]" => "VA" } } 961 | mutate { replace => { "[zimbra_geoip][latitude]" => "39.044" } } 962 | mutate { convert => { "[zimbra_geoip][latitude]" => "float" } } 963 | mutate { replace => { "[zimbra_geoip][longitude]" => "-77.4875" } } 964 | mutate { convert => { "[zimbra_geoip][longitude]" => "float" } } 965 | mutate { replace => { "[zimbra_geoip][location]" => "-77.4875, 39.044" } } 966 | } 967 | mutate { 968 | add_field => { "zimbra_account_hostname" => "%{zimbra_account_ip}" } 969 | } 970 | dns { 971 | reverse => [ "zimbra_account_hostname" ] 972 | action => "replace" 973 | hit_cache_size => 8000 974 | hit_cache_ttl => 300 975 | failed_cache_size => 1000 976 | failed_cache_ttl => 300 977 | } 978 | } 979 | if [zimbra_account_oip] { 980 | # testing private plugin : https://github.com/nxhack/logstash-filter-geoasn 981 | #geoasn { 982 | # database => "/etc/logstash/geoip/GeoIPASNum.dat" 983 | # source => "zimbra_account_oip" 984 | # target => "zimbra_origin_bgp" 985 | #} 986 | geoip { 987 | database => "/etc/logstash/geoip/GeoLite2-ASN.mmdb" 988 | source => "zimbra_account_oip" 989 | target => "zimbra_origin_bgp" 990 | } 991 | geoip { 992 | #database => "/etc/logstash/geoip/GeoLiteCity.dat" 993 | database => "/etc/logstash/geoip/GeoLite2-City.mmdb" 994 | source => "zimbra_account_oip" 995 | target => "zimbra_origin_geoip" 996 | } 997 | # AWS us-east-1 : Virginia 998 | if [zimbra_account_oip] =~ /^10\./ { 999 | mutate { replace => { "[zimbra_origin_geoip][timezone]" => "America/New_York" } } 1000 | mutate { replace => { "[zimbra_origin_geoip][country_name]" => "United States" } } 1001 | mutate { replace => { "[zimbra_origin_geoip][country_code2]" => "US" } } 1002 | mutate { replace => { "[zimbra_origin_geoip][country_code3]" => "USA" } } 1003 | mutate { replace => { "[zimbra_origin_geoip][area_code]" => "703" } } 1004 | mutate { convert => { "[zimbra_origin_geoip][area_code]" => "integer" } } 1005 | mutate { replace => { "[zimbra_origin_geoip][city_name]" => "Ashburn" } } 1006 | mutate { replace => { "[zimbra_origin_geoip][continent_code]" => "NA" } } 1007 | mutate { replace => { "[zimbra_origin_geoip][dma_code]" => "511" } } 1008 | mutate { convert => { "[zimbra_origin_geoip][dma_code]" => "integer" } } 1009 | mutate { replace => { "[zimbra_origin_geoip][ip]" => "%{zimbra_account_oip}" } } 1010 | mutate { replace => { "[zimbra_origin_geoip][postal_code]" => "20147" } } 1011 | mutate { replace => { "[zimbra_origin_geoip][real_region_name]" => "Virginia" } } 1012 | mutate { replace => { "[zimbra_origin_geoip][region_name]" => "VA" } } 1013 | mutate { replace => { "[zimbra_origin_geoip][latitude]" => "39.044" } } 1014 | mutate { convert => { "[zimbra_origin_geoip][latitude]" => "float" } } 1015 | mutate { replace => { "[zimbra_origin_geoip][longitude]" => "-77.4875" } } 1016 | mutate { convert => { "[zimbra_origin_geoip][longitude]" => "float" } } 1017 | mutate { replace => { "[zimbra_origin_geoip][location]" => "-77.4875, 39.044" } } 1018 | } 1019 | mutate { 1020 | add_field => { "zimbra_account_origin_hostname" => "%{zimbra_account_oip}" } 1021 | } 1022 | dns { 1023 | reverse => [ "zimbra_account_origin_hostname" ] 1024 | action => "replace" 1025 | hit_cache_size => 8000 1026 | hit_cache_ttl => 300 1027 | failed_cache_size => 1000 1028 | failed_cache_ttl => 300 1029 | } 1030 | } 1031 | if [zimbra_trace_log] { 1032 | mutate { 1033 | add_field => { "zimbra_component" => "JAVA-Exception" } 1034 | } 1035 | } 1036 | } 1037 | 1038 | # zimbra /var/log/zimbra.log (syslog format) use amavisd, clamd, opendkim logs only. 1039 | if [log_type] == "zimbralog" { 1040 | grok { 1041 | match => { "message" => "(?:%{SYSLOGTIMESTAMP:syslog_timestamp}|%{TIMESTAMP_ISO8601:syslog_timestamp}) %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" } 1042 | add_field => { 1043 | "syslog_received_at" => "%{@timestamp}" 1044 | "syslog_received_from" => "%{[host][name]}" 1045 | } 1046 | } 1047 | syslog_pri { 1048 | } 1049 | date { 1050 | match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ] 1051 | timezone => "Asia/Tokyo" 1052 | locale => "en" 1053 | } 1054 | mutate { 1055 | replace => { "syslog_timestamp" => "%{@timestamp}" } 1056 | } 1057 | # for check grok data type conversion bug??? 1058 | mutate { 1059 | convert => { 1060 | "syslog_pid" => "integer" 1061 | } 1062 | } 1063 | 1064 | if [syslog_program] == "amavis" { 1065 | grok { 1066 | patterns_dir => "/etc/logstash/patterns" 1067 | match => { "syslog_message" => "%{AMAVIS}" } 1068 | } 1069 | mutate { 1070 | replace => { "log_type" => "amavis" } 1071 | } 1072 | # for check grok data type conversion bug??? 1073 | mutate { 1074 | convert => { 1075 | "amavis_relay_port" => "integer" 1076 | "amavis_size" => "integer" 1077 | "amavis_delay" => "integer" 1078 | "amavis_hits" => "float" 1079 | } 1080 | } 1081 | if [amavis_relay_ip] { 1082 | # testing private plugin : https://github.com/nxhack/logstash-filter-geoasn 1083 | #geoasn { 1084 | # database => "/etc/logstash/geoip/GeoIPASNum.dat" 1085 | # source => "amavis_relay_ip" 1086 | # target => "amavis_bgp" 1087 | #} 1088 | geoip { 1089 | database => "/etc/logstash/geoip/GeoLite2-ASN.mmdb" 1090 | source => "amavis_relay_ip" 1091 | target => "amavis_bgp" 1092 | } 1093 | geoip { 1094 | #database => "/etc/logstash/geoip/GeoLiteCity.dat" 1095 | database => "/etc/logstash/geoip/GeoLite2-City.mmdb" 1096 | source => "amavis_relay_ip" 1097 | target => "amavis_geoip" 1098 | } 1099 | # AWS us-east-1 : Virginia 1100 | if [amavis_relay_ip] =~ /^10\./ { 1101 | mutate { replace => { "[amavis_geoip][timezone]" => "America/New_York" } } 1102 | mutate { replace => { "[amavis_geoip][country_name]" => "United States" } } 1103 | mutate { replace => { "[amavis_geoip][country_code2]" => "US" } } 1104 | mutate { replace => { "[amavis_geoip][country_code3]" => "USA" } } 1105 | mutate { replace => { "[amavis_geoip][area_code]" => "703" } } 1106 | mutate { convert => { "[amavis_geoip][area_code]" => "integer" } } 1107 | mutate { replace => { "[amavis_geoip][city_name]" => "Ashburn" } } 1108 | mutate { replace => { "[amavis_geoip][continent_code]" => "NA" } } 1109 | mutate { replace => { "[amavis_geoip][dma_code]" => "511" } } 1110 | mutate { convert => { "[amavis_geoip][dma_code]" => "integer" } } 1111 | mutate { replace => { "[amavis_geoip][ip]" => "%{amavis_relay_ip}" } } 1112 | mutate { replace => { "[amavis_geoip][postal_code]" => "20147" } } 1113 | mutate { replace => { "[amavis_geoip][real_region_name]" => "Virginia" } } 1114 | mutate { replace => { "[amavis_geoip][region_name]" => "VA" } } 1115 | mutate { replace => { "[amavis_geoip][latitude]" => "39.044" } } 1116 | mutate { convert => { "[amavis_geoip][latitude]" => "float" } } 1117 | mutate { replace => { "[amavis_geoip][longitude]" => "-77.4875" } } 1118 | mutate { convert => { "[amavis_geoip][longitude]" => "float" } } 1119 | mutate { replace => { "[amavis_geoip][location]" => "-77.4875, 39.044" } } 1120 | } 1121 | mutate { 1122 | add_field => { "amavis_relay_hostname" => "%{amavis_relay_ip}" } 1123 | } 1124 | dns { 1125 | reverse => [ "amavis_relay_hostname" ] 1126 | action => "replace" 1127 | hit_cache_size => 8000 1128 | hit_cache_ttl => 300 1129 | failed_cache_size => 1000 1130 | failed_cache_ttl => 300 1131 | } 1132 | } 1133 | } else if [syslog_program] == "clamd" { 1134 | mutate { 1135 | replace => { "log_type" => "clamd" } 1136 | } 1137 | } else if [syslog_program] == "opendkim" { 1138 | mutate { 1139 | replace => { "log_type" => "opendkim" } 1140 | } 1141 | } else { 1142 | drop { 1143 | } 1144 | } 1145 | } 1146 | 1147 | # Arduino YUN : Weather Station send via python beaver -> redis 1148 | if "wsdl" in [log_type] { 1149 | if [host] == "Arduino" { 1150 | grok { 1151 | match => { "message" => "%{TIMESTAMP_ISO8601:wsdl_timestamp} T1:%{BASE10NUM:wsdl_temperature1} T2:%{BASE10NUM:wsdl_temperature2} H:%{BASE10NUM:wsdl_humidity} P:%{BASE10NUM:wsdl_barometer}" } 1152 | add_field => { 1153 | "wsdl_received_at" => "%{@timestamp}" 1154 | "wsdl_received_from" => "%{host}" 1155 | } 1156 | } 1157 | } else { 1158 | grok { 1159 | match => { "message" => "%{TIMESTAMP_ISO8601:wsdl_timestamp} T:%{BASE10NUM:wsdl_temperature1} H:%{DATA:wsdl_humidity} P:%{BASE10NUM:wsdl_barometer}" } 1160 | add_field => { 1161 | "wsdl_received_at" => "%{@timestamp}" 1162 | "wsdl_received_from" => "%{host}" 1163 | } 1164 | } 1165 | mutate { 1166 | add_field => { 1167 | "wsdl_temperature2" => "%{wsdl_temperature1}" 1168 | } 1169 | } 1170 | } 1171 | date { 1172 | match => [ "esdl_timestamp", "ISO8601" ] 1173 | timezone => "Asia/Tokyo" 1174 | locale => "en" 1175 | } 1176 | # NaN check 1177 | if [wsdl_humidity] == "nan" { 1178 | mutate { 1179 | remove_field => [ "wsdl_humidity" ] 1180 | } 1181 | } 1182 | # for check grok data type conversion bug??? 1183 | mutate { 1184 | convert => { 1185 | "wsdl_temperature1" => "float" 1186 | "wsdl_temperature2" => "float" 1187 | "wsdl_humidity" => "float" 1188 | "wsdl_barometer" => "float" 1189 | } 1190 | } 1191 | # Remove beaver metadata 1192 | mutate { 1193 | rename => { "host" => "wsdl_host" } 1194 | rename => { "file" => "wsdl_file" } 1195 | } 1196 | mutate { 1197 | add_field => { "[host][name]" => "%{wsdl_host}" } 1198 | add_field => { "[log][file][path]" => "%{wsdl_file}" } 1199 | } 1200 | } 1201 | 1202 | # Fail2Ban: TEST 1203 | if [log_type] == "fail2ban" { 1204 | grok { 1205 | patterns_dir => "/etc/logstash/patterns" 1206 | match => { "message" => "%{FAIL2BAN}" } 1207 | } 1208 | date { 1209 | match => [ "fail2ban_timestamp", "ISO8601" ] 1210 | timezone => "Asia/Tokyo" 1211 | locale => "en" 1212 | } 1213 | if [fail2ban_source_ip] { 1214 | # testing private plugin : https://github.com/nxhack/logstash-filter-geoasn 1215 | #geoasn { 1216 | # database => "/etc/logstash/geoip/GeoIPASNum.dat" 1217 | # source => "fail2ban_source_ip" 1218 | # target => "fail2ban_bgp" 1219 | #} 1220 | geoip { 1221 | database => "/etc/logstash/geoip/GeoLite2-ASN.mmdb" 1222 | source => "fail2ban_source_ip" 1223 | target => "fail2ban_bgp" 1224 | } 1225 | geoip { 1226 | #database => "/etc/logstash/geoip/GeoLiteCity.dat" 1227 | database => "/etc/logstash/geoip/GeoLite2-City.mmdb" 1228 | source => "fail2ban_source_ip" 1229 | target => "fail2ban_geoip" 1230 | } 1231 | # AWS us-east-1 : Virginia 1232 | if [fail2ban_source_ip] =~ /^10\./ { 1233 | mutate { replace => { "[fail2ban_geoip][timezone]" => "America/New_York" } } 1234 | mutate { replace => { "[fail2ban_geoip][country_name]" => "United States" } } 1235 | mutate { replace => { "[fail2ban_geoip][country_code2]" => "US" } } 1236 | mutate { replace => { "[fail2ban_geoip][country_code3]" => "USA" } } 1237 | mutate { replace => { "[fail2ban_geoip][area_code]" => "703" } } 1238 | mutate { convert => { "[fail2ban_geoip][area_code]" => "integer" } } 1239 | mutate { replace => { "[fail2ban_geoip][city_name]" => "Ashburn" } } 1240 | mutate { replace => { "[fail2ban_geoip][continent_code]" => "NA" } } 1241 | mutate { replace => { "[fail2ban_geoip][dma_code]" => "511" } } 1242 | mutate { convert => { "[fail2ban_geoip][dma_code]" => "integer" } } 1243 | mutate { replace => { "[fail2ban_geoip][ip]" => "%{fail2ban_source_ip}" } } 1244 | mutate { replace => { "[fail2ban_geoip][postal_code]" => "20147" } } 1245 | mutate { replace => { "[fail2ban_geoip][real_region_name]" => "Virginia" } } 1246 | mutate { replace => { "[fail2ban_geoip][region_name]" => "VA" } } 1247 | mutate { replace => { "[fail2ban_geoip][latitude]" => "39.044" } } 1248 | mutate { convert => { "[fail2ban_geoip][latitude]" => "float" } } 1249 | mutate { replace => { "[fail2ban_geoip][longitude]" => "-77.4875" } } 1250 | mutate { convert => { "[fail2ban_geoip][longitude]" => "float" } } 1251 | mutate { replace => { "[fail2ban_geoip][location]" => "-77.4875, 39.044" } } 1252 | } 1253 | mutate { 1254 | add_field => { "fail2ban_source_hostname" => "%{fail2ban_source_ip}" } 1255 | } 1256 | dns { 1257 | reverse => [ "fail2ban_source_hostname" ] 1258 | action => "replace" 1259 | hit_cache_size => 8000 1260 | hit_cache_ttl => 300 1261 | failed_cache_size => 1000 1262 | failed_cache_ttl => 300 1263 | } 1264 | } 1265 | } 1266 | 1267 | # TEST implementation of parse for sysdig 1268 | # https://gist.github.com/jordansissel/5f260954e95085294096 1269 | # ex. # sysdig -t a "not(proc.name = sysdig)" | /opt/logstash/bin/logstash -f /etc/logstash/conf.d/sysdig-shipper.conf 1270 | if [log_type] == "sysdig" { 1271 | # With absolute time via `sysdig -t a` 1272 | # 1273 | # Example: 1274 | # 22232 1397121223.916379436 0 sysdig (2816) < write res=62 data=3452 1397121223.855286080 0 sysdig (2816) > switch next=1822 . 1275 | grok { 1276 | patterns_dir => "/etc/logstash/patterns" 1277 | match => { "message" => "%{SYSDIG}" } 1278 | } 1279 | date { 1280 | match => [ "sysdig_timestamp", "UNIX" ] 1281 | timezone => "Asia/Tokyo" 1282 | locale => "en" 1283 | } 1284 | mutate { 1285 | replace => { "sysdig_timestamp" => "%{@timestamp}" } 1286 | } 1287 | # for check grok data type conversion bug??? 1288 | mutate { 1289 | convert => { 1290 | "sysdig_num" => "integer" 1291 | "sysdig_cpu" => "integer" 1292 | "sysdig_time" => "float" 1293 | } 1294 | } 1295 | if [sysdig_kv_args] { 1296 | kv { 1297 | source => "sysdig_kv_args" 1298 | include_keys => [ "addr", "args", "argument", "backlog", "cgroups", "clockid", "cmd", "comm", "cur", "cwd", "data", "dirfd", "domain", "dpid", "dqb_bhardlimit", "dqb_bsoftlimit", "dqb_btime", "dqb_curspace", "dqb_ihardlimit", "dqb_isoftlimit", "dqb_itime", "dqi_bgrace", "dqi_flags", "dqi_igrace", "egid", "env", "euid", "event_data", "event_type", "exe", "fd1", "fd2", "fd_in", "fdlimit", "fd_out", "fd", "fds", "flags", "gid", "how", "id", "ID", "image", "in_fd", "initval", "ino", "interval", "length", "linkdirfd", "linkpath", "mask", "maxevents", "max", "mode", "name", "nativeID", "newcur", "newdirfd", "newdir", "newmax", "newpath", "next", "offset", "oldcur", "olddirfd", "olddir", "oldmax", "oldpath", "op", "out_fd", "path", "peer", "pgft_maj", "pgft_min", "pgoffset", "pid", "pos", "proto", "prot", "ptid", "queuepct", "quotafilepath", "quota_fmt_out", "quota_fmt", "ratio", "request", "resource", "res", "rgid", "ruid", "sgid", "sig", "size", "source", "special", "spid", "status", "suid", "target", "tid", "timeout", "tuple", "type", "uid", "val", "vm_rss", "vm_size", "vm_swap", "vpid", "vtid", "whence" ] 1299 | prefix => "sysdig_" 1300 | #remove_field => [ "sysdig_kv_args" ] 1301 | } 1302 | mutate { 1303 | convert => { 1304 | "sysdig_clockid" => "integer" 1305 | "sysdig_queuepct" => "integer" 1306 | "sysdig_nativeID" => "integer" 1307 | "sysdig_backlog" => "integer" 1308 | "sysdig_event_type" => "integer" 1309 | "sysdig_proto" => "integer" 1310 | "sysdig_ratio" => "integer" 1311 | "sysdig_vm_rss" => "integer" 1312 | "sysdig_vm_size" => "integer" 1313 | "sysdig_vm_swap" => "integer" 1314 | } 1315 | } 1316 | } 1317 | } 1318 | } 1319 | 1320 | output { 1321 | elasticsearch { 1322 | hosts => ["127.0.0.1"] 1323 | template => "/etc/logstash/elasticsearch-template-es7x.json" 1324 | template_overwrite => true 1325 | ilm_enabled => false 1326 | } 1327 | 1328 | # send to Datadog DogStatsD 1329 | if [apache_response] and [log_type] != "varnish" { 1330 | # logstash-output-statsd plugins were removed from the 6.0 default bundle. 1331 | # ./bin/logstash-plugin install logstash-output-statsd 1332 | statsd { 1333 | count => [ "apache.count.bytes", "%{apache_bytes}" ] 1334 | increment => "apache.count.response.%{apache_response}" 1335 | timing => [ "apache.timing.request", "%{apache_responsetime}" ] 1336 | } 1337 | # "apache_response" as long 1338 | if [apache_response] >= 200 and [apache_response] <= 299 { 1339 | statsd { increment => "apache.count.response.2XX" } 1340 | } else if [apache_response] >= 300 and [apache_response] <= 399 { 1341 | statsd { increment => "apache.count.response.3XX" } 1342 | } else if [apache_response] >= 400 and [apache_response] <= 499 { 1343 | statsd { increment => "apache.count.response.4XX" } 1344 | } else if [apache_response] >= 500 and [apache_response] <= 599 { 1345 | statsd { increment => "apache.count.response.5XX" } 1346 | } 1347 | } 1348 | } 1349 | -------------------------------------------------------------------------------- /patterns/amavis: -------------------------------------------------------------------------------- 1 | # amavis 2 | AMAVIS_QUEUEID (?:[0-9A-F]{6,}|[0-9a-zA-Z]{15,}) 3 | AMAVIS_STATUS (?:Passed|Blocked) 4 | AMAVIS_ORIGINATING (?:ORIGINATING|ORIGINATING_POST)(?:/MYNETS)? 5 | 6 | AMAVIS_INIT (?:starting\.|perl=|SpamControl:|Net::Server:|Module|SQL::Quarantine|%{GREEDYDATA} code |Found|No|Internal|Using|initializing|extra) 7 | AMAVIS_STARTUP ^%{AMAVIS_INIT} 8 | AMAVIS_REEXEC ^\(\!\)Net::Server: %{GREEDYDATA} Re-exec server during HUP 9 | AMAVIS_EXTRA \(%{DATA:amavis_thread}\) extra modules loaded: %{GREEDYDATA:amavis_extra_module} 10 | AMAVIS_PREPARE \(%{DATA:amavis_thread}\) TempDir::prepare_file: %{GREEDYDATA:amavis_prepare} 11 | AMAVIS_UNZIP \(%{DATA:amavis_thread}\) (?:\(\!\)|)do_unzip: %{GREEDYDATA:amavis_unzip_error} 12 | AMAVIS_ASCII \(%{DATA:amavis_thread}\) do_ascii: %{GREEDYDATA:amavis_ascii_error} 13 | AMAVIS_INFO \(%{DATA:amavis_thread}\) INFO: %{GREEDYDATA:amavis_info} 14 | AMAVIS_WARN \(%{DATA:amavis_thread}\) WARN: %{GREEDYDATA:amavis_warning} 15 | AMAVIS_LOCAL_DELIVERY \(%{DATA:amavis_thread}\) local delivery: %{GREEDYDATA:amavis_local_delivery} 16 | AMAVIS_SA \(%{DATA:amavis_thread}\) SA info: %{GREEDYDATA:amavis_sa_info} 17 | AMAVIS_FILECHECK \(%{DATA:amavis_thread}\) ESMTP \[%{IP:amavis_source_ip}\]?:%{POSINT:amavis_relay_port} %{DATA:amavis_quarantine_file}:\s+<(?:%{DATA:amavis_sender})?> -> <(?:%{DATA:amavis_recipient})?>(?:,<%{GREEDYDATA:amavis_recipientlist}>)?(?: SIZE=%{POSINT:amavis_size})?(?: RET=%{DATA:amavis_ret})?(?: BODY=%{DATA:amavis_body})?(?: ENVID=%{DATA:amavis_envid})? Received: %{GREEDYDATA:amavis_summary} 18 | AMAVIS_CHECKING \(%{DATA:amavis_thread}\) Checking: %{DATA:amavis_mail_id} (?:%{AMAVIS_ORIGINATING} )?(?:\[%{IP:amavis_source_ip}\] )?<(?:%{DATA:amavis_sender})?> -> <(?:%{DATA:amavis_recipient})?>(?:,<%{GREEDYDATA:amavis_recipientlist}>)? 19 | AMAVIS_CHECKRELAY \(%{DATA:amavis_thread}\) Open relay\? Nonlocal recips but not originating: %{DATA:amavis_recipient} 20 | AMAVIS_QUEUED \(%{DATA:amavis_thread}\) %{DATA:amavis_mail_id} (FWD|SEND) from <(?:%{DATA:amavis_sender})?> -> <(?:%{DATA:amavis_recipient})?>,( )?(?:ENVID=%{DATA:amavis_envid})?(?:RET=%{DATA:amavis_ret})?(?:BODY=%{DATA:amavis_body})? 250 2.0.0 from MTA\(smtp:\[%{IP:amavis_source_ip}\]:%{POSINT:amavis_relay_port}\): 250 2.0.0 Ok: queued as %{AMAVIS_QUEUEID:amavis_queued_as} 21 | AMAVIS_RESULT \(%{DATA:amavis_thread}\) %{AMAVIS_STATUS:amavis_status} %{DATA:amavis_result} (?:\(%{DATA:amavis_result_summary}\) )?\{%{DATA:amavis_actions}\}, (?:(?:(?:(?:%{AMAVIS_ORIGINATING} LOCAL) )?\[%{IP:amavis_relay_ip}\](?::%{POSINT:amavis_relay_port})?( \[%{IP:amavis_origin_ip}\])?)? )?<(?:%{DATA:amavis_sender})?> -> <(?:%{DATA:amavis_recipient})?>(?:,<%{GREEDYDATA:amavis_recipientlist}>)?, (?:quarantine: %{DATA:amavis_quarantine}, )?(?:Queue-ID: %{AMAVIS_QUEUEID:amavis_queueid}, )?Message-ID: <(?:%{DATA:amavis_message-id})?>, (?:mail_id: %{DATA:amavis_mail_id}, )?(?:Hits: %{DATA:amavis_hits}, )?(?:size: %{NONNEGINT:amavis_size}, )?(?:queued_as: %{AMAVIS_QUEUEID:amavis_queued_as}(?:/%{AMAVIS_QUEUEID:amavis_queued_as2})?, )?(?:dkim_sd=%{DATA:amavis_dkim_sd}, )?%{NONNEGINT:amavis_delay} ms 22 | 23 | ### 24 | AMAVIS %{AMAVIS_STARTUP}|%{AMAVIS_REEXEC}|%{AMAVIS_EXTRA}|%{AMAVIS_PREPARE}|%{AMAVIS_UNZIP}|%{AMAVIS_ASCII}|%{AMAVIS_INFO}|%{AMAVIS_WARN}|%{AMAVIS_LOCAL_DELIVERY}|%{AMAVIS_SA}|%{AMAVIS_FILECHECK}|%{AMAVIS_CHECKING}|%{AMAVIS_CHECKRELAY}|%{AMAVIS_QUEUED}|%{AMAVIS_RESULT} 25 | -------------------------------------------------------------------------------- /patterns/apache: -------------------------------------------------------------------------------- 1 | # apache log patterns 2 | 3 | APACHE_COMMON %{IPORHOST:apache_client_ip} %{HTTPDUSER:apache_ident} %{USER:apache_auth} \[%{HTTPDATE:apache_timestamp}\] "(?:%{WORD:apache_verb} %{NOTSPACE:apache_request}(?: HTTP/%{NUMBER:apache_httpversion})?|%{DATA:apache_rawrequest})" (?:%{NUMBER:apache_response}|-) (?:%{NUMBER:apache_bytes}|-) 4 | APACHE_COMBINED %{APACHE_COMMON} %{QS:apache_referrer} %{QS:apache_agent} 5 | APACHE_EXT_COMBINED %{APACHE_COMBINED}(?: %{NUMBER:apache_responsetime})?(?: %{TIMESTAMP_ISO8601:apache_timestamp2})? 6 | 7 | APACHE20_ERROR \[%{HTTPDERROR_DATE:apache_error_timestamp}\] \[%{LOGLEVEL:apache_error_loglevel}\] (?:\[client %{IPORHOST:apache_error_client_ip}\] )?%{GREEDYDATA:apache_error_message} 8 | APACHE24_ERROR \[%{HTTPDERROR_DATE:apache_error_timestamp}\] \[(%{WORD:apache_error_module})?:%{LOGLEVEL:apache_error_loglevel}\] \[pid %{POSINT:apache_error_pid}(:tid %{NUMBER:apache_error_tid})?\]( \(%{POSINT:apache_error_proxy_errorcode}\)%{DATA:apache_error_proxy_errormessage}:)?( \[client %{IPORHOST:apache_error_client_ip}:%{POSINT:apache_error_client_port}\])?( %{DATA:apache_error_errorcode}:)? %{GREEDYDATA:apache_error_message} 9 | APACHE_ERROR %{APACHE20_ERROR}|%{APACHE24_ERROR} 10 | 11 | APACHE_OTHER_VHOST %{IPORHOST:apache_vhost_host}:%{POSINT:apache_vhost_port} %{APACHE_COMBINED} 12 | APACHE_OTHER_VHOST_EXT %{IPORHOST:apache_vhost_host}:%{POSINT:apache_vhost_port} %{APACHE_EXT_COMBINED} 13 | -------------------------------------------------------------------------------- /patterns/fail2ban: -------------------------------------------------------------------------------- 1 | # fail2ban grok pattern 2 | 3 | FAIL2BAN_ACT (Ban|Unban|Found|Restore Ban) 4 | FAIL2BAN_HEAD %{TIMESTAMP_ISO8601:fail2ban_timestamp} fail2ban.%{WORD:fail2ban_class}\s*\[%{NUMBER:fail2ban_pid}\]: %{LOGLEVEL:fail2ban_level} 5 | 6 | FAIL2BAN_BAN %{FAIL2BAN_HEAD}\s+\[%{DATA:fail2ban_service}\] %{FAIL2BAN_ACT:fail2ban_action} %{IP:fail2ban_source_ip} 7 | FAIL2BAN_OTHERS %{FAIL2BAN_HEAD}\s+%{GREEDYDATA:fail2ban_info} 8 | 9 | ### 10 | FAIL2BAN %{FAIL2BAN_BAN}|%{FAIL2BAN_OTHERS} 11 | -------------------------------------------------------------------------------- /patterns/postfix: -------------------------------------------------------------------------------- 1 | # common postfix patterns 2 | POSTFIX_QUEUEID (?:[0-9A-F]{6,}|[0-9a-zA-Z]{15,}|NOQUEUE) 3 | POSTFIX_CLIENT_INFO %{HOSTNAME:postfix_client_hostname}?\[%{IPORHOST:postfix_client_ip}\](?::%{INT:postfix_client_port})? 4 | POSTFIX_RELAY_INFO %{HOSTNAME:postfix_relay_hostname}?\[%{IP:postfix_relay_ip}\](?::%{INT:postfix_relay_port})?|%{WORD:postfix_relay_service} 5 | POSTFIX_SMTP_STAGE (?:CONNECT|HELO|EHLO|STARTTLS|AUTH|MAIL|RCPT|DATA|RSET|UNKNOWN|END-OF-MESSAGE|VRFY|NOOP|QUIT|\.) 6 | POSTFIX_ACTION (?:reject|defer) 7 | POSTFIX_STATUS_CODE \d{3} 8 | POSTFIX_STATUS_CODE_ENHANCED \d\.\d\.\d 9 | POSTFIX_DNSBL_MESSAGE Service unavailable; .* \[%{GREEDYDATA:postfix_status_data}\] %{GREEDYDATA:postfix_status_message}; 10 | POSTFIX_PS_ACCESS_ACTION (?:DISCONNECT|BLACKLISTED|WHITELISTED|WHITELIST VETO|PASS NEW|PASS OLD) 11 | POSTFIX_PS_VIOLATION (?:BARE NEWLINE|COMMAND (?:TIME|COUNT|LENGTH) LIMIT|COMMAND PIPELINING|DNSBL|HANGUP|NON-SMTP COMMAND|PREGREET) 12 | POSTFIX_TIME_UNIT %{NUMBER}[smhd] 13 | POSTFIX_KEYVALUE %{POSTFIX_QUEUEID:postfix_queueid}: %{GREEDYDATA:postfix_keyvalue_data} 14 | POSTFIX_WARNING (?:warning|fatal): %{GREEDYDATA:postfix_warning} 15 | POSTFIX_TLSCONN (?:Anonymous|Trusted|Untrusted|Verified) TLS connection established (?:to %{POSTFIX_RELAY_INFO}|from %{POSTFIX_CLIENT_INFO}): %{DATA:postfix_tls_version} with cipher %{DATA:postfix_tls_cipher} \(%{DATA:postfix_tls_cipher_size} bits\) 16 | POSTFIX_DELAYS %{NUMBER:postfix_delay_before_qmgr}/%{NUMBER:postfix_delay_in_qmgr}/%{NUMBER:postfix_delay_conn_setup}/%{NUMBER:postfix_delay_transmission} 17 | POSTFIX_LOSTCONN (?:lost connection|timeout|Connection timed out|Connection reset by peer|-1|0) 18 | 19 | # smtpd patterns 20 | POSTFIX_SMTPD_CONNECT connect from %{POSTFIX_CLIENT_INFO} 21 | POSTFIX_SMTPD_DISCONNECT disconnect from %{POSTFIX_CLIENT_INFO} 22 | POSTFIX_SMTPD_LOSTCONN (?:%{POSTFIX_LOSTCONN:postfix_smtpd_lostconn_data} after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage}(?: \(%{INT} bytes\))? from %{POSTFIX_CLIENT_INFO}|%{GREEDYDATA:postfix_action} from %{POSTFIX_CLIENT_INFO}: %{POSTFIX_LOSTCONN:postfix_smtpd_lostconn_data}) 23 | POSTFIX_SMTPD_NOQUEUE NOQUEUE: %{POSTFIX_ACTION:postfix_action}: %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO}: %{POSTFIX_STATUS_CODE:postfix_status_code} %{POSTFIX_STATUS_CODE_ENHANCED:postfix_status_code_enhanced}(?: <%{DATA:postfix_status_data}>:)? (?:%{POSTFIX_DNSBL_MESSAGE}|%{GREEDYDATA:postfix_status_message};) %{GREEDYDATA:postfix_keyvalue_data} 24 | POSTFIX_SMTPD_PIPELINING improper command pipelining after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO}: 25 | POSTFIX_SMTPD_ERROR too many errors after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO} 26 | 27 | # cleanup patterns 28 | POSTFIX_CLEANUP_MILTER_REDIRECT %{POSTFIX_QUEUEID:postfix_queueid}: milter-header-redirect: %{GREEDYDATA:postfix_milter_redirect_data}; %{GREEDYDATA:postfix_keyvalue_data}: %{GREEDYDATA:postfix_milter_redirect_target} 29 | POSTFIX_CLEANUP_MILTER_REJECT %{POSTFIX_QUEUEID:postfix_queueid}: milter-reject: %{GREEDYDATA:postfix_milter_reject_data}; %{GREEDYDATA:postfix_keyvalue_data} 30 | 31 | # qmgr patterns 32 | POSTFIX_QMGR_REMOVED %{POSTFIX_QUEUEID:postfix_queueid}: removed 33 | POSTFIX_QMGR_SKIPPED %{POSTFIX_QUEUEID:postfix_queueid}: skipped, still being delivered 34 | POSTFIX_QMGR_ACTIVE %{POSTFIX_QUEUEID:postfix_queueid}: %{GREEDYDATA:postfix_keyvalue_data} \(queue active\) 35 | POSTFIX_QMGR_RETURN %{POSTFIX_QUEUEID:postfix_queueid}: %{GREEDYDATA:postfix_keyvalue_data}, returned to sender 36 | 37 | # pipe patterns 38 | POSTFIX_PIPE_DELIVERED %{POSTFIX_QUEUEID:postfix_queueid}: %{GREEDYDATA:postfix_keyvalue_data} \(delivered via %{WORD:postfix_pipe_service} service\) 39 | 40 | # postscreen patterns 41 | POSTFIX_PS_CONNECT CONNECT from %{POSTFIX_CLIENT_INFO} to \[%{IP:postfix_server_ip}\]:%{INT:postfix_server_port} 42 | POSTFIX_PS_ACCESS %{POSTFIX_PS_ACCESS_ACTION:postfix_postscreen_access} %{POSTFIX_CLIENT_INFO} 43 | POSTFIX_PS_NOQUEUE %{POSTFIX_SMTPD_NOQUEUE} 44 | POSTFIX_PS_TOOBUSY NOQUEUE: reject: CONNECT from %{POSTFIX_CLIENT_INFO}: %{GREEDYDATA:postfix_postscreen_toobusy_data} 45 | POSTFIX_PS_DNSBL %{POSTFIX_PS_VIOLATION:postfix_postscreen_violation} rank %{INT:postfix_postscreen_dnsbl_rank} for %{POSTFIX_CLIENT_INFO} 46 | POSTFIX_PS_CACHE cache %{DATA} (full|partial) cleanup: retained=%{NUMBER:postfix_postscreen_cache_retained} dropped=%{NUMBER:postfix_postscreen_cache_dropped} entries 47 | POSTFIX_PS_VIOLATIONS %{POSTFIX_PS_VIOLATION:postfix_postscreen_violation}(?: %{INT})?(?: after %{NUMBER:postfix_postscreen_violation_time})? from %{POSTFIX_CLIENT_INFO}(?: after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage})? 48 | 49 | # dnsblog patterns 50 | POSTFIX_DNSBLOG_LISTING addr %{IP:postfix_client_ip} listed by domain %{HOSTNAME:postfix_dnsbl_domain} as %{IP:postfix_dnsbl_result} 51 | 52 | # tlsproxy patterns 53 | POSTFIX_TLSPROXY_CONN (?:DIS)?CONNECT(?: from)? %{POSTFIX_CLIENT_INFO} 54 | 55 | # anvil patterns 56 | POSTFIX_ANVIL_CONN_RATE statistics: max connection rate %{NUMBER:postfix_anvil_conn_rate}/%{POSTFIX_TIME_UNIT:postfix_anvil_conn_period} for \(%{DATA:postfix_protocol}:%{IPORHOST:postfix_client_ip}\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp} 57 | POSTFIX_ANVIL_CONN_CACHE statistics: max cache size %{NUMBER:postfix_anvil_cache_size} at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp} 58 | POSTFIX_ANVIL_CONN_COUNT statistics: max connection count %{NUMBER:postfix_anvil_conn_count} for \(%{DATA:postfix_protocol}:%{IPORHOST:postfix_client_ip}\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp} 59 | 60 | # smtp patterns 61 | POSTFIX_SMTP_DELIVERY %{POSTFIX_KEYVALUE} \(%{GREEDYDATA:postfix_smtp_response}\) 62 | POSTFIX_SMTP_CONNERR connect to %{POSTFIX_RELAY_INFO}: (?:Connection timed out|No route to host|Connection refused) 63 | POSTFIX_SMTP_LOSTCONN %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_LOSTCONN} with %{POSTFIX_RELAY_INFO} 64 | POSTFIX_SMTP_TLSERR %{POSTFIX_QUEUEID:postfix_queueid}: Cannot start TLS: %{GREEDYDATA:postfix_tls_error} 65 | POSTFIX_SMTP_TLSLOSTCONN SSL_connect error to %{POSTFIX_RELAY_INFO}: %{POSTFIX_LOSTCONN} 66 | 67 | # master patterns 68 | POSTFIX_MASTER_START (?:daemon started|reload) -- version %{DATA:postfix_version}, configuration %{PATH:postfix_config_path} 69 | POSTFIX_MASTER_EXIT terminating on signal %{INT:postfix_termination_signal} 70 | 71 | # bounce patterns 72 | POSTFIX_BOUNCE_NOTIFICATION %{POSTFIX_QUEUEID:postfix_queueid}: sender (?:non-delivery|delivery status|delay) notification: %{POSTFIX_QUEUEID:postfix_bounce_queueid} 73 | 74 | # scache patterns 75 | POSTFIX_SCACHE_LOOKUPS statistics: (?:address|domain) lookup hits=%{INT:postfix_scache_hits} miss=%{INT:postfix_scache_miss} success=%{INT:postfix_scache_success} 76 | POSTFIX_SCACHE_SIMULTANEOUS statistics: max simultaneous domains=%{INT:postfix_scache_domains} addresses=%{INT:postfix_scache_addresses} connection=%{INT:postfix_scache_connection} 77 | POSTFIX_SCACHE_TIMESTAMP statistics: start interval %{SYSLOGTIMESTAMP:postfix_scache_timestamp} 78 | 79 | # aggregate all patterns 80 | POSTFIX_SMTPD %{POSTFIX_SMTPD_CONNECT}|%{POSTFIX_SMTPD_DISCONNECT}|%{POSTFIX_SMTPD_LOSTCONN}|%{POSTFIX_SMTPD_NOQUEUE}|%{POSTFIX_SMTPD_PIPELINING}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_KEYVALUE}|%{POSTFIX_SMTPD_ERROR} 81 | POSTFIX_CLEANUP %{POSTFIX_CLEANUP_MILTER_REDIRECT}|%{POSTFIX_CLEANUP_MILTER_REJECT}|%{POSTFIX_WARNING}|%{POSTFIX_KEYVALUE} 82 | POSTFIX_QMGR %{POSTFIX_QMGR_REMOVED}|%{POSTFIX_QMGR_SKIPPED}|%{POSTFIX_QMGR_ACTIVE}|%{POSTFIX_QMGR_RETURN}|%{POSTFIX_WARNING} 83 | POSTFIX_PIPE %{POSTFIX_PIPE_DELIVERED} 84 | POSTFIX_POSTSCREEN %{POSTFIX_PS_CONNECT}|%{POSTFIX_PS_ACCESS}|%{POSTFIX_PS_NOQUEUE}|%{POSTFIX_PS_TOOBUSY}|%{POSTFIX_PS_CACHE}|%{POSTFIX_PS_DNSBL}|%{POSTFIX_PS_VIOLATIONS}|%{POSTFIX_WARNING} 85 | POSTFIX_DNSBLOG %{POSTFIX_DNSBLOG_LISTING} 86 | POSTFIX_ANVIL %{POSTFIX_ANVIL_CONN_RATE}|%{POSTFIX_ANVIL_CONN_CACHE}|%{POSTFIX_ANVIL_CONN_COUNT} 87 | POSTFIX_SMTP %{POSTFIX_SMTP_DELIVERY}|%{POSTFIX_SMTP_CONNERR}|%{POSTFIX_SMTP_LOSTCONN}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTP_TLSERR}|%{POSTFIX_SMTP_TLSLOSTCONN} 88 | POSTFIX_PICKUP %{POSTFIX_KEYVALUE} 89 | POSTFIX_TLSPROXY %{POSTFIX_TLSPROXY_CONN} 90 | POSTFIX_MASTER %{POSTFIX_MASTER_START}|%{POSTFIX_MASTER_EXIT} 91 | POSTFIX_BOUNCE %{POSTFIX_BOUNCE_NOTIFICATION} 92 | POSTFIX_SENDMAIL %{POSTFIX_WARNING} 93 | POSTFIX_POSTDROP %{POSTFIX_WARNING} 94 | POSTFIX_SCACHE %{POSTFIX_SCACHE_LOOKUPS}|%{POSTFIX_SCACHE_SIMULTANEOUS}|%{POSTFIX_SCACHE_TIMESTAMP} 95 | POSTFIX_TRIVIAL_REWRITE %{POSTFIX_WARNING} 96 | POSTFIX_TLSMGR %{POSTFIX_WARNING} 97 | -------------------------------------------------------------------------------- /patterns/saslauthd: -------------------------------------------------------------------------------- 1 | # saslauthd grok pattern 2 | 3 | SASLAUTHD %{DATA:saslauthd_type}\s*: %{GREEDYDATA:saslauthd_message} 4 | -------------------------------------------------------------------------------- /patterns/sshd: -------------------------------------------------------------------------------- 1 | # sshd grok pattern 2 | 3 | # Start/Stop 4 | SSHD_LISTEN Server listening on %{IP:sshd_listen_ip} port %{NUMBER:sshd_listen_port}. 5 | SSHD_TERMINATE Received signal %{NUMBER:sshd_signal}; terminating. 6 | 7 | # SSH Tunnel 8 | SSHD_TUNN_ERR1 error: connect_to %{IP:sshd_listen_ip} port %{NUMBER:sshd_listen_port}: failed. 9 | SSHD_TUNN_ERR2 error: channel_setup_fwd_listener: cannot listen to port: %{NUMBER:sshd_listen_port} 10 | SSHD_TUNN_ERR3 error: bind: Address already in use 11 | SSHD_TUNN_ERR4 error: channel_setup_fwd_listener_tcpip: cannot listen to port: %{NUMBER:sshd_listen_port} 12 | SSHD_TUNN_TIMEOUT Timeout, client not responding. 13 | 14 | # Normal 15 | SSHD_SUCCESS (?Accepted) %{WORD:sshd_auth_type} for %{USERNAME:sshd_user} from %{IP:sshd_client_ip} port %{NUMBER:sshd_port} %{WORD:sshd_protocol}: %{GREEDYDATA:sshd_cipher} 16 | SSHD_DISCONNECT Received disconnect from %{IP:sshd_client_ip} port %{NUMBER:sshd_port}:%{NUMBER:sshd_disconnect_code}: %{GREEDYDATA:sshd_disconnect_status} 17 | SSHD_CONN_CLOSE Connection closed by %{IP:sshd_client_ip}$ 18 | SSHD_SESSION_OPEN pam_unix\(sshd:session\): session opened for user %{USERNAME:sshd_user} by \(uid=\d+\) 19 | SSHD_SESSION_CLOSE pam_unix\(sshd:session\): session closed for user %{USERNAME:sshd_user} 20 | SSHD_SESSION_FAIL pam_systemd\(sshd:session\): Failed to release session: %{GREEDYDATA:sshd_disconnect_status} 21 | SSHD_LOGOUT_ERR syslogin_perform_logout: logout\(\) returned an error 22 | 23 | # Probe 24 | SSHD_REFUSE_CONN (?refused) connect from %{DATA:sshd_client_hostname} \(%{IPORHOST:sshd_client_ip}\) 25 | SSHD_TCPWRAP_FAIL1 warning: %{DATA:sshd_tcpd_file}, line %{NUMBER}: can't verify hostname: getaddrinfo\(%{DATA:sshd_paranoid_hostname}, %{DATA:sshd_sa_family}\) failed 26 | SSHD_TCPWRAP_FAIL2 warning: %{DATA:sshd_tcpd_file}, line %{NUMBER}: host name/address mismatch: %{IPORHOST:sshd_client_ip} != %{HOSTNAME:sshd_paranoid_hostname} 27 | SSHD_TCPWRAP_FAIL3 warning: %{DATA:sshd_tcpd_file}, line %{NUMBER}: host name/name mismatch: %{HOSTNAME:sshd_paranoid_hostname_1} != %{HOSTNAME:sshd_paranoid_hostname_2} 28 | SSHD_TCPWRAP_FAIL4 warning: %{DATA:sshd_tcpd_file}, line %{NUMBER}: host name/name mismatch: reverse lookup results in non-FQDN %{HOSTNAME:sshd_paranoid_hostname} 29 | SSHD_TCPWRAP_FAIL5 warning: can't get client address: Connection reset by peer 30 | SSHD_FAIL Failed %{WORD:sshd_auth_type} for %{USERNAME:sshd_invalid_user} from %{IP:sshd_client_ip} port %{NUMBER:sshd_port} %{WORD:sshd_protocol} 31 | SSHD_USER_FAIL Failed password for invalid user %{USERNAME:sshd_invalid_user} from %{IP:sshd_client_ip} port %{NUMBER:sshd_port} %{WORD:sshd_protocol} 32 | SSHD_INVAL_USER Invalid user\s*%{USERNAME:sshd_invalid_user}? from %{IP:sshd_client_ip} 33 | 34 | # preauth 35 | SSHD_DISC_PREAUTH Disconnected from %{IP:sshd_client_ip} port %{NUMBER:sshd_port}\s*(?:\[%{GREEDYDATA:sshd_privsep}\]|) 36 | SSHD_RECE_PREAUTH (?:error: |)Received disconnect from %{IP:sshd_client_ip} port %{NUMBER:sshd_port}:%{NUMBER:sshd_disconnect_code}: %{GREEDYDATA:sshd_disconnect_status}? \[%{GREEDYDATA:sshd_privsep}\] 37 | SSHD_MAXE_PREAUTH error: maximum authentication attempts exceeded for (?:invalid user |)%{USERNAME:sshd_invalid_user} from %{IP:sshd_client_ip} port %{NUMBER:sshd_port} %{WORD:sshd_protocol}\s*(?:\[%{GREEDYDATA:sshd_privsep}\]|) 38 | SSHD_DISR_PREAUTH Disconnecting: %{GREEDYDATA:sshd_disconnect_status} \[%{GREEDYDATA:sshd_privsep}\] 39 | SSHD_INVA_PREAUTH input_userauth_request: invalid user %{USERNAME:sshd_invalid_user}?\s*(?:\[%{GREEDYDATA:sshd_privsep}\]|) 40 | SSHD_REST_PREAUTH Connection reset by %{IP:sshd_client_ip} port %{NUMBER:sshd_port}\s*(?:\[%{GREEDYDATA:sshd_privsep}\]|) 41 | SSHD_CLOS_PREAUTH Connection closed by %{IP:sshd_client_ip} port %{NUMBER:sshd_port}\s*(?:\[%{GREEDYDATA:sshd_privsep}\]|) 42 | SSHD_FAIL_PREAUTH fatal: Unable to negotiate with %{IP:sshd_client_ip} port %{NUMBER:sshd_port}:\s*%{GREEDYDATA:sshd_disconnect_status}? \[%{GREEDYDATA:sshd_privsep}\] 43 | SSHD_FAI2_PREAUTH fatal: %{GREEDYDATA:sshd_fatal_status}: Connection from %{IP:sshd_client_ip} port %{NUMBER:sshd_port}:\s*%{GREEDYDATA:sshd_disconnect_status}? \[%{GREEDYDATA:sshd_privsep}\] 44 | SSHD_BADL_PREAUTH Bad packet length %{NUMBER:sshd_packet_length}. \[%{GREEDYDATA:sshd_privsep}\] 45 | 46 | # Corrupted 47 | SSHD_IDENT_FAIL Did not receive identification string from %{IP:sshd_client_ip} 48 | SSHD_MAPB_FAIL Address %{IP:sshd_client_ip} maps to %{HOSTNAME:sshd_client_hostname}, but this does not map back to the address - POSSIBLE BREAK-IN ATTEMPT! 49 | SSHD_RMAP_FAIL reverse mapping checking getaddrinfo for %{HOSTNAME:sshd_client_hostname} \[%{IP:sshd_client_ip}\] failed - POSSIBLE BREAK-IN ATTEMPT! 50 | SSHD_TOOMANY_AUTH Disconnecting: Too many authentication failures for %{USERNAME:sshd_invalid_user} 51 | SSHD_CORRUPT_MAC Corrupted MAC on input 52 | SSHD_PACKET_CORRUPT Disconnecting: Packet corrupt 53 | SSHD_BAD_VERSION Bad protocol version identification '%{GREEDYDATA}' from %{IP:sshd_client_ip} 54 | 55 | #### 56 | SSHD_INIT %{SSHD_LISTEN}|%{SSHD_TERMINATE} 57 | SSHD_TUNN %{SSHD_TUNN_ERR1}|%{SSHD_TUNN_ERR2}|%{SSHD_TUNN_ERR3}|%{SSHD_TUNN_ERR4}|%{SSHD_TUNN_TIMEOUT} 58 | SSHD_NORMAL_LOG %{SSHD_SUCCESS}|%{SSHD_DISCONNECT}|%{SSHD_CONN_CLOSE}|%{SSHD_SESSION_OPEN}|%{SSHD_SESSION_CLOSE}|%{SSHD_SESSION_FAIL}|%{SSHD_LOGOUT_ERR} 59 | SSHD_PROBE_LOG %{SSHD_REFUSE_CONN}|%{SSHD_TCPWRAP_FAIL1}|%{SSHD_TCPWRAP_FAIL2}|%{SSHD_TCPWRAP_FAIL3}|%{SSHD_TCPWRAP_FAIL4}|%{SSHD_TCPWRAP_FAIL5}|%{SSHD_FAIL}|%{SSHD_USER_FAIL}|%{SSHD_INVAL_USER} 60 | SSHD_PREAUTH %{SSHD_DISC_PREAUTH}|%{SSHD_RECE_PREAUTH}|%{SSHD_MAXE_PREAUTH}|%{SSHD_DISR_PREAUTH}|%{SSHD_INVA_PREAUTH}|%{SSHD_REST_PREAUTH}|%{SSHD_FAIL_PREAUTH}|%{SSHD_CLOS_PREAUTH}|%{SSHD_FAI2_PREAUTH}|%{SSHD_BADL_PREAUTH} 61 | SSHD_CORRUPTED %{SSHD_IDENT_FAIL}|%{SSHD_MAPB_FAIL}|%{SSHD_RMAP_FAIL}|%{SSHD_TOOMANY_AUTH}|%{SSHD_CORRUPT_MAC}|%{SSHD_PACKET_CORRUPT}|%{SSHD_BAD_VERSION} 62 | SSHD_LOG %{SSHD_INIT}|%{SSHD_NORMAL_LOG}|%{SSHD_PROBE_LOG}|%{SSHD_CORRUPTED}|%{SSHD_TUNN}|%{SSHD_PREAUTH} 63 | -------------------------------------------------------------------------------- /patterns/sysdig: -------------------------------------------------------------------------------- 1 | SYSDIG %{NUMBER:sysdig_num} %{NUMBER:sysdig_time} %{INT:sysdig_cpu} %{NOTSPACE:sysdig_procname} %{NOTSPACE:sysdig_thread} (?[<>]) %{WORD:sysdig_event}? ?%{GREEDYDATA:sysdig_kv_args} 2 | -------------------------------------------------------------------------------- /patterns/zimbra: -------------------------------------------------------------------------------- 1 | # zimbra mailbox.log 2 | ZIMBRA_COMPONENT Timer-Zimbra|Junk-NotJunk-Handler|MailboxPurge|btpool|pool|LmtpServer|ImapServer|ImapSSLServer|Imap-Idle-Proxy-ImapServer|Pop3Server|Pop3SSLServer|ScheduledTask|AnonymousIoService|CloudRoutingReaderThread|GC|SocketAcceptor|Thread|Index|FileLogWriter.FsyncThread|IncomingDirectorySweeper|mailboxd.csv|main|SyncGalGroup|%{WORD} 3 | 4 | ZIMBRA_MAILBOX %{TIMESTAMP_ISO8601:zimbra_timestamp} %{DATA:zimbra_loglevel} \[%{ZIMBRA_COMPONENT:zimbra_component}(?:-%{NONNEGINT:zimbra_thread_number}(?::%{DATA:zimbra_soap_request})?)?\] \[%{DATA:zimbra_account}\] %{GREEDYDATA:zimbra_message} 5 | ZIMBRA_REMOTE %{TIMESTAMP_ISO8601:zimbra_timestamp} %{DATA:zimbra_loglevel} \[{%{DATA:zimbra_component}: %{HOSTNAME:zimbra_hostname}->%{DATA:zimbra_mailaddress}:%{NONNEGINT:zimbra_port}}-%{GREEDYDATA:zimbra_status} \[%{DATA:zimbra_account}\] %{GREEDYDATA:zimbra_message} 6 | 7 | ZIMBRA_TRACE_ID (?^ExceptionId:.*) 8 | ZIMBRA_TRACE_CAUSED_BY (?^Caused by:.*) 9 | ZIMBRA_TRACE_ERROR (?^.*Exception: .*) 10 | ZIMBRA_TRACE_NEST (?^.*nested exception is: .*) 11 | ZIMBRA_TRACE_CODE (?^Code:.*) 12 | ZIMBRA_TRACE_OTHERS (?^\s+at .*) 13 | ZIMBRA_TRACE_MORE (?^\s+... .* more) 14 | ZIMBRA_TRACE %{ZIMBRA_TRACE_CAUSED_BY}|%{ZIMBRA_TRACE_ID}|%{ZIMBRA_TRACE_CODE}|%{ZIMBRA_TRACE_OTHERS}|%{ZIMBRA_TRACE_MORE}|%{ZIMBRA_TRACE_ERROR}|%{ZIMBRA_TRACE_NEST} 15 | 16 | ZIMBRA_MULTILINE (?^bytesConsumed .*) 17 | 18 | ### 19 | ZIMBRA_MAILBOX_LOG %{ZIMBRA_MAILBOX}|%{ZIMBRA_REMOTE}|%{ZIMBRA_TRACE}|%{ZIMBRA_MULTILINE} 20 | -------------------------------------------------------------------------------- /shipper.d/shipper.conf: -------------------------------------------------------------------------------- 1 | input { 2 | beats { 3 | port => 5000 4 | ssl => true 5 | ssl_certificate => "/etc/logstash/logstash-forwarder.crt" 6 | ssl_key => "/etc/logstash/logstash-forwarder.key" 7 | } 8 | beats { 9 | port => 5001 10 | ssl => true 11 | ssl_certificate => "/etc/logstash/logstash-forwarder.crt" 12 | ssl_key => "/etc/logstash/logstash-forwarder.key" 13 | } 14 | beats { 15 | port => 5002 16 | ssl => true 17 | ssl_certificate => "/etc/logstash/logstash-forwarder.crt" 18 | ssl_key => "/etc/logstash/logstash-forwarder.key" 19 | } 20 | } 21 | 22 | output { 23 | redis { 24 | host => "127.0.0.1" 25 | data_type => "list" 26 | key => "logstash" 27 | } 28 | } 29 | --------------------------------------------------------------------------------