├── 01-inputs.conf ├── 10-syslog.conf ├── 11-pfsense.conf ├── 30-outputs.conf ├── Dockerfile ├── README.md ├── pfsense2_4_3.grok └── screenshot.jpg /01-inputs.conf: -------------------------------------------------------------------------------- 1 | input { 2 | tcp { 3 | type => "syslog" 4 | port => 5140 5 | } 6 | } 7 | input { 8 | udp { 9 | type => "syslog" 10 | port => 5140 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /10-syslog.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | if [type] == "syslog" { 3 | if [host] =~ /172\.17\.0\.1/ { 4 | mutate { 5 | add_tag => ["pfsense", "Ready"] 6 | } 7 | } 8 | if "Ready" not in [tags] { 9 | mutate { 10 | add_tag => [ "syslog" ] 11 | } 12 | } 13 | } 14 | } 15 | filter { 16 | if [type] == "syslog" { 17 | mutate { 18 | remove_tag => "Ready" 19 | } 20 | } 21 | } 22 | filter { 23 | if "syslog" in [tags] { 24 | grok { 25 | match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}" } 26 | add_field => [ "received_at", "%{@timestamp}" ] 27 | add_field => [ "received_from", "%{host}" ] 28 | } 29 | syslog_pri { } 30 | date { 31 | match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] 32 | locale => "en" 33 | } 34 | if !("_grokparsefailure" in [tags]) { 35 | mutate { 36 | replace => [ "@source_host", "%{syslog_hostname}" ] 37 | replace => [ "@message", "%{syslog_message}" ] 38 | } 39 | } 40 | mutate { 41 | remove_field => [ "syslog_hostname", "syslog_message", "syslog_timestamp" ] 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /11-pfsense.conf: -------------------------------------------------------------------------------- 1 | filter { 2 | if "pfsense" in [tags] { 3 | grok { 4 | add_tag => [ "firewall" ] 5 | match => [ "message", "<(?.*)>(?(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\s+(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) (?:2[0123]|[01]?[0-9]):(?:[0-5][0-9]):(?:[0-5][0-9])) (?.*?): (?.*)" ] 6 | } 7 | mutate { 8 | gsub => ["datetime"," "," "] 9 | } 10 | date { 11 | match => [ "datetime", "MMM dd HH:mm:ss" ] 12 | timezone => "Asia/Shanghai" 13 | } 14 | mutate { 15 | replace => [ "message", "%{msg}" ] 16 | } 17 | mutate { 18 | remove_field => [ "msg", "datetime" ] 19 | } 20 | if [prog] =~ /^dhcpd$/ { 21 | mutate { 22 | add_tag => [ "dhcpd" ] 23 | } 24 | grok { 25 | patterns_dir => ["/etc/logstash/conf.d/patterns"] 26 | match => [ "message", "%{DHCPD}"] 27 | } 28 | } 29 | if [prog] =~ /^suricata/ { 30 | mutate { 31 | add_tag => [ "SuricataIDPS" ] 32 | } 33 | grok { 34 | patterns_dir => ["/etc/logstash/conf.d/patterns"] 35 | match => [ "message", "%{PFSENSE_SURICATA}"] 36 | } 37 | if ![geoip] and [ids_src_ip] !~ /^(10\.|192\.168\.)/ { 38 | geoip { 39 | add_tag => [ "GeoIP" ] 40 | source => "ids_src_ip" 41 | database => "/etc/logstash/GeoLite2-City.mmdb" 42 | } 43 | } 44 | if [prog] =~ /^suricata/ { 45 | mutate { 46 | add_tag => [ "ET-Sig" ] 47 | add_field => [ "Signature_Info", "http://doc.emergingthreats.net/bin/view/Main/%{[ids_sig_id]}" ] 48 | } 49 | } 50 | } 51 | if [prog] =~ /^charon$/ { 52 | mutate { 53 | add_tag => [ "ipsec" ] 54 | } 55 | } 56 | if [prog] =~ /^barnyard2/ { 57 | mutate { 58 | add_tag => [ "barnyard2" ] 59 | } 60 | } 61 | if [prog] =~ /^openvpn/ { 62 | mutate { 63 | add_tag => [ "openvpn" ] 64 | } 65 | } 66 | if [prog] =~ /^ntpd/ { 67 | mutate { 68 | add_tag => [ "ntpd" ] 69 | } 70 | } 71 | if [prog] =~ /^php-fpm/ { 72 | mutate { 73 | add_tag => [ "web_portal" ] 74 | } 75 | grok { 76 | patterns_dir => ["/etc/logstash/conf.d/patterns"] 77 | match => [ "message", "%{PFSENSE_APP}%{PFSENSE_APP_DATA}"] 78 | } 79 | mutate { 80 | lowercase => [ 'pfsense_ACTION' ] 81 | } 82 | } 83 | if [prog] =~ /^apinger/ { 84 | mutate { 85 | add_tag => [ "apinger" ] 86 | } 87 | } 88 | if [prog] =~ /^filterlog$/ { 89 | mutate { 90 | remove_field => [ "msg", "datetime" ] 91 | } 92 | grok { 93 | add_tag => [ "firewall" ] 94 | patterns_dir => ["/etc/logstash/conf.d/patterns"] 95 | match => [ "message", "%{PFSENSE_LOG_DATA}%{PFSENSE_IP_SPECIFIC_DATA}%{PFSENSE_IP_DATA}%{PFSENSE_PROTOCOL_DATA}", 96 | "message", "%{PFSENSE_IPv4_SPECIFIC_DATA}%{PFSENSE_IP_DATA}%{PFSENSE_PROTOCOL_DATA}", 97 | "message", "%{PFSENSE_IPv6_SPECIFIC_DATA}%{PFSENSE_IP_DATA}%{PFSENSE_PROTOCOL_DATA}"] 98 | } 99 | mutate { 100 | lowercase => [ 'proto' ] 101 | } 102 | if ![geoip] and [src_ip] !~ /^(10\.|192\.168\.)/ { 103 | geoip { 104 | add_tag => [ "GeoIP" ] 105 | source => "src_ip" 106 | database => "/etc/logstash/GeoLite2-City.mmdb" 107 | } 108 | } 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /30-outputs.conf: -------------------------------------------------------------------------------- 1 | output { 2 | elasticsearch { 3 | hosts => ["http://localhost:9200"] 4 | index => "logstash-%{+YYYY.MM.dd}" } 5 | } 6 | 7 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM sebp/elk 2 | 3 | RUN apt-get install wget 4 | 5 | # for Synology NAS 6 | RUN echo 'bootstrap.system_call_filter: false' >> ${ES_PATH_CONF}/elasticsearch.yml 7 | 8 | # filters for pfSense and Suricata 9 | RUN rm ${LOGSTASH_PATH_CONF}/conf.d/* 10 | ADD ./01-inputs.conf ${LOGSTASH_PATH_CONF}/conf.d/01-inputs.conf 11 | ADD ./10-syslog.conf ${LOGSTASH_PATH_CONF}/conf.d/10-syslog.conf 12 | ADD ./11-pfsense.conf ${LOGSTASH_PATH_CONF}/conf.d/11-pfsense.conf 13 | ADD ./30-outputs.conf ${LOGSTASH_PATH_CONF}/conf.d/30-outputs.conf 14 | ADD ./pfsense2_4_3.grok ${LOGSTASH_PATH_CONF}/conf.d/patterns/pfsense2_4_3.grok 15 | 16 | # host files 17 | RUN echo '192.168.3.10 logs.blanboom.org logs' >> /etc/hosts 18 | RUN echo 'logs' >> /etc/hostname 19 | 20 | # GeoIP 21 | RUN cd /etc/logstash \ 22 | && wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz \ 23 | && gunzip GeoLite2-City.mmdb.gz 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 适用于 [pfSense](https://www.pfsense.org) + [Suricata](https://suricata-ids.org) 的 [ELK Stack](https://www.elastic.co/elk-stack)，能够可视化 pfSense 路由器/防火墙中的日志信息。已对 [Synology](https://www.synology.com/zh-cn) NAS 进行适配。 2 | 3 | 相关文章：[我的 pfSense 软路由配置：QoS、IDS 与流量监控](https://blanboom.org/2018/pfsense-setup/) 4 | 5 | Docker Hub 地址：https://hub.docker.com/r/blanboom/elk-suricata/ 6 | 7 | ![](https://raw.githubusercontent.com/blanboom/docker-elk-suricata/master/screenshot.jpg) 8 | 9 | ## 关于本镜像 10 | 11 | 1. 本 Docker 镜像基于 [spujadas/elk-docker](https://github.com/spujadas/elk-docker) 12 | 2. 配置文件来自：[a3ilson/pfelk](https://github.com/a3ilson/pfelk/) 13 | 3. 根据 [andig/syno-elk-docker](https://github.com/andig/syno-elk-docker) 对 Synology NAS 进行适配 14 | 15 | ## 使用方法 16 | 17 | ``` 18 | docker run -p 5601:5601 -p 9200:9200 -p 5140:5140 -p 5140:5140/udp -v /your/local/log/data/dir:/var/lib/elasticsearch --name elk blanboom/elk-suricata 19 | ``` 20 | 21 | 端口信息： 22 | 23 | - 5601 (Kibana web 界面) 24 | - 9200 (Elasticsearch JSON 接口) 25 | - 5140 (Syslog 服务器端口，可通过 TCP 与 UDP 协议接收 syslog) 26 | 27 | 此外，还需要： 28 | 29 | 1. 设置将 Suricata 的 Alert 记录在 syslog，并将 pfSense 的 syslog 发送到 5140 端口 30 | 2. 在 kibana 网页中导入 [a3ilson/pfelk](https://github.com/a3ilson/pfelk/) 中的配置 31 | 3. 对于部分网络环境，可能需要对相关文件进行修改，具体请参考 [http://pfelk.3ilson.com](http://pfelk.3ilson.com) 32 | 33 | Docker 镜像的更多使用方法，可参考：http://elk-docker.readthedocs.io 34 | -------------------------------------------------------------------------------- /pfsense2_4_3.grok: -------------------------------------------------------------------------------- 1 | # GROK Custom Patterns (add to patterns directory and reference in GROK filter for pfSense events): 2 | # GROK Patterns for pfSense 2.4.X Logging Format 3 | # 4 | # Created 12 November 2017 by Andrew Wilson 5 | # 6 | 7 | PFSENSE_LOG_ENTRY %{PFSENSE_LOG_DATA}%{PFSENSE_IP_SPECIFIC_DATA}%{PFSENSE_IP_DATA}%{PFSENSE_PROTOCOL_DATA}? 8 | PFSENSE_LOG_DATA %{INT:rule},%{INT:sub_rule}?,,%{INT:tracker},%{WORD:iface},%{WORD:reason},%{WORD:action},%{WORD:direction}, 9 | PFSENSE_IP_DATA %{INT:length},%{IP:src_ip},%{IP:dest_ip}, 10 | PFSENSE_IP_SPECIFIC_DATA %{PFSENSE_IPv4_SPECIFIC_DATA}|%{PFSENSE_IPv6_SPECIFIC_DATA} 11 | PFSENSE_IPv4_SPECIFIC_DATA (?(4)),%{BASE16NUM:tos},%{WORD:ecn}?,%{INT:ttl},%{INT:id},%{INT:offset},%{WORD:flags},%{INT:proto_id},%{WORD:proto}, 12 | PFSENSE_IPv6_SPECIFIC_DATA (?(6)),%{BASE16NUM:IPv6_Flag1},%{WORD:IPv6_Flag2},%{WORD:flow_label},%{WORD:proto_type},%{INT:proto_id}, 13 | PFSENSE_PROTOCOL_DATA %{PFSENSE_UDP_DATA}|%{PFSENSE_TCP_DATA}|%{PFSENSE_ICMP_DATA}|%{PFSENSE_IGMP_DATA}|%{PFSENSE_IPv6_VAR} 14 | PFSENSE_UDP_DATA %{INT:src_port},%{INT:dest_port},%{INT:data_length} 15 | PFSENSE_TCP_DATA %{INT:src_port},%{INT:dest_port},%{INT:data_length},%{WORD:tcp_flags},%{INT:sequence_number},%{INT:ack_number},%{INT:tcp_window},%{DATA:urg_data},%{GREEDYDATA:tcp_options} 16 | PFSENSE_IGMP_DATA datalength=%{INT:data_length} 17 | PFSENSE_ICMP_DATA %{PFSENSE_ICMP_TYPE}%{PFSENSE_ICMP_RESPONSE} 18 | PFSENSE_ICMP_TYPE (?(request|reply|unreachproto|unreachport|unreach|timeexceed|paramprob|redirect|maskreply|needfrag|tstamp|tstampreply)), 19 | PFSENSE_ICMP_RESPONSE %{PFSENSE_ICMP_ECHO_REQ_REPLY}|%{PFSENSE_ICMP_UNREACHPORT}| %{PFSENSE_ICMP_UNREACHPROTO}|%{PFSENSE_ICMP_UNREACHABLE}|%{PFSENSE_ICMP_NEED_FLAG}|%{PFSENSE_ICMP_TSTAMP}|%{PFSENSE_ICMP_TSTAMP_REPLY} 20 | PFSENSE_ICMP_ECHO_REQ_REPLY %{INT:icmp_echo_id},%{INT:icmp_echo_sequence} 21 | PFSENSE_ICMP_UNREACHPORT %{IP:icmp_unreachport_dest_ip},%{WORD:icmp_unreachport_protocol},%{INT:icmp_unreachport_port} 22 | PFSENSE_ICMP_UNREACHPROTO %{IP:icmp_unreach_dest_ip},%{WORD:icmp_unreachproto_protocol} 23 | PFSENSE_ICMP_UNREACHABLE %{GREEDYDATA:icmp_unreachable} 24 | PFSENSE_ICMP_NEED_FLAG %{IP:icmp_need_flag_ip},%{INT:icmp_need_flag_mtu} 25 | PFSENSE_ICMP_TSTAMP %{INT:icmp_tstamp_id},%{INT:icmp_tstamp_sequence} 26 | PFSENSE_ICMP_TSTAMP_REPLY %{INT:icmp_tstamp_reply_id},%{INT:icmp_tstamp_reply_sequence},%{INT:icmp_tstamp_reply_otime},%{INT:icmp_tstamp_reply_rtime},%{INT:icmp_tstamp_reply_ttime} 27 | 28 | PFSENSE_IPv6_VAR %{WORD:Type},%{WORD:Option},%{WORD:Flags},%{WORD:Flags} 29 | 30 | # DHCP (Optional) 31 | DHCPD (%{DHCPDISCOVER}|%{DHCPOFFER}|%{DHCPREQUEST}|%{DHCPACK}|%{DHCPINFORM}|%{DHCPRELEASE}) 32 | DHCPDISCOVER %{WORD:dhcp_action} from %{COMMONMAC:dhcp_client_mac}%{SPACE}(\(%{GREEDYDATA:dhcp_client_hostname}\))? via (?[0-9a-z_]*)(: %{GREEDYDATA:dhcp_load_balance})? 33 | DHCPOFFER %{WORD:dhcp_action} on %{IPV4:dhcp_client_ip} to %{COMMONMAC:dhcp_client_mac}%{SPACE}(\(%{GREEDYDATA:dhcp_client_hostname}\))? via (?[0-9a-z_]*) 34 | DHCPREQUEST %{WORD:dhcp_action} for %{IPV4:dhcp_client_ip}%{SPACE}(\(%{IPV4:dhcp_ip_unknown}\))? from %{COMMONMAC:dhcp_client_mac}%{SPACE}(\(%{GREEDYDATA:dhcp_client_hostname}\))? via (?[0-9a-z_]*)(: %{GREEDYDATA:dhcp_request_message})? 35 | DHCPACK %{WORD:dhcp_action} on %{IPV4:dhcp_client_ip} to %{COMMONMAC:dhcp_client_mac}%{SPACE}(\(%{GREEDYDATA:dhcp_client_hostname}\))? via (?[0-9a-z_]*) 36 | DHCPINFORM %{WORD:dhcp_action} from %{IPV4:dhcp_client_ip} via %(?[0-9a-z_]*) 37 | DHCPRELEASE %{WORD:dhcp_action} of %{IPV4:dhcp_client_ip} from %{COMMONMAC:dhcp_client_mac}%{SPACE}(\(%{GREEDYDATA:dhcp_client_hostname}\))? via 38 | 39 | # PFSENSE 40 | PFSENSE_CARP_DATA (%{WORD:carp_type}),(%{INT:carp_ttl}),(%{INT:carp_vhid}),(%{INT:carp_version}),(%{INT:carp_advbase}),(%{INT:carp_advskew}) 41 | PFSENSE_APP (%{DATA:pfsense_APP}): 42 | PFSENSE_APP_DATA (%{PFSENSE_APP_LOGOUT}|%{PFSENSE_APP_LOGIN}|%{PFSENSE_APP_ERROR}|%{PFSENSE_APP_GEN}) 43 | PFSENSE_APP_LOGIN (%{DATA:pfsense_ACTION}) for user \'(%{DATA:pfsense_USER})\' from: (%{GREEDYDATA:pfsense_REMOTE_IP}) 44 | PFSENSE_APP_LOGOUT User (%{DATA:pfsense_ACTION}) for user \'(%{DATA:pfsense_USER})\' from: (%{GREEDYDATA:pfsense_REMOTE_IP}) 45 | PFSENSE_APP_ERROR webConfigurator (%{DATA:pfsense_ACTION}) for \'(%{DATA:pfsense_USER})\' from (%{GREEDYDATA:pfsense_REMOTE_IP}) 46 | PFSENSE_APP_GEN (%{GREEDYDATA:pfsense_ACTION}) 47 | 48 | # SURICATA 49 | PFSENSE_SURICATA %{SPACE}\[%{NUMBER:ids_gen_id}:%{NUMBER:ids_sig_id}:%{NUMBER:ids_sig_rev}\]%{SPACE}%{GREEDYDATA:ids_desc}%{SPACE}\[Classification:%{SPACE}%{GREEDYDATA:ids_class}\]%{SPACE}\[Priority:%{SPACE}%{NUMBER:ids_pri}\]%{SPACE}{%{WORD:ids_proto}}%{SPACE}%{IP:ids_src_ip}:%{NUMBER:ids_src_port}%{SPACE}->%{SPACE}%{IP:ids_dest_ip}:%{NUMBER:ids_dest_port} 50 | -------------------------------------------------------------------------------- /screenshot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/blanboom/docker-elk-suricata/aa30a1b40817d2510cf7906727e4745b0b759aac/screenshot.jpg --------------------------------------------------------------------------------