├── logstash-forwarder.conf ├── README.md ├── logstash.conf └── patterns.conf /logstash-forwarder.conf: -------------------------------------------------------------------------------- 1 | { 2 | "network": { 3 | "servers": [ "foo.netflix.com:5043" ], 4 | "ssl ca": "C:\\Program Files (x86)\\logstash-forwarder\\logstash-forwarder.crt" 5 | }, 6 | "files": [ 7 | { 8 | "paths": [ 9 | "C:\\Program Files (x86)\\Sophos\\Reporting Interface\\Log Files\\DefaultThreats.log", 10 | "C:\\Program Files (x86)\\Sophos\\Reporting Interface\\Log Files\\Web.log" 11 | ], 12 | "fields": { "type": "syslog" } 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # logstash-configs 2 | Logstash configuration files used by Netflix. 3 | 4 | The Netflix Security Team uses the ELK stack (Elasticsearch, Logstash, Kibana) for log collection. We collect logs from a variety of security appliances and have written custom Grok filters for Logstash to parse them. We figured we'd save everyone else the time and effort and release them here. 5 | 6 | Other teams within Netflix may add their configs here if there's interest. 7 | 8 | 9 | #Grok Filters 10 | All filters are for parsing logs via syslog unless otherwise noted. 11 | 12 | ###Cyphort 13 | * File Download 14 | * File Submission 15 | * CNC Traffic 16 | * Exploit 17 | * Data Theft 18 | * Cyphort's custom timestamp 19 | 20 | ###Palo Alto Networks (BSD Syslog) 21 | * Threat Log 22 | * PAN's custom timestamp 23 | 24 | ###Sophos AV Reporting Log Writer (via Logstash-Forwarder) 25 | * EventsWebData (Web.log) 26 | * ThreatEventData (DefaultThreats.log) 27 | 28 | ###Riverbed SteelApp Traffic Manager (Formerly Stingray Traffic Manager) 29 | * Default Slave Log 30 | * Pagespeed 31 | 32 | ###Generic CEF Logs (With and without Syslog) 33 | 34 | 35 | #Config Files 36 | ###logstash.conf 37 | Main Logstash config. It references patterns.conf, which holds all of the Grok filters. 38 | 39 | Note: We're in the Pacific time zone. You'll want to modify the "timezone" lines to match your time zone. 40 | 41 | ###patterns.conf 42 | The Grok filters mentioned at the start of this document. 43 | 44 | ###logstash-forwarder.conf 45 | Config for sending logs from a Windows Sophos AV server to logstash via logstash-forwarder. 46 | 47 | 48 | #Coming Soon 49 | * Carbon Black and SentinelOne Grok filters 50 | * Configs for hoziontally scaling Logstash in AWS via ELBs, rsyslog, and Redis. 51 | -------------------------------------------------------------------------------- /logstash.conf: -------------------------------------------------------------------------------- 1 | input { 2 | tcp { 3 | port => 5140 4 | type => syslog 5 | codec => plain { 6 | charset => "ISO-8859-1" 7 | } 8 | } 9 | udp { 10 | port => 5140 11 | type => syslog 12 | codec => plain { 13 | charset => "ISO-8859-1" 14 | } 15 | } 16 | lumberjack { 17 | port => 5043 18 | ssl_certificate => "/etc/logstash/conf.d/ssl/logstash-forwarder.crt" 19 | ssl_key => "/etc/logstash/conf.d/ssl/logstash-forwarder.key" 20 | type => syslog 21 | } 22 | } 23 | 24 | 25 | filter { 26 | if [type] == "syslog" { 27 | grok { 28 | patterns_dir => "/etc/logstash/conf.d/patterns" 29 | match => [ "message", "%{CYPHORT_SYSLOG_DOWNLOAD}" ] 30 | match => [ "message", "%{CYPHORT_SYSLOG_SUBMISSION}" ] 31 | match => [ "message", "%{CYPHORT_SYSLOG_CNC}" ] 32 | match => [ "message", "%{CYPHORT_SYSLOG_EXPLOIT}" ] 33 | match => [ "message", "%{CYPHORT_SYSLOG_DATA_THEFT}" ] 34 | match => [ "message", "%{PAN_BSD_SYSLOG}" ] 35 | match => [ "message", "%{SOPHOS_WEB_LUMBERJACK}" ] 36 | match => [ "message", "%{SOPHOS_WEB_LUMBERJACK_MISSING_IP}" ] 37 | match => [ "message", "%{SOPHOS_DEFAULTTHREATS_LUMBERJACK}" ] 38 | match => [ "message", "%{SOPHOS_DEFAULTTHREATS_LUMBERJACK_MISSING_IP}" ] 39 | match => [ "message", "%{STM_WAF_DEFAULT_SLAVE_SYSLOGS}" ] 40 | match => [ "message", "%{STM_PAGESPEED_SYSLOG}" ] 41 | add_field => [ "received_at", "%{@timestamp}" ] 42 | add_field => [ "received_from", "%{host}" ] 43 | } 44 | 45 | 46 | # If the src_ip field doesn't exist (some Sophos logs) and it's an alert 47 | # we're parsing (no _grokparsefailure), put in a placeholder IP of 0.0.0.0 48 | if ( ![src_ip] and "_grokparsefailure" not in [tags] ) { 49 | mutate { add_field => { "src_ip" => "0.0.0.0" } } 50 | } 51 | 52 | 53 | # If the src_user field doesn't exist and it's an alert we're parsing 54 | # (no _grokparsefailure), put in a placeholder name of "unknown" 55 | if ( ![src_user] and "_grokparsefailure" not in [tags] ) { 56 | mutate { add_field => { "src_user" => "unknown" } } 57 | } 58 | 59 | 60 | # If the dst_user field doesn't exist and it's an alert we're parsing 61 | # (no _grokparsefailure), put in a placeholder name of "unknown" 62 | if ( ![dst_user] and "_grokparsefailure" not in [tags] ) { 63 | mutate { add_field => { "dst_user" => "unknown" } } 64 | } 65 | 66 | 67 | # Normalize "AD\foo" and "ad\foo" user names 68 | mutate { lowercase => [ "src_user" ] } 69 | mutate { lowercase => [ "dst_user" ] } 70 | 71 | 72 | syslog_pri { } 73 | 74 | 75 | date { 76 | match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "yyyy-MM-dd HH:mm:ss", "ISO8601" ] 77 | # move this to a per-host statement 78 | timezone => "America/Los_Angeles" 79 | target => [ "@timestamp" ] 80 | } 81 | 82 | 83 | date { 84 | match => [ "generation_timestamp", "yyyy/MM/dd HH:mm:ss" ] 85 | # move this to a per-host statement 86 | timezone => "America/Los_Angeles" 87 | target => [ "@timestamp" ] 88 | } 89 | 90 | 91 | date { 92 | match => [ "start_time", "yyyy-MM-dd HH:mm:ss.SSSSSSZ" ] 93 | #timezone => "UTC" 94 | #timezone => "America/Los_Angeles" 95 | target => [ "@timestamp" ] 96 | } 97 | 98 | 99 | date { 100 | match => [ "stm_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] 101 | timezone => "UTC" 102 | target => [ "@timestamp" ] 103 | } 104 | } 105 | } 106 | 107 | 108 | output { 109 | # the following line is for debugging only. outputs to /var/log/upstart/logstash.log 110 | # stdout {codec => rubydebug} 111 | elasticsearch { 112 | host => "x.x.x.x" 113 | port => 7104 114 | protocol => http 115 | index => "foolog-%{+YYYY.MM.dd}" 116 | } 117 | } -------------------------------------------------------------------------------- /patterns.conf: -------------------------------------------------------------------------------- 1 | CYPHORT_TIMESTAMP %{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND}.%{NONNEGINT}[+-]%{NONNEGINT} 2 | 3 | 4 | CYPHORT_SYSLOG_DOWNLOAD ^<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} CEF:%{INT:cef_version}\|%{DATA:vendor}\|%{DATA:product}\|%{DATA:product_version}\|%{DATA:protocol}\|%{DATA:threat_id}\|%{INT:severity}\|externalId=%{INT:external_id} eventId=%{INT:event_id} lastActivityTime=%{CYPHORT_TIMESTAMP:last_activity_time} src=%{IP:src_ip} dst=%{IP:dst_ip} fileHash=%{DATA:file_hash} fileName=%{DATA:file_name} fileType=%{DATA:file_type} startTime=%{CYPHORT_TIMESTAMP:start_time}%{GREEDYDATA:unparsed_data} 5 | 6 | 7 | CYPHORT_SYSLOG_SUBMISSION ^<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} CEF:%{INT:cef_version}\|%{DATA:vendor}\|%{DATA:product}\|%{DATA:product_version}\|%{DATA:protocol}\|%{DATA:threat_id}\|%{INT:severity}\|externalId=%{INT:external_id} eventId=%{INT:event_id} lastActivityTime=%{CYPHORT_TIMESTAMP:last_activity_time} src= dst= fileHash=%{DATA:file_hash} fileName= fileType=%{DATA:file_type} submissionTime=%{CYPHORT_TIMESTAMP:submission_time}%{GREEDYDATA:unparsed_data} 8 | 9 | 10 | CYPHORT_SYSLOG_CNC ^<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} CEF:%{INT:cef_version}\|%{DATA:vendor}\|%{DATA:product}\|%{DATA:product_version}\|%{DATA:protocol}\|%{DATA:threat_id}\|%{INT:severity}\|externalId=%{INT:external_id} eventId=%{INT:event_id} lastActivityTime=%{CYPHORT_TIMESTAMP:last_activity_time} src=%{IP:src_ip} dst=%{IP:dst_ip} malwareSeverity=%{DATA:malware_severity} malwareCategory=%{DATA:malware_category} cncServers=%{IP:src_ip}%{GREEDYDATA:unparsed_data} 11 | 12 | 13 | CYPHORT_SYSLOG_EXPLOIT ^<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} CEF:%{INT:cef_version}\|%{DATA:vendor}\|%{DATA:product}\|%{DATA:product_version}\|%{DATA:protocol}\|%{DATA:threat_id}\|%{INT:severity}\|externalId=%{INT:external_id} eventId=%{INT:event_id} lastActivityTime=%{CYPHORT_TIMESTAMP:last_activity_time} src=%{IP:src_ip} dst=%{IP:dst_ip} reqReferer=%{DATA:referer} url=%{GREEDYDATA:url} 14 | 15 | 16 | CYPHORT_SYSLOG_DATA_THEFT ^<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} CEF:%{INT:cef_version}\|%{DATA:vendor}\|%{DATA:product}\|%{DATA:product_version}\|%{DATA:protocol}\|%{DATA:threat_id}\|%{INT:severity}\|externalId=%{INT:external_id} eventId=%{INT:event_id} lastActivityTime=%{CYPHORT_TIMESTAMP:last_activity_time} src=%{IP:src_ip} dst=%{IP:dst_ip} description=%{DATA:description} port=%{INT:port} protocol=%{DATA:protocol} startTime=%{CYPHORT_TIMESTAMP:start_time}%{GREEDYDATA:unparsed_data} 17 | 18 | 19 | CEF %{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} CEF:%{INT:cef_version}\|%{DATA:vendor}\|%{DATA:product}\|%{DATA:product_version}\|%{DATA:signature_id}\|%{DATA:threat_id}\|%{DATA:severity}\|src=%{IP:src_ip} dst=%{IP:dst_ip}%{GREEDYDATA:unparsed_data} 20 | 21 | 22 | CEF_SYSLOG ^<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} CEF:%{INT:cef_version}\|%{DATA:vendor}\|%{DATA:product}\|%{DATA:product_version}\|%{DATA:signature_id}\|%{DATA:threat_id}\|%{DATA:severity}\|src=%{IP:src_ip} dst=%{IP:dst_ip}%{GREEDYDATA:unparsed_data} 23 | 24 | 25 | PAN_TIMESTAMP %{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} 26 | 27 | 28 | PAN_BSD_SYSLOG %{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:pan_reserved},%{DATA},%{WORD:serial_number},%{WORD:log_type},%{WORD:log_subtype},%{DATA:pan_unknown},%{PAN_TIMESTAMP:generation_timestamp},%{IP:src_ip},%{IP:dst_ip},%{IP:nat_src_ip},%{IP:nat_dst_ip},%{DATA:rule},%{DATA:src_user},%{DATA:dst_user},%{DATA:app},%{DATA:vsys},%{DATA:src_zone},%{DATA:dst_zone},%{DATA:ingress_int},%{DATA:egress_int},%{DATA:log_fwd_profile},%{DATA},%{DATA:session_id},%{DATA:repeat_count},%{DATA:src_port},%{DATA:dst_port},%{DATA:nat_src_port},%{DATA:nat_dst_port},%{DATA:pan_log_flags},%{DATA:ip_proto},%{DATA:action},"%{DATA:url_or_filename}",%{DATA:threat_id},%{DATA:url_category_or_wildfire_verdict},%{DATA:severity},%{DATA:direction},%{DATA:log_seq_num},%{DATA:fwd_to_panorama},%{DATA:src_country},%{DATA:dst_country},%{DATA:http_content_type},%{DATA:pcap_id},%{DATA:file_hash},%{DATA:wildfire_server},%{GREEDYDATA:unparsed_data} 29 | 30 | 31 | SOPHOS_TIMESTAMP %{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} 32 | 33 | 34 | SOPHOS_WEB_LUMBERJACK ^%{DATA}; EventID=%{INT:sophos_event_id}; EventTime=%{SOPHOS_TIMESTAMP:syslog_timestamp}; EventTypeID=%{DATA}; EventType=%{DATA:event_type}; Name=%{DATA}; ReportingName=%{DATA}; UserName=%{DATA:src_user}; ActionID=%{DATA}; Action=%{DATA:action}; SubTypeID=%{DATA}; SubType=%{DATA:sub_type}; RuleID=%{DATA:rule_id}; BlockedSite=%{DATA:url_or_filename}; ReferringURL=%{DATA:referring_url}; ComputerName=%{DATA:src_ip_dns}; ComputerDomain=%{DATA:src_ip_ad_domain}; ComputerIPAddress=%{IP:src_ip}%{GREEDYDATA:unparsed_data} 35 | 36 | 37 | SOPHOS_WEB_LUMBERJACK_MISSING_IP ^%{DATA}; EventID=%{INT:sophos_event_id}; EventTime=%{SOPHOS_TIMESTAMP:syslog_timestamp}; EventTypeID=%{DATA}; EventType=%{DATA:event_type}; Name=%{DATA}; ReportingName=%{DATA}; UserName=%{DATA:src_user}; ActionID=%{DATA}; Action=%{DATA:action}; SubTypeID=%{DATA}; SubType=%{DATA:sub_type}; RuleID=%{DATA:rule_id}; BlockedSite=%{DATA:url_or_filename}; ReferringURL=%{DATA:referring_url}; ComputerName=%{DATA:src_ip_dns}; ComputerDomain=%{DATA:src_ip_ad_domain}; ComputerIPAddress=$ 38 | 39 | 40 | SOPHOS_DEFAULTTHREATS_LUMBERJACK ^%{DATA}; EventID=%{INT:sophos_event_id}; EventTime=%{SOPHOS_TIMESTAMP:syslog_timestamp}; ActionTakenID=%{DATA}; ActionTaken=%{DATA:action}; UserName=%{DATA:src_user}; ScannerTypeID=%{DATA}; ScannerType=%{DATA:scanner_type}; StatusID=%{DATA}; Status=%{DATA:status}; ThreatTypeID=%{DATA}; ThreatType=%{DATA:threat_type}; ThreatName=%{DATA:threat_id}; FullFilePath=%{DATA:url_or_filename}; ComputerName=%{DATA:src_ip_dns}; ComputerDomain=%{DATA:src_ip_ad_domain}; ComputerIPAddress=%{IP:src_ip}%{GREEDYDATA:unparsed_data} 41 | 42 | 43 | SOPHOS_DEFAULTTHREATS_LUMBERJACK_MISSING_IP ^%{DATA}; EventID=%{INT:sophos_event_id}; EventTime=%{SOPHOS_TIMESTAMP:syslog_timestamp}; ActionTakenID=%{DATA}; ActionTaken=%{DATA:action}; UserName=%{DATA:src_user}; ScannerTypeID=%{DATA}; ScannerType=%{DATA:scanner_type}; StatusID=%{DATA}; Status=%{DATA:status}; ThreatTypeID=%{DATA}; ThreatType=%{DATA:threat_type}; ThreatName=%{DATA:threat_id}; FullFilePath=%{DATA:url_or_filename}; ComputerName=%{DATA:src_ip_dns}; ComputerDomain=%{DATA:src_ip_ad_domain}; ComputerIPAddress=$ 44 | 45 | 46 | STM_WAF_DEFAULT_SLAVE_SYSLOGS ^<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:stm_timestamp} %{SYSLOGHOST:syslog_hostname} zeusafm: "%{DATA:src_user}"\,"%{DATA:app}"\,"%{DATA:src_ip}"\,"%{DATA:unparsed_data}"\,"%{DATA:url_or_filename}"\,"%{DATA:unparsed_data}"\,"%{DATA:unparsed_data}"\,"%{DATA:action}"\,"%{DATA:protection_detection}"\,"%{DATA:unparsed_data}"\,"%{DATA:unparsed_data}"\,"%{DATA:unparsed_data}"\,"%{DATA:unparsed_data}"\,"%{DATA:unparsed_data}"\,"%{DATA:severity_level}"\,"%{DATA:unparsed_data} 47 | 48 | 49 | STM_PAGESPEED_SYSLOG ^<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:stm_timestamp} %{DATA:unparsed} vservers/%{DATA:app} %{DATA:unparsed}, %{DATA:src_ip}, %{DATA:src_country}, %{DATA:region}, %{DATA:url_or_filename}, %{DATA:action}, %{DATA:render_time}, %{GREEDYDATA:syslog_hostname} --------------------------------------------------------------------------------