├── .github └── workflows │ └── go.yml ├── .gitignore ├── .goreleaser-centos.yml ├── .goreleaser-deb.yml ├── .goreleaser-release.yml ├── .goreleaser-win.yml ├── LICENSE ├── README.md ├── alerts ├── alerts.go ├── file_writer.go ├── graylog_writer.go ├── poller.go ├── poller_test.go ├── qradar_writer.go ├── syslog_writer.go ├── syslog_writer_windows.go ├── wirter_test.go └── writer.go ├── appveyor.yml ├── client ├── account_register.go ├── account_register_test.go ├── account_status.go ├── account_status_test.go ├── alerts.go ├── alerts_test.go ├── client.go ├── client_test.go ├── events.go ├── events_dns.go ├── events_dns_test.go ├── events_http.go ├── events_ip.go ├── events_ip_test.go ├── events_tls.go ├── key_request.go ├── key_request_test.go ├── key_reset.go ├── key_reset_test.go └── mock_client.go ├── cmd ├── account.go ├── account_register.go ├── account_reset.go ├── account_status.go ├── read.go ├── root.go ├── start.go └── version.go ├── config.yml ├── config ├── config.go └── config_test.go ├── elastic ├── client.go ├── config.go ├── config_test.go ├── cursor.go ├── errors.go ├── pit.go ├── search_query.go ├── search_result.go ├── search_test.go └── transport.go ├── executor └── executor.go ├── gelf └── gelf.go ├── go.mod ├── go.sum ├── gopacket └── ssl │ └── ssl.go ├── groups ├── groups.go └── groups_test.go ├── ja3 ├── ja3.go └── ja3_test.go ├── leef └── leef.go ├── logger └── logger.go ├── logs ├── bro │ ├── parser.go │ ├── parser_test.go │ └── timestamp.go ├── edge │ ├── parser.go │ └── parser_test.go ├── msdns │ ├── parser.go │ └── parser_test.go ├── parser.go ├── pcap │ ├── pcap.log │ ├── reader.go │ └── reader_test.go ├── suricata │ ├── parser.go │ ├── parser_test.go │ └── timestamp.go └── syslognamed │ ├── parser.go │ └── parser_test.go ├── main.go ├── matchers ├── domains.go ├── domains_test.go ├── networks.go └── networks_test.go ├── nfr.service ├── packet ├── dns_packet_buffer.go ├── http_packet_buffer.go ├── ip_packet_buffer.go ├── packet.go ├── packet_buffer_test.go ├── packet_test.go ├── packet_writer.go └── packet_writer_test.go ├── scope.yml ├── scripts ├── postinst ├── postrm └── prerm ├── sniffer ├── sniffer.go ├── sniffer_test.data └── sniffer_test.go ├── utils ├── net.go ├── special_ips.go ├── special_ips_test.go ├── strings.go └── utils.go └── version └── version.go /.gitignore: -------------------------------------------------------------------------------- 1 | nfr 2 | nfr.exe 3 | *swp 4 | -------------------------------------------------------------------------------- /.goreleaser-centos.yml: -------------------------------------------------------------------------------- 1 | archives: 2 | - 3 | name_template: "{{ .ProjectName }}_{{ .Version }}_centos_{{ .Arch }}" 4 | 5 | builds: 6 | # You can have multiple builds defined as a yaml list. 7 | - 8 | id: "centos" 9 | goos: 10 | - linux 11 | # GOARCH to build for. 12 | # For more info refer to: https://golang.org/doc/install/source#environment 13 | # Defaults are 386, amd64 and arm64. 14 | goarch: 15 | - amd64 16 | ldflags: 17 | - -X github.com/alphasoc/nfr/version.Version={{ .Version }} 18 | 19 | 20 | checksum: 21 | name_template: "{{ .ProjectName }}_{{ .Version }}_checksums_centos.txt" 22 | 23 | nfpms: 24 | # note that this is an array of nfpm configs 25 | - 26 | # Replacements for GOOS and GOARCH in the package name. 27 | # Keys should be valid GOOSs or GOARCHs. 28 | # Values are the respective replacements. 29 | # Default is empty. 30 | replacements: 31 | amd64: 64-bit 32 | darwin: macOS 33 | vendor: alphasoc 34 | homepage: https://alphasoc.com/ 35 | maintainer: AlphaSOC 36 | description: A lightweight application that processes and analyzes network traffic 37 | using the AlphaSOC Analytics Engine. 38 | license: CCPL 39 | # Formats to be generated. 40 | formats: 41 | - rpm 42 | dependencies: 43 | - libpcap 44 | contents: 45 | - src: ./config.yml 46 | dst: /etc/nfr/config.yml 47 | type: "config|noreplace" 48 | - src: ./scope.yml 49 | dst: /etc/nfr/scope.yml 50 | type: "config|noreplace" 51 | - src: ./nfr.service 52 | dst: /etc/systemd/system/nfr.service 53 | type: "config|noreplace" 54 | scripts: 55 | postinstall: "scripts/postinst" 56 | preremove: "scripts/prerm" 57 | postremove: "scripts/postrm" 58 | file_name_template: "{{ .PackageName }}_{{ .Version }}_centos_{{ .Arch }}" 59 | # Override default /usr/local/bin destination for binaries. 60 | bindir: /usr/bin 61 | 62 | release: 63 | # Disable the actual release. Done elsewhere. 64 | disable: true 65 | 66 | # Sign all artifacts. 67 | signs: 68 | - artifacts: all 69 | -------------------------------------------------------------------------------- /.goreleaser-deb.yml: -------------------------------------------------------------------------------- 1 | archives: 2 | - 3 | name_template: "{{ .ProjectName }}_{{ .Version }}_debian_{{ .Arch }}" 4 | 5 | builds: 6 | # You can have multiple builds defined as a yaml list. 7 | - 8 | id: "debian" 9 | goos: 10 | - linux 11 | # GOARCH to build for. 12 | # For more info refer to: https://golang.org/doc/install/source#environment 13 | # Defaults are 386, amd64 and arm64. 14 | goarch: 15 | - amd64 16 | ldflags: 17 | - -X github.com/alphasoc/nfr/version.Version={{ .Version }} 18 | 19 | checksum: 20 | name_template: "{{ .ProjectName }}_{{ .Version }}_checksums_debian.txt" 21 | 22 | nfpms: 23 | # note that this is an array of nfpm configs 24 | - 25 | # Replacements for GOOS and GOARCH in the package name. 26 | # Keys should be valid GOOSs or GOARCHs. 27 | # Values are the respective replacements. 28 | # Default is empty. 29 | replacements: 30 | amd64: 64-bit 31 | darwin: macOS 32 | vendor: alphasoc 33 | homepage: https://alphasoc.com/ 34 | maintainer: AlphaSOC 35 | description: A lightweight application that processes and analyzes network traffic 36 | using the AlphaSOC Analytics Engine. 37 | license: CCPL 38 | # Formats to be generated. 39 | formats: 40 | - deb 41 | dependencies: 42 | - libpcap0.8 43 | contents: 44 | - src: ./config.yml 45 | dst: /etc/nfr/config.yml 46 | type: config 47 | - src: ./scope.yml 48 | dst: /etc/nfr/scope.yml 49 | type: config 50 | - src: ./nfr.service 51 | dst: /etc/systemd/system/nfr.service 52 | type: config 53 | scripts: 54 | postinstall: "scripts/postinst" 55 | preremove: "scripts/prerm" 56 | postremove: "scripts/postrm" 57 | file_name_template: "{{ .PackageName }}_{{ .Version }}_debian_{{ .Arch }}" 58 | # Override default /usr/local/bin destination for binaries. 59 | bindir: /usr/bin 60 | 61 | release: 62 | # Disable the actual release. Done elsewhere. 63 | disable: true 64 | 65 | # Sign all artifacts. 66 | signs: 67 | - artifacts: all 68 | -------------------------------------------------------------------------------- /.goreleaser-release.yml: -------------------------------------------------------------------------------- 1 | # Don't build anything. Just relelase. 2 | builds: 3 | - 4 | ignore: 5 | - goos: darwin 6 | - goos: linux 7 | - goos: windows 8 | - goos: freebsd 9 | 10 | release: 11 | # If set to auto, will mark the release as not ready for production 12 | # in case there is an indicator for this in the tag e.g. v1.0.0-rc1 13 | # If set to true, will mark the release as not ready for production. 14 | # Default is false. 15 | prerelease: auto 16 | # If set to true, will not auto-publish the release. 17 | # Default is false. 18 | draft: true 19 | # Header template for the release body. 20 | header: | 21 | ## Network Flight Recorder v{{.Version}} ({{ time "2006-02-01" }}) 22 | 23 | Welcome to this new release! 24 | 25 | # Footer template for the release body. 26 | footer: | 27 | ## Enjoy! 28 | 29 | Those were the changes on {{ .Tag }}! 30 | 31 | # You can change the name of the release. 32 | # Default is `{{.Tag}}` on OSS and `{{.PrefixedTag}}` on Pro. 33 | name_template: "{{.ProjectName}}-v{{.Version}}" 34 | # Where all the files to be released live. 35 | extra_files: 36 | - glob: ./rel/* 37 | -------------------------------------------------------------------------------- /.goreleaser-win.yml: -------------------------------------------------------------------------------- 1 | archives: 2 | - 3 | # No .Os map in checksum.name_template, so hardcoding here as well. 4 | name_template: "{{ .ProjectName }}_{{ .Version }}_windows_{{ .Arch }}" 5 | 6 | builds: 7 | # You can have multiple builds defined as a yaml list. 8 | - 9 | id: "windows" 10 | goos: 11 | - windows 12 | # GOARCH to build for. 13 | # For more info refer to: https://golang.org/doc/install/source#environment 14 | # Defaults are 386, amd64 and arm64. 15 | goarch: 16 | - amd64 17 | ldflags: 18 | - -X github.com/alphasoc/nfr/version.Version={{ .Version }} 19 | 20 | 21 | checksum: 22 | name_template: "{{ .ProjectName }}_{{ .Version }}_checksums_windows.txt" 23 | 24 | release: 25 | # Disable the actual release. Done elsewhere. 26 | disable: true 27 | 28 | # Sign all artifacts. 29 | signs: 30 | - artifacts: all 31 | -------------------------------------------------------------------------------- /alerts/alerts.go: -------------------------------------------------------------------------------- 1 | package alerts 2 | 3 | import ( 4 | "github.com/alphasoc/nfr/client" 5 | "github.com/alphasoc/nfr/groups" 6 | ) 7 | 8 | // AlertMapper maps response to internal alert struct. 9 | type AlertMapper struct { 10 | groups *groups.Groups 11 | } 12 | 13 | // Alert represents alert api struct. 14 | type Alert struct { 15 | Follow string `json:"follow"` 16 | More bool `json:"more"` 17 | Events []Event `json:"events"` 18 | } 19 | 20 | // Event from alert. 21 | type Event struct { 22 | Severity int `json:"severity"` 23 | Threats map[string]Threat `json:"threats"` 24 | 25 | Flags []string `json:"flags"` 26 | Labels []string `json:"labels,omitempty"` 27 | Groups []Group `json:"groups"` 28 | 29 | EventType string `json:"eventType"` 30 | 31 | client.EventUnified 32 | } 33 | 34 | // Threat for event. 35 | type Threat struct { 36 | Severity int `json:"severity"` 37 | Description string `json:"desc"` 38 | Policy bool `json:"policy,omitempty"` 39 | } 40 | 41 | // Group describe group event belongs to. 42 | type Group struct { 43 | Label string `json:"label"` 44 | Description string `json:"desc"` 45 | } 46 | 47 | // NewAlertMapper creates new alert mapper. 48 | func NewAlertMapper(groups *groups.Groups) *AlertMapper { 49 | return &AlertMapper{groups: groups} 50 | } 51 | 52 | // Map maps client response to alert. 53 | func (m *AlertMapper) Map(resp *client.AlertsResponse) *Alert { 54 | var alert = &Alert{ 55 | Follow: resp.Follow, 56 | More: resp.More, 57 | Events: make([]Event, len(resp.Alerts)), 58 | } 59 | 60 | for i := range resp.Alerts { 61 | ev := Event{ 62 | EventType: resp.Alerts[i].EventType, 63 | Flags: resp.Alerts[i].Wisdom.Flags, 64 | Labels: resp.Alerts[i].Wisdom.Labels, 65 | Threats: make(map[string]Threat), 66 | EventUnified: resp.Alerts[i].Event, 67 | } 68 | 69 | for _, tid := range resp.Alerts[i].Threats { 70 | threat := Threat{ 71 | Severity: resp.Threats[tid].Severity, 72 | Description: resp.Threats[tid].Title, 73 | Policy: resp.Threats[tid].Policy, 74 | } 75 | ev.Threats[tid] = threat 76 | if threat.Severity > ev.Severity { 77 | ev.Severity = threat.Severity 78 | } 79 | } 80 | 81 | for _, group := range m.groups.FindGroupsBySrcIP(resp.Alerts[i].Event.SrcIP) { 82 | ev.Groups = append(alert.Events[i].Groups, Group{ 83 | Label: group.Name, 84 | Description: group.Label, 85 | }) 86 | } 87 | 88 | alert.Events[i] = ev 89 | } 90 | 91 | return alert 92 | } 93 | -------------------------------------------------------------------------------- /alerts/file_writer.go: -------------------------------------------------------------------------------- 1 | package alerts 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | // JSONFileWriter implements Writer interface and writes alerts in json format. 8 | type FileWriter struct { 9 | f *os.File 10 | format Formatter 11 | } 12 | 13 | // NewJSONFileWriter creates new json file writer. 14 | func NewFileWriter(file string, format Formatter) (*FileWriter, error) { 15 | switch file { 16 | case "stdout": 17 | return &FileWriter{os.Stdout, format}, nil 18 | case "stderr": 19 | return &FileWriter{os.Stderr, format}, nil 20 | default: 21 | f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) 22 | if err != nil { 23 | return nil, err 24 | } 25 | return &FileWriter{f, format}, nil 26 | } 27 | } 28 | 29 | // Write writes alerts response to the file in json format. 30 | func (l *FileWriter) Write(event *Event) error { 31 | bs, err := l.format.Format(event) 32 | if err != nil { 33 | return err 34 | } 35 | 36 | for n := range bs { 37 | if _, err = l.f.Write(append(bs[n], '\n')); err != nil { 38 | return err 39 | } 40 | } 41 | 42 | return nil 43 | } 44 | 45 | // Close closes the file. 46 | func (l *FileWriter) Close() error { 47 | return l.f.Close() 48 | } 49 | -------------------------------------------------------------------------------- /alerts/graylog_writer.go: -------------------------------------------------------------------------------- 1 | package alerts 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "net/url" 7 | "os" 8 | "strconv" 9 | "strings" 10 | "time" 11 | 12 | "github.com/alphasoc/nfr/client" 13 | "github.com/alphasoc/nfr/gelf" 14 | ) 15 | 16 | // GraylogWriter implements Writer interface and write 17 | // api alerts to graylog server. 18 | type GraylogWriter struct { 19 | g *gelf.Gelf 20 | hostname string 21 | level int 22 | uriScheme string 23 | uriHost string 24 | } 25 | 26 | // NewGraylogWriter creates new graylog writer. 27 | func NewGraylogWriter(uri string, level int) (*GraylogWriter, error) { 28 | // ie. uri: tcp://localhost:12201 -> uriSchem==tcp, uriHost==localhost;12201 29 | parsedURI, err := url.Parse(uri) 30 | if err != nil { 31 | return nil, err 32 | } 33 | // Check uri validity. 34 | if _, _, err = net.SplitHostPort(parsedURI.Host); err != nil { 35 | return nil, fmt.Errorf( 36 | "failed initializing graylog output: invalid uri '%v': %v", 37 | parsedURI.Host, 38 | err) 39 | } 40 | if parsedURI.Scheme != "udp" && parsedURI.Scheme != "tcp" { 41 | return nil, fmt.Errorf( 42 | "failed initializing graylog output: unsupported scheme: %s", 43 | parsedURI.Scheme) 44 | } 45 | hostname, _ := os.Hostname() 46 | // NOTE: The actual graylog/gelf writer instance, g, is nil until Connect(). 47 | w := GraylogWriter{ 48 | g: nil, 49 | hostname: hostname, 50 | level: level, 51 | uriScheme: parsedURI.Scheme, 52 | uriHost: parsedURI.Host, 53 | } 54 | if err := w.Connect(); err != nil { 55 | return nil, err 56 | } 57 | return &w, nil 58 | } 59 | 60 | // writeAndRetry will attempt to send m to the graylog instance. On a network error, 61 | // reconnect and re-send will be attempted. Returns error. 62 | func (w *GraylogWriter) writeAndRetry(m *gelf.Message) error { 63 | // We _appear_ to have an active graylog connection. Let's try sending. 64 | if w.g != nil { 65 | // If we encounter a network error, try a reconnect. 66 | if err := w.g.Send(m); err == nil { 67 | // Success! 68 | return nil 69 | } else if _, ok := err.(net.Error); !ok { 70 | return err 71 | } else { 72 | // We have a network error. Keep going. 73 | } 74 | } 75 | // Either w.g == nil, or Send() attempt yielded a network error. Try a reconnect 76 | // and attempt another Send(). 77 | if err := w.Connect(); err != nil { 78 | return err 79 | } 80 | return w.g.Send(m) 81 | } 82 | 83 | // Write writes alert response to graylog server. 84 | func (w *GraylogWriter) Write(event *Event) error { 85 | for tid, threat := range event.Threats { 86 | m := gelf.Message{ 87 | Version: "1.1", 88 | Host: w.hostname, 89 | ShortMessage: threat.Description, 90 | Timestamp: time.Now().Unix(), 91 | Level: w.level, 92 | Extra: map[string]interface{}{ 93 | "severity": threat.Severity, 94 | "policy": strconv.FormatBool(threat.Policy), 95 | "flags": strings.Join(event.Flags, ","), 96 | "threat": tid, 97 | "engine_agent": client.DefaultUserAgent, 98 | }, 99 | } 100 | 101 | m.Extra["original_event"] = event.Timestamp.String() 102 | m.Extra["src_ip"] = event.SrcIP 103 | m.Extra["query"] = event.Query 104 | m.Extra["record_type"] = event.QueryType 105 | 106 | m.Extra["protocol"] = event.Proto 107 | m.Extra["src_port"] = event.SrcPort 108 | m.Extra["dest_ip"] = event.DestIP 109 | m.Extra["dest_port"] = event.DestPort 110 | m.Extra["bytes_in"] = event.BytesIn 111 | m.Extra["bytes_out"] = event.BytesOut 112 | m.Extra["ja3"] = event.Ja3 113 | if err := w.writeAndRetry(&m); err != nil { 114 | return err 115 | } 116 | } 117 | return nil 118 | 119 | } 120 | 121 | // Connect creates a new and connected gelf client, assigning it to w. 122 | // An error is returned. 123 | func (w *GraylogWriter) Connect() error { 124 | // Be sure to disconnect the old client; disregard the error. 125 | w.Close() 126 | // Create a new Gelf client. 127 | g, err := gelf.NewConnected(w.uriScheme, w.uriHost) 128 | if err != nil { 129 | return fmt.Errorf("connect to graylog input failed: %v", err) 130 | } 131 | // Set our gelf client. 132 | w.g = g 133 | return nil 134 | } 135 | 136 | // Close closes a connecion with the graylog server. 137 | func (w *GraylogWriter) Close() error { 138 | if w.g != nil { 139 | return w.g.Close() 140 | } 141 | return nil 142 | } 143 | -------------------------------------------------------------------------------- /alerts/poller.go: -------------------------------------------------------------------------------- 1 | // Package alerts polls and writes alerts from AlphaSOC Engine. 2 | package alerts 3 | 4 | import ( 5 | "io/ioutil" 6 | "os" 7 | "time" 8 | 9 | "github.com/alphasoc/nfr/client" 10 | ) 11 | 12 | // Poller polls alerts from AlphaSOC api and user logger 13 | // to store it into writer 14 | type Poller struct { 15 | c client.Client 16 | writers []Writer 17 | ticker *time.Ticker 18 | follow string 19 | followFile string 20 | mapper *AlertMapper 21 | } 22 | 23 | // NewPoller creates new poller base on give client and writer. 24 | func NewPoller(c client.Client, mapper *AlertMapper) *Poller { 25 | return &Poller{ 26 | c: c, 27 | writers: make([]Writer, 0), 28 | mapper: mapper, 29 | } 30 | } 31 | 32 | // AddWriter adds writer to poller. 33 | func (p *Poller) AddWriter(w Writer) { 34 | p.writers = append(p.writers, w) 35 | } 36 | 37 | // SetFollowDataFile sets file for storing follow id. 38 | // If not used then poller will be retriving all alerts from the beging. 39 | // If set then only new alerts are polled. 40 | func (p *Poller) SetFollowDataFile(file string) error { 41 | p.followFile = file 42 | 43 | // try to read existing follow id 44 | b, err := ioutil.ReadFile(file) 45 | if err != nil && !os.IsNotExist(err) { 46 | return err 47 | } 48 | p.follow = string(b) 49 | return nil 50 | } 51 | 52 | // Do polls alerts within a period specified by the interval argument. 53 | // The alerts are written to writer used to create new poller. 54 | // If the error occurrs Do method should be call again. 55 | func (p *Poller) Do(interval time.Duration) error { 56 | return p.do(interval, 0) 57 | } 58 | 59 | // do polls alerts. If maxTries <=0 then it polls forever. 60 | func (p *Poller) do(interval time.Duration, maxTries int) error { 61 | var tries = 0 62 | var more bool 63 | 64 | for { 65 | // if there is more to fetch then don't wait for ticker 66 | if !more { 67 | time.Sleep(interval) 68 | } 69 | 70 | if maxTries > 0 && tries >= maxTries { 71 | break 72 | } 73 | tries++ 74 | 75 | alerts, err := p.c.Alerts(p.follow) 76 | if err == client.ErrTooManyRequests { 77 | time.Sleep(30 * time.Second) 78 | more = true 79 | continue 80 | } else if err != nil { 81 | return err 82 | } 83 | more = alerts.More 84 | 85 | if len(alerts.Alerts) == 0 { 86 | continue 87 | } 88 | 89 | newAlerts := p.mapper.Map(alerts) 90 | 91 | for _, w := range p.writers { 92 | for _, ev := range newAlerts.Events { 93 | if err := w.Write(&ev); err != nil { 94 | return err 95 | } 96 | } 97 | } 98 | 99 | if p.follow == alerts.Follow { 100 | continue 101 | } 102 | 103 | p.follow = alerts.Follow 104 | if p.followFile != "" { 105 | if err := ioutil.WriteFile(p.followFile, []byte(p.follow), 0644); err != nil { 106 | return err 107 | } 108 | } 109 | } 110 | return nil 111 | } 112 | 113 | // stop stops poller do, by stoping ticker. 114 | func (p *Poller) stop() { 115 | if p.ticker != nil { 116 | p.ticker.Stop() 117 | p.ticker = nil 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /alerts/poller_test.go: -------------------------------------------------------------------------------- 1 | // Package alerts polls and writes alerts from AlphaSOC Engine. 2 | package alerts 3 | 4 | import ( 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | "github.com/alphasoc/nfr/client" 10 | "github.com/alphasoc/nfr/groups" 11 | ) 12 | 13 | func TestPollerDo(t *testing.T) { 14 | const fname = "_alerts" 15 | defer os.Remove(fname) 16 | 17 | w, err := NewFileWriter(fname, FormatterJSON{}) 18 | if err != nil { 19 | t.Fatal(err) 20 | } 21 | 22 | p := NewPoller(client.NewMock(), NewAlertMapper(groups.New())) 23 | p.AddWriter(w) 24 | p.follow = "1" 25 | p.do(1, 1) 26 | 27 | b, err := ioutil.ReadFile(fname) 28 | if err != nil { 29 | t.Fatal(err) 30 | } 31 | 32 | if string(b) != "" { 33 | t.Fatal("no alerts should be written to file") 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /alerts/qradar_writer.go: -------------------------------------------------------------------------------- 1 | // +build !windows,!nacl,!plan9 2 | 3 | package alerts 4 | 5 | import ( 6 | "fmt" 7 | "log/syslog" 8 | "strings" 9 | 10 | "github.com/alphasoc/nfr/leef" 11 | "github.com/alphasoc/nfr/version" 12 | ) 13 | 14 | const ( 15 | logalert syslog.Priority = 14 16 | tag = "NFR" 17 | ) 18 | 19 | // QRadarWriter implements Writer interface and write 20 | // api alerts to syslog server. 21 | type QRadarWriter struct { 22 | w *syslog.Writer 23 | } 24 | 25 | // NewQRadarWriter creates new syslog writer. 26 | func NewQRadarWriter(raddr string) (*QRadarWriter, error) { 27 | w, err := syslog.Dial("tcp", raddr, logalert, tag) 28 | if err != nil { 29 | return nil, fmt.Errorf("connect to qradar syslog input failed: %s", err) 30 | } 31 | 32 | return &QRadarWriter{w: w}, nil 33 | } 34 | 35 | // Write writes alert response to the qradar syslog input. 36 | func (w *QRadarWriter) Write(event *Event) error { 37 | for tid, threat := range event.Threats { 38 | e := leef.NewEvent() 39 | e.SetHeader("AlphaSOC", tag, strings.TrimPrefix(version.Version, "v"), tid) 40 | 41 | e.SetSevAttr(threat.Severity * 2) 42 | if threat.Policy { 43 | e.SetPolicyAttr("1") 44 | } else { 45 | e.SetPolicyAttr("0") 46 | } 47 | e.SetAttr("flags", strings.Join(event.Flags, ",")) 48 | e.SetAttr("description", threat.Description) 49 | 50 | e.SetDevTimeFormatAttr("MMM dd yyyy HH:mm:ss") 51 | e.SetDevTimeAttr(event.Timestamp.Format("Jan 02 2006 15:04:05")) 52 | e.SetProtoAttr(event.Proto) 53 | e.SetSrcAttr(event.SrcIP) 54 | e.SetSrcAttr(event.SrcIP) 55 | e.SetSrcPortAttr(int(event.SrcPort)) 56 | e.SetDstAttr(event.DestIP) 57 | e.SetDstPortAttr(int(event.DestPort)) 58 | e.SetSrcBytesAttr(int(event.BytesIn)) 59 | e.SetDstBytesAttr(int(event.BytesOut)) 60 | e.SetAttr("query", event.Query) 61 | e.SetAttr("recordType", event.QueryType) 62 | 63 | if err := w.w.Alert(e.String()); err != nil { 64 | return err 65 | } 66 | } 67 | return nil 68 | } 69 | 70 | // Close closes a connecion to the syslog server. 71 | func (w *QRadarWriter) Close() error { 72 | return w.w.Close() 73 | } 74 | -------------------------------------------------------------------------------- /alerts/syslog_writer.go: -------------------------------------------------------------------------------- 1 | // +build !windows,!nacl,!plan9 2 | 3 | package alerts 4 | 5 | import ( 6 | "fmt" 7 | "log/syslog" 8 | "net" 9 | ) 10 | 11 | // SyslogWriter implements Writer interface and write 12 | // api alerts to syslog server. 13 | type SyslogWriter struct { 14 | w *syslog.Writer 15 | f Formatter 16 | proto string 17 | raddr string 18 | } 19 | 20 | // NewSyslogWriter creates new syslog writer. 21 | func NewSyslogWriter(proto, raddr string, format Formatter) (*SyslogWriter, error) { 22 | if proto == "" { 23 | proto = "tcp" 24 | } 25 | // NOTE: The actual syslog writer instance, w, is nil until Connect(). 26 | w := SyslogWriter{w: nil, f: format, proto: proto, raddr: raddr} 27 | if err := w.Connect(); err != nil { 28 | return nil, err 29 | } 30 | return &w, nil 31 | } 32 | 33 | // writeAndRetry will attempt to log s to the syslog instance. On a network error, 34 | // reconnect and logging will be attempted. Returns error. 35 | func (w *SyslogWriter) writeAndRetry(s string) error { 36 | // We _appear_ to have an active syslog connection. Let's try logging. 37 | if w.w != nil { 38 | // If we encounter a network error, try a reconnect. 39 | if err := w.w.Alert(s); err == nil { 40 | // Success! 41 | return nil 42 | } else if _, ok := err.(net.Error); !ok { 43 | return err 44 | } else { 45 | // We have a network error. Keep going. 46 | } 47 | } 48 | // Either w.w == nil, or Alert() attempt yielded a network error. Try a reconnect 49 | // and attempt another Alert(). 50 | if err := w.Connect(); err != nil { 51 | return err 52 | } 53 | return w.w.Alert(s) 54 | } 55 | 56 | // Write writes alert response to the syslog input. 57 | func (w *SyslogWriter) Write(event *Event) error { 58 | b, err := w.f.Format(event) 59 | if err != nil { 60 | return err 61 | } 62 | for n := range b { 63 | if err := w.writeAndRetry(string(b[n])); err != nil { 64 | return err 65 | } 66 | } 67 | return nil 68 | } 69 | 70 | // Connect creates syslog server connection, assigning it in w and returns an error. 71 | func (w *SyslogWriter) Connect() error { 72 | // Close out previous connection; disregard error. 73 | w.Close() 74 | sw, err := syslog.Dial(w.proto, w.raddr, logalert, tag) 75 | if err != nil { 76 | return fmt.Errorf("connect to syslog input failed: %v", err) 77 | } 78 | // Set our syslog writer. 79 | w.w = sw 80 | return nil 81 | } 82 | 83 | // Close closes a connecion to the syslog server. 84 | func (w *SyslogWriter) Close() error { 85 | if w.w != nil { 86 | return w.w.Close() 87 | } 88 | return nil 89 | } 90 | -------------------------------------------------------------------------------- /alerts/syslog_writer_windows.go: -------------------------------------------------------------------------------- 1 | package alerts 2 | 3 | import "errors" 4 | 5 | var errSyslogNotImplemented = errors.New("syslog not implemented for this platform") 6 | 7 | // SyslogWriter implements Writer interface and write 8 | // api alerts to syslog server. 9 | type SyslogWriter struct { 10 | } 11 | 12 | // NewSyslogWriter creates new syslog writer. 13 | func NewSyslogWriter(proto, raddr string, format Formatter) (*SyslogWriter, error) { 14 | return nil, errSyslogNotImplemented 15 | } 16 | 17 | // Write writes alert response to the syslog input. 18 | func (w *SyslogWriter) Write(event *Event) error { 19 | return errSyslogNotImplemented 20 | } 21 | 22 | // Close closes a connecion to the syslog server. 23 | func (w *SyslogWriter) Close() error { 24 | return errSyslogNotImplemented 25 | } 26 | -------------------------------------------------------------------------------- /alerts/wirter_test.go: -------------------------------------------------------------------------------- 1 | package alerts 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "sort" 7 | "strings" 8 | "time" 9 | 10 | "github.com/alphasoc/nfr/client" 11 | ) 12 | 13 | func bytesToSortedStrings(bs [][]byte) []string { 14 | s := make([]string, len(bs)) 15 | for n := range bs { 16 | s[n] = string(bs[n]) 17 | } 18 | sort.Strings(s) 19 | return s 20 | } 21 | 22 | func ExampleFormatterCEF_dns() { 23 | f := NewFormatterCEF() 24 | 25 | bs, err := f.Format(&Event{ 26 | EventType: "dns", 27 | Flags: []string{"c2", "young_domain"}, 28 | Groups: []Group{Group{Label: "boston"}}, 29 | Threats: map[string]Threat{ 30 | "c2_comm": Threat{ 31 | Severity: 5, 32 | Description: "C2 communication", 33 | }, 34 | "interesting": Threat{ 35 | Severity: 2, 36 | Description: "Interesting event", 37 | }, 38 | }, 39 | EventUnified: client.EventUnified{ 40 | Timestamp: time.Unix(1536242944, 123e6).UTC(), 41 | SrcIP: net.IPv4(1, 2, 3, 4), 42 | Query: "virus.com", 43 | QueryType: "A", 44 | }, 45 | }) 46 | 47 | if err != nil { 48 | panic(err) 49 | } 50 | 51 | fmt.Print(strings.Join(bytesToSortedStrings(bs), "\n")) 52 | 53 | // Output: 54 | // CEF:0|AlphaSOC|NFR|0.0.0|c2_comm|C2 communication|10|app=dns rt=Sep 06 2018 14:09:04.123 UTC src=1.2.3.4 cs1=c2,young_domain cs1Label=flags cs2=boston cs2Label=groups query=virus.com requestMethod=A 55 | // CEF:0|AlphaSOC|NFR|0.0.0|interesting|Interesting event|4|app=dns rt=Sep 06 2018 14:09:04.123 UTC src=1.2.3.4 cs1=c2,young_domain cs1Label=flags cs2=boston cs2Label=groups query=virus.com requestMethod=A 56 | } 57 | 58 | func ExampleFormatterCEF_ip() { 59 | f := NewFormatterCEF() 60 | 61 | bs, err := f.Format(&Event{ 62 | EventType: "ip", 63 | Flags: []string{"c2", "young_domain"}, 64 | Groups: []Group{Group{Label: "boston"}}, 65 | Threats: map[string]Threat{ 66 | "c2_comm": Threat{ 67 | Severity: 5, 68 | Description: "C2 communication", 69 | }, 70 | "interesting": Threat{ 71 | Severity: 2, 72 | Description: "Interesting event", 73 | }, 74 | }, 75 | EventUnified: client.EventUnified{ 76 | Timestamp: time.Unix(1536242944, 123e6).UTC(), 77 | SrcIP: net.IPv4(1, 2, 3, 4), 78 | SrcPort: 16830, 79 | DestIP: net.IPv4(4, 3, 2, 1), 80 | DestPort: 443, 81 | Proto: "tcp", 82 | BytesIn: 744, 83 | BytesOut: 1376, 84 | }, 85 | }) 86 | 87 | if err != nil { 88 | panic(err) 89 | } 90 | 91 | fmt.Print(strings.Join(bytesToSortedStrings(bs), "\n")) 92 | 93 | // Output: 94 | // CEF:0|AlphaSOC|NFR|0.0.0|c2_comm|C2 communication|10|app=ip rt=Sep 06 2018 14:09:04.123 UTC src=1.2.3.4 cs1=c2,young_domain cs1Label=flags cs2=boston cs2Label=groups spt=16830 dst=4.3.2.1 dpt=443 proto=tcp in=744 out=1376 95 | // CEF:0|AlphaSOC|NFR|0.0.0|interesting|Interesting event|4|app=ip rt=Sep 06 2018 14:09:04.123 UTC src=1.2.3.4 cs1=c2,young_domain cs1Label=flags cs2=boston cs2Label=groups spt=16830 dst=4.3.2.1 dpt=443 proto=tcp in=744 out=1376 96 | } 97 | -------------------------------------------------------------------------------- /alerts/writer.go: -------------------------------------------------------------------------------- 1 | package alerts 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/alphasoc/nfr/version" 11 | "github.com/xoebus/ceflog" 12 | ) 13 | 14 | // Writer interface for log api alerts response. 15 | type Writer interface { 16 | Write(*Event) error 17 | } 18 | 19 | type Formatter interface { 20 | Format(*Event) ([][]byte, error) 21 | } 22 | 23 | type FormatterJSON struct { 24 | } 25 | 26 | func (FormatterJSON) Format(event *Event) ([][]byte, error) { 27 | b, err := json.Marshal(event) 28 | return [][]byte{b}, err 29 | } 30 | 31 | type FormatterCEF struct { 32 | vendor, product, version string 33 | } 34 | 35 | func NewFormatterCEF() *FormatterCEF { 36 | return &FormatterCEF{ 37 | vendor: DefaultLogVendor, 38 | product: DefaultLogProduct, 39 | version: DefaultLogVersion, 40 | } 41 | } 42 | 43 | var ( 44 | DefaultLogVendor = "AlphaSOC" 45 | DefaultLogProduct = "NFR" 46 | DefaultLogVersion = version.Version 47 | 48 | cefTimeFormat = "Jan 02 2006 15:04:05.000 MST" 49 | ) 50 | 51 | func cefCustomString(id int, label, value string) []ceflog.Pair { 52 | key := fmt.Sprintf("cs%d", id) 53 | return []ceflog.Pair{ 54 | {Key: key, Value: value}, 55 | {Key: key + "Label", Value: label}, 56 | } 57 | } 58 | 59 | func (f *FormatterCEF) Format(event *Event) ([][]byte, error) { 60 | var res [][]byte 61 | 62 | // CEF log extensions 63 | ext := ceflog.Extension{ 64 | {Key: "app", Value: event.EventType}, 65 | {Key: "rt", Value: event.Timestamp.Format(cefTimeFormat)}, 66 | {Key: "src", Value: event.SrcIP.String()}, 67 | } 68 | 69 | if v := strings.Join(event.Flags, ","); v != "" { 70 | ext = append(ext, cefCustomString(1, "flags", v)...) 71 | } 72 | if len(event.Groups) > 0 { 73 | groups := make([]string, len(event.Groups)) 74 | for n := range event.Groups { 75 | groups[n] = event.Groups[n].Label 76 | } 77 | ext = append(ext, cefCustomString(2, "groups", strings.Join(groups, ","))...) 78 | } 79 | 80 | switch event.EventType { 81 | case "dns": 82 | ext = append(ext, ceflog.Extension{ 83 | {Key: "query", Value: event.Query}, 84 | {Key: "requestMethod", Value: event.QueryType}, 85 | }...) 86 | case "ip": 87 | ext = append(ext, ceflog.Extension{ 88 | {Key: "spt", Value: strconv.Itoa(int(event.SrcPort))}, 89 | {Key: "dst", Value: event.DestIP.String()}, 90 | {Key: "dpt", Value: strconv.Itoa(int(event.DestPort))}, 91 | {Key: "proto", Value: event.Proto}, 92 | {Key: "in", Value: strconv.Itoa(int(event.BytesIn))}, 93 | {Key: "out", Value: strconv.Itoa(int(event.BytesOut))}, 94 | }...) 95 | } 96 | 97 | // Format each threat as a separate event 98 | for threatID, threat := range event.Threats { 99 | var buf bytes.Buffer 100 | l := ceflog.New(&buf, f.vendor, f.product, f.version) 101 | 102 | // write event to buffer 103 | l.LogEvent( 104 | threatID, 105 | threat.Description, 106 | ceflog.Severity(threat.Severity*2), // 0-10 scale 107 | ext) 108 | 109 | res = append(res, bytes.TrimRight(buf.Bytes(), "\n")) 110 | } 111 | 112 | return res, nil 113 | } 114 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | platform: x64 2 | 3 | matrix: 4 | fast_finish: true 5 | 6 | environment: 7 | GOPATH: c:\gopath 8 | 9 | clone_folder: c:\gopath\src\github.com\alphasoc\nfr 10 | 11 | install: 12 | # install winpcap dev 13 | - appveyor DownloadFile http://www.winpcap.org/install/bin/WpdPack_4_1_2.zip 14 | - 7z x .\WpdPack_4_1_2.zip -oc:\ 15 | # install winpcap - testing requires it 16 | - choco install winpcap 17 | 18 | cache: 19 | - C:\tools\mingw64 20 | 21 | test_script: 22 | - go test -v ./... 23 | 24 | build_script: 25 | # AppVeyor installed mingw is 32-bit only so install 64-bit version. 26 | - ps: >- 27 | if(!(Test-Path "C:\tools\mingw64")) { 28 | cinst mingw > mingw-install.txt 29 | Push-AppveyorArtifact mingw-install.txt 30 | } 31 | - set PATH=C:\tools\mingw64\bin;%PATH% 32 | - ps: | 33 | $version = git describe 34 | go build -o nfr-windows-amd64.exe -ldflags "-X github.com/alphasoc/nfr/version.Version=$version" 35 | 36 | artifacts: 37 | - path: nfr-windows-amd64.exe 38 | name: exe 39 | 40 | deploy: 41 | - release: $(APPVEYOR_REPO_TAG_NAME) 42 | provider: GitHub 43 | auth_token: 44 | secure: EjLxBojyCRGndhLcI/FWddpdjCjCzZdJctIIG/hXC6zqXg7VrpyJlfZHsTjEB6Xk 45 | artifact: exe 46 | force_update: true 47 | draft: false 48 | prerelease: false 49 | on: 50 | branch: /^v[0-9]+\.[0-9]+\.[0-9]+$/ 51 | appveyor_repo_tag: true 52 | -------------------------------------------------------------------------------- /client/account_register.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/mail" 7 | ) 8 | 9 | // AccountRegisterRequest contains information needed to 10 | // register alphasoc account and obtain API key. 11 | type AccountRegisterRequest struct { 12 | Details struct { 13 | Name string `json:"name"` 14 | Email string `json:"email"` 15 | } `json:"details"` 16 | } 17 | 18 | // AccountRegister registers new alphasoc account. 19 | func (c *AlphaSOCClient) AccountRegister(req *AccountRegisterRequest) error { 20 | if req.Details.Name == "" { 21 | return fmt.Errorf("name is required to register account") 22 | } 23 | if req.Details.Email == "" { 24 | return fmt.Errorf("email is required to register account") 25 | } 26 | 27 | email, err := mail.ParseAddress(req.Details.Email) 28 | if err != nil { 29 | return fmt.Errorf("invalid email: %s", err) 30 | } 31 | req.Details.Email = email.Address 32 | 33 | resp, err := c.post(context.Background(), "account/register", nil, req) 34 | if err != nil { 35 | return err 36 | } 37 | return resp.Body.Close() 38 | } 39 | -------------------------------------------------------------------------------- /client/account_register_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestAccountRegister(t *testing.T) { 12 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 13 | checkMethodAndPath(t, r, http.MethodPost, "/account/register") 14 | })) 15 | defer ts.Close() 16 | 17 | var accountRegisterRequest = &AccountRegisterRequest{} 18 | accountRegisterRequest.Details.Name = "test-name" 19 | accountRegisterRequest.Details.Email = "test-email@alphasoc.com" 20 | 21 | require.NoError(t, New(ts.URL, "test-key").AccountRegister(accountRegisterRequest)) 22 | } 23 | 24 | func TestAccountRegisterFail(t *testing.T) { 25 | var accountRegisterRequest = &AccountRegisterRequest{} 26 | 27 | require.Error(t, New(noopServer.URL, "test-key").AccountRegister(accountRegisterRequest)) 28 | 29 | accountRegisterRequest.Details.Name = "test-name" 30 | require.Error(t, New(noopServer.URL, "test-key").AccountRegister(accountRegisterRequest)) 31 | 32 | accountRegisterRequest.Details.Email = "test-emailalphasoc.com" 33 | require.Error(t, New(noopServer.URL, "test-key").AccountRegister(accountRegisterRequest)) 34 | } 35 | -------------------------------------------------------------------------------- /client/account_status.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "time" 8 | ) 9 | 10 | // AccountStatusResponse represents response for /account/status call. 11 | type AccountStatusResponse struct { 12 | Registered bool `json:"registered"` 13 | Expired bool `json:"expired"` 14 | Messages []struct { 15 | Level int `json:"level"` 16 | Body string `json:"body"` 17 | } `json:"messages"` 18 | } 19 | 20 | // AccountStatus returns AlphaSOC account details status. 21 | func (c *AlphaSOCClient) AccountStatus() (*AccountStatusResponse, error) { 22 | if c.key == "" { 23 | return nil, ErrNoAPIKey 24 | } 25 | ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 26 | defer cancel() 27 | resp, err := c.get(ctx, "account/status", nil) 28 | if err != nil { 29 | return nil, err 30 | } 31 | defer resp.Body.Close() 32 | 33 | var r AccountStatusResponse 34 | if err := json.NewDecoder(resp.Body).Decode(&r); err != nil { 35 | return nil, fmt.Errorf("json decoding error: %s", err) 36 | } 37 | return &r, nil 38 | } 39 | -------------------------------------------------------------------------------- /client/account_status_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestAccountStatus(t *testing.T) { 13 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 14 | checkMethodAndPath(t, r, http.MethodGet, "/account/status") 15 | json.NewEncoder(w).Encode(&AccountStatusResponse{}) 16 | })) 17 | defer ts.Close() 18 | _, err := New(ts.URL, "test-key").AccountStatus() 19 | require.NoError(t, err) 20 | } 21 | 22 | func TestAccountStatusFail(t *testing.T) { 23 | _, err := New(internalServerErrorServer.URL, "test-key").AccountStatus() 24 | require.Error(t, err) 25 | } 26 | 27 | func TestAccountStatusNoKey(t *testing.T) { 28 | _, err := New("", "").AccountStatus() 29 | require.Error(t, err) 30 | } 31 | 32 | func TestAccountStatusInvalidJSON(t *testing.T) { 33 | _, err := New(noopServer.URL, "test-key").AccountStatus() 34 | require.Error(t, err) 35 | } 36 | -------------------------------------------------------------------------------- /client/alerts.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "net" 7 | "net/url" 8 | "time" 9 | ) 10 | 11 | // AlertsResponse represents response for /events call. 12 | type AlertsResponse struct { 13 | Follow string `json:"follow"` 14 | More bool `json:"more"` 15 | After string `json:"after,omitempty"` 16 | Before string `json:"before,omitempty"` 17 | 18 | Alerts []Alert `json:"alerts,omitempty"` 19 | Threats map[string]Threat `json:"threats,omitempty"` 20 | } 21 | 22 | type EventUnified struct { 23 | // Header 24 | Timestamp time.Time `json:"ts"` 25 | SrcIP net.IP `json:"srcIP"` 26 | SrcPort uint16 `json:"srcPort,omitempty"` 27 | SrcHost string `json:"srcHost,omitempty"` 28 | SrcMac string `json:"srcMac,omitempty"` 29 | SrcUser string `json:"srcUser,omitempty"` 30 | SrcID string `json:"srcID,omitempty"` 31 | ConnID string `json:"connID,omitempty"` 32 | 33 | // Bytes transferred 34 | BytesIn int64 `json:"bytesIn,omitempty"` 35 | BytesOut int64 `json:"bytesOut,omitempty"` 36 | 37 | // DNS fields 38 | Query string `json:"query,omitempty"` 39 | QueryType string `json:"qtype,omitempty"` 40 | 41 | // IP fields 42 | DestIP net.IP `json:"destIP,omitempty"` 43 | DestPort uint16 `json:"destPort,omitempty"` 44 | Proto string `json:"proto,omitempty"` 45 | Ja3 string `json:"ja3,omitempty"` 46 | 47 | // HTTP fields 48 | URL string `json:"url,omitempty"` 49 | Method string `json:"method,omitempty"` 50 | Status int32 `json:"status,omitempty"` 51 | Action string `json:"action,omitempty"` 52 | ContentType string `json:"contentType,omitempty"` 53 | Referrer string `json:"referrer,omitempty"` 54 | UserAgent string `json:"userAgent,omitempty"` 55 | } 56 | 57 | // Alert provides result of AlphaSOC Engine analysis, which was found to be threat. 58 | type Alert struct { 59 | EventType string `json:"eventType"` 60 | Event EventUnified `json:"event"` 61 | 62 | // IPEvent IPEntry `json:"-"` 63 | // DNSEvent DNSEntry `json:"-"` 64 | // HTTPEvent HTTPEntry `json:"-"` 65 | 66 | Threats []string `json:"threats"` 67 | Wisdom struct { 68 | Flags []string `json:"flags"` 69 | Labels []string `json:"labels"` 70 | } `json:"wisdom"` 71 | } 72 | 73 | // Threat provides more details about threat, 74 | // like human-readable description. 75 | type Threat struct { 76 | Title string `json:"title"` 77 | Severity int `json:"severity"` 78 | Policy bool `json:"policy"` 79 | } 80 | 81 | // Alerts returns AlphaSOC events that informs about potential risk. 82 | func (c *AlphaSOCClient) Alerts(follow string) (*AlertsResponse, error) { 83 | if c.key == "" { 84 | return nil, ErrNoAPIKey 85 | } 86 | query := url.Values{} 87 | if follow != "" { 88 | query.Add("follow", follow) 89 | } 90 | resp, err := c.get(context.Background(), "alerts", query) 91 | if err != nil { 92 | return nil, err 93 | } 94 | defer resp.Body.Close() 95 | 96 | var r AlertsResponse 97 | if err := json.NewDecoder(resp.Body).Decode(&r); err != nil { 98 | return nil, err 99 | } 100 | 101 | return &r, nil 102 | } 103 | -------------------------------------------------------------------------------- /client/alerts_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestAlerts(t *testing.T) { 13 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 14 | checkMethodAndPath(t, r, http.MethodGet, "/alerts") 15 | json.NewEncoder(w).Encode(&AlertsResponse{}) 16 | })) 17 | defer ts.Close() 18 | 19 | _, err := New(ts.URL, "test-key").Alerts("") 20 | require.NoError(t, err) 21 | } 22 | 23 | func TestAlertsFollow(t *testing.T) { 24 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 25 | if r.URL.RawQuery != "follow=1" { 26 | t.Fatalf("invalid query %s", r.URL.Path) 27 | } 28 | json.NewEncoder(w).Encode(&AlertsResponse{}) 29 | })) 30 | defer ts.Close() 31 | 32 | _, err := New(ts.URL, "test-key").Alerts("1") 33 | require.NoError(t, err) 34 | } 35 | 36 | func TestAlertsFail(t *testing.T) { 37 | _, err := New(internalServerErrorServer.URL, "test-key").Alerts("") 38 | require.Error(t, err) 39 | } 40 | 41 | func TestAlertsNoKey(t *testing.T) { 42 | _, err := New("", "").Alerts("") 43 | require.Equal(t, ErrNoAPIKey, err) 44 | } 45 | 46 | func TestAlertsInvalidJSON(t *testing.T) { 47 | _, err := New(noopServer.URL, "test-key").Alerts("") 48 | require.Error(t, err) 49 | } 50 | -------------------------------------------------------------------------------- /client/client.go: -------------------------------------------------------------------------------- 1 | // Package client provides functions to handle AlphaSOC public API. 2 | package client 3 | 4 | import ( 5 | "bytes" 6 | "context" 7 | "encoding/json" 8 | "errors" 9 | "fmt" 10 | "io" 11 | "net/http" 12 | "net/url" 13 | "strings" 14 | 15 | "github.com/alphasoc/nfr/version" 16 | "golang.org/x/net/context/ctxhttp" 17 | ) 18 | 19 | // Client interface for AlphaSOC API. 20 | type Client interface { 21 | AccountRegister(*AccountRegisterRequest) error 22 | AccountStatus() (*AccountStatusResponse, error) 23 | Alerts(string) (*AlertsResponse, error) 24 | EventsDNS(*EventsDNSRequest) (*EventsDNSResponse, error) 25 | EventsIP(*EventsIPRequest) (*EventsIPResponse, error) 26 | EventsHTTP([]*HTTPEntry) (*EventsHTTPResponse, error) 27 | EventsTLS([]*TLSEntry) (*EventsTLSResponse, error) 28 | KeyRequest() (*KeyRequestResponse, error) 29 | KeyReset(*KeyResetRequest) error 30 | } 31 | 32 | // ErrorResponse represents AlphaSOC API error response. 33 | type ErrorResponse struct { 34 | Message string `json:"message"` 35 | } 36 | 37 | var ( 38 | // ErrNoAPIKey is returned when Client method is called without 39 | // api key set if it's required. 40 | ErrNoAPIKey = errors.New("no api key") 41 | 42 | // ErrNoRequest is returned when nil request is pass to method. 43 | ErrNoRequest = errors.New("request is empty") 44 | 45 | // ErrTooManyRequests is returned when API returns "429 Too Many Requests" 46 | ErrTooManyRequests = errors.New("too many requests to API") 47 | ) 48 | 49 | // DefaultVersion for AlphaSOC API. 50 | const DefaultVersion = "v1" 51 | 52 | // DefaultUserAgent for nfr. 53 | var DefaultUserAgent = "AlphaSOC NFR/" + strings.TrimLeft(version.Version, "v") 54 | 55 | // AlphaSOCClient handles connection to AlphaSOC server. 56 | type AlphaSOCClient struct { 57 | host string 58 | client *http.Client 59 | version string 60 | key string 61 | } 62 | 63 | // New creates new AlphaSOC client with given host. 64 | // It also sets timeout to 30 seconds. 65 | func New(host, key string) *AlphaSOCClient { 66 | return &AlphaSOCClient{ 67 | client: &http.Client{}, 68 | host: strings.TrimSuffix(host, "/"), 69 | version: DefaultVersion, 70 | key: key, 71 | } 72 | } 73 | 74 | // SetKey sets API key. 75 | func (c *AlphaSOCClient) SetKey(key string) { 76 | c.key = key 77 | } 78 | 79 | // CheckKey check if client has valid AlphaSOC key. 80 | func (c *AlphaSOCClient) CheckKey() error { 81 | _, err := c.AccountStatus() 82 | return err 83 | } 84 | 85 | // getAPIPath returns the versioned request path to call the api. 86 | // It appends the query parameters to the path if they are not empty. 87 | func (c *AlphaSOCClient) getAPIPath(path string, query url.Values) string { 88 | if query == nil { 89 | return fmt.Sprintf("%s/%s/%s", c.host, c.version, path) 90 | } 91 | return fmt.Sprintf("%s/%s/%s?%s", c.host, c.version, path, query.Encode()) 92 | } 93 | 94 | func (c *AlphaSOCClient) get(ctx context.Context, path string, query url.Values) (*http.Response, error) { 95 | return c.do(ctx, http.MethodGet, path, query, nil, nil) 96 | } 97 | 98 | func (c *AlphaSOCClient) post(ctx context.Context, path string, query url.Values, obj interface{}) (*http.Response, error) { 99 | var buffer bytes.Buffer 100 | headers := http.Header{ 101 | "Content-Type": []string{"application/json"}, 102 | } 103 | 104 | switch obj.(type) { 105 | case []byte: 106 | buffer.Write(obj.([]byte)) 107 | default: 108 | if err := json.NewEncoder(&buffer).Encode(obj); err != nil { 109 | return nil, err 110 | } 111 | } 112 | return c.do(ctx, http.MethodPost, path, query, &buffer, headers) 113 | } 114 | 115 | func (c *AlphaSOCClient) do(ctx context.Context, method, path string, query url.Values, body io.Reader, headers http.Header) (*http.Response, error) { 116 | fullPath := c.getAPIPath(path, query) 117 | req, err := http.NewRequest(method, fullPath, body) 118 | if err != nil { 119 | return nil, err 120 | } 121 | if c.key != "" { 122 | req.SetBasicAuth(c.key, "") 123 | } 124 | req.Header.Set("User-Agent", DefaultUserAgent) 125 | for key, value := range headers { 126 | req.Header[key] = value 127 | } 128 | 129 | resp, err := ctxhttp.Do(ctx, c.client, req) 130 | if err != nil { 131 | if err == context.DeadlineExceeded { 132 | return nil, fmt.Errorf("%s %s i/o timeout", method, fullPath) 133 | } 134 | return nil, err 135 | } 136 | 137 | if code := resp.StatusCode; code != http.StatusOK { 138 | defer resp.Body.Close() 139 | 140 | if code == http.StatusTooManyRequests { 141 | return nil, ErrTooManyRequests 142 | } 143 | 144 | var errorResponse ErrorResponse 145 | if err := json.NewDecoder(resp.Body).Decode(&errorResponse); err != nil { 146 | return nil, err 147 | } 148 | return nil, errors.New(errorResponse.Message) 149 | } 150 | return resp, nil 151 | } 152 | -------------------------------------------------------------------------------- /client/client_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "net/http" 7 | "net/http/httptest" 8 | "os" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | // noop server and handler for testing 15 | var ( 16 | noopServer *httptest.Server 17 | noopHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}) 18 | 19 | internalServerErrorHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 20 | w.WriteHeader(http.StatusBadRequest) 21 | json.NewEncoder(w).Encode(&ErrorResponse{"no key"}) 22 | }) 23 | internalServerErrorServer *httptest.Server 24 | ) 25 | 26 | func checkMethodAndPath(t *testing.T, r *http.Request, method string, path string) { 27 | require.Equal(t, method, r.Method, "method not found") 28 | require.Equal(t, "/"+DefaultVersion+path, r.URL.Path, "invalid url path") 29 | } 30 | 31 | func TestMain(m *testing.M) { 32 | noopServer = httptest.NewServer(noopHandler) 33 | internalServerErrorServer = httptest.NewServer(internalServerErrorHandler) 34 | 35 | defer noopServer.Close() 36 | defer internalServerErrorServer.Close() 37 | os.Exit(m.Run()) 38 | } 39 | 40 | func TestCheckKey(t *testing.T) { 41 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 42 | checkMethodAndPath(t, r, http.MethodGet, "/account/status") 43 | json.NewEncoder(w).Encode(&AccountStatusResponse{}) 44 | })) 45 | defer ts.Close() 46 | 47 | require.NoError(t, New(ts.URL, "test-key").CheckKey()) 48 | } 49 | 50 | func TestSetKey(t *testing.T) { 51 | c := New("", "") 52 | c.SetKey("test-api-key") 53 | require.Equal(t, "test-api-key", c.key) 54 | } 55 | 56 | func TestBasicAuth(t *testing.T) { 57 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 58 | if username, password, ok := r.BasicAuth(); !ok || username != "test-key" || password != "" { 59 | t.Fatalf("invalid basic auth") 60 | } 61 | })) 62 | defer ts.Close() 63 | 64 | _, err := New(ts.URL, "test-key").post(context.Background(), "/", nil, nil) 65 | require.NoError(t, err) 66 | } 67 | 68 | func TestUserAgent(t *testing.T) { 69 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 70 | require.Equal(t, DefaultUserAgent, r.UserAgent(), "invalid user agent") 71 | })) 72 | defer ts.Close() 73 | 74 | _, err := New(ts.URL, "test-key").get(context.Background(), "/", nil) 75 | require.NoError(t, err) 76 | } 77 | 78 | func TestResponseStatusNotOk(t *testing.T) { 79 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 80 | w.WriteHeader(http.StatusInternalServerError) 81 | })) 82 | defer ts.Close() 83 | 84 | _, err := New(ts.URL, "").get(context.Background(), "/", nil) 85 | require.Error(t, err) 86 | } 87 | 88 | func TestResponseErrorMessage(t *testing.T) { 89 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 90 | w.WriteHeader(http.StatusInternalServerError) 91 | json.NewEncoder(w).Encode(&ErrorResponse{Message: "test-error"}) 92 | })) 93 | defer ts.Close() 94 | 95 | _, err := New(ts.URL, "").get(context.Background(), "/", nil) 96 | require.Error(t, err) 97 | require.Equal(t, err.Error(), "test-error") 98 | } 99 | 100 | func TestDoInvalidMethod(t *testing.T) { 101 | if _, err := New("", "").do(context.Background(), "/", "/", nil, nil, nil); err == nil { 102 | t.Fatal("exptected invalid method error") 103 | } 104 | } 105 | 106 | func TestPostMarshalError(t *testing.T) { 107 | _, err := New(noopServer.URL, "").post(context.Background(), "/", nil, func() {}) 108 | require.Error(t, err, "exptected json marshal error") 109 | } 110 | 111 | func TestDoInvalidRequest(t *testing.T) { 112 | _, err := New("", "").do(context.Background(), "noop", "/", nil, nil, nil) 113 | require.Error(t, err, "exptected invalid method error") 114 | } 115 | -------------------------------------------------------------------------------- /client/events.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | // EventType defines a network event type. 4 | type EventType string 5 | 6 | // Telemetry types processed by AlphaSOC. 7 | const ( 8 | EventTypeDNS EventType = "dns" 9 | EventTypeIP EventType = "ip" 10 | EventTypeHTTP EventType = "http" 11 | EventTypeTLS EventType = "tls" 12 | ) 13 | -------------------------------------------------------------------------------- /client/events_dns.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "net" 8 | "time" 9 | ) 10 | 11 | // DNSEntry is single dns entry for analize. 12 | type DNSEntry struct { 13 | Timestamp time.Time `json:"ts"` 14 | SrcIP net.IP `json:"srcIP"` 15 | Query string `json:"query"` 16 | QType string `json:"qtype"` 17 | } 18 | 19 | // EventsDNSRequest contains slice of ip events. 20 | type EventsDNSRequest struct { 21 | Entries []*DNSEntry 22 | } 23 | 24 | // EventsDNSResponse represents response for /events/dns call. 25 | type EventsDNSResponse struct { 26 | Received int `json:"received"` 27 | Accepted int `json:"accepted"` 28 | Rejected map[string]int `json:"rejected"` 29 | } 30 | 31 | // EventsDNS sends dns queries to AlphaSOC api for analize. 32 | func (c *AlphaSOCClient) EventsDNS(req *EventsDNSRequest) (*EventsDNSResponse, error) { 33 | if c.key == "" { 34 | return nil, ErrNoAPIKey 35 | } 36 | 37 | if req == nil { 38 | return nil, ErrNoRequest 39 | } 40 | 41 | var ( 42 | buffer = &bytes.Buffer{} 43 | enc = json.NewEncoder(buffer) 44 | ) 45 | 46 | for _, entry := range req.Entries { 47 | if err := enc.Encode(entry); err != nil { 48 | return nil, err 49 | } 50 | } 51 | 52 | resp, err := c.post(context.Background(), "events/dns", nil, buffer.Bytes()) 53 | if err != nil { 54 | return nil, err 55 | } 56 | defer resp.Body.Close() 57 | 58 | var r EventsDNSResponse 59 | if err := json.NewDecoder(resp.Body).Decode(&r); err != nil { 60 | return nil, err 61 | } 62 | return &r, nil 63 | } 64 | -------------------------------------------------------------------------------- /client/events_dns_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestEventsDNS(t *testing.T) { 13 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 14 | checkMethodAndPath(t, r, http.MethodPost, "/events/dns") 15 | json.NewEncoder(w).Encode(&EventsDNSResponse{}) 16 | })) 17 | defer ts.Close() 18 | 19 | _, err := New(ts.URL, "test-key").EventsDNS(&EventsDNSRequest{Entries: []*DNSEntry{{}, {}}}) 20 | require.NoError(t, err) 21 | } 22 | 23 | func TestEventsDNSFail(t *testing.T) { 24 | _, err := New(internalServerErrorServer.URL, "test-key").EventsDNS(nil) 25 | require.Error(t, err) 26 | } 27 | 28 | func TestEventsDNSNoKey(t *testing.T) { 29 | _, err := New("", "").EventsDNS(nil) 30 | require.Equal(t, ErrNoAPIKey, err) 31 | } 32 | 33 | func TestEventsDNSInvalidJSON(t *testing.T) { 34 | _, err := New(noopServer.URL, "test-key").EventsDNS(nil) 35 | require.Error(t, err) 36 | } 37 | -------------------------------------------------------------------------------- /client/events_http.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "net" 8 | "time" 9 | ) 10 | 11 | // HTTPEntry is single dns entry for analize. 12 | type HTTPEntry struct { 13 | Timestamp time.Time `json:"ts"` 14 | SrcIP net.IP `json:"srcIP"` 15 | SrcPort uint16 `json:"srcPort"` 16 | 17 | URL string `json:"url"` 18 | Method string `json:"method"` 19 | Status int `json:"status"` 20 | Action string `json:"action"` 21 | BytesIn int64 `json:"bytesIn"` 22 | BytesOut int64 `json:"bytesOut"` 23 | 24 | // headers 25 | ContentType string `json:"contentType"` 26 | Referrer string `json:"referrer"` 27 | UserAgent string `json:"userAgent"` 28 | } 29 | 30 | // EventsHTTPResponse represents response for /events/http call. 31 | type EventsHTTPResponse struct { 32 | Received int `json:"received"` 33 | Accepted int `json:"accepted"` 34 | Rejected map[string]int `json:"rejected"` 35 | } 36 | 37 | // EventsHTTP sends http queries to AlphaSOC api for analize. 38 | func (c *AlphaSOCClient) EventsHTTP(events []*HTTPEntry) (*EventsHTTPResponse, error) { 39 | if c.key == "" { 40 | return nil, ErrNoAPIKey 41 | } 42 | 43 | if len(events) == 0 { 44 | return nil, ErrNoRequest 45 | } 46 | 47 | var ( 48 | buffer = &bytes.Buffer{} 49 | enc = json.NewEncoder(buffer) 50 | ) 51 | 52 | for _, entry := range events { 53 | if err := enc.Encode(entry); err != nil { 54 | return nil, err 55 | } 56 | } 57 | 58 | resp, err := c.post(context.Background(), "events/http", nil, buffer.Bytes()) 59 | if err != nil { 60 | return nil, err 61 | } 62 | defer resp.Body.Close() 63 | 64 | var r EventsHTTPResponse 65 | if err := json.NewDecoder(resp.Body).Decode(&r); err != nil { 66 | return nil, err 67 | } 68 | return &r, nil 69 | } 70 | -------------------------------------------------------------------------------- /client/events_ip.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "net" 8 | "time" 9 | ) 10 | 11 | // IPEntry is single ip entry for analize. 12 | type IPEntry struct { 13 | Timestamp time.Time `json:"ts"` 14 | SrcIP net.IP `json:"srcIP"` 15 | SrcPort int `json:"srcPort"` 16 | DstIP net.IP `json:"destIP"` 17 | DstPort int `json:"destPort"` 18 | Protocol string `json:"proto"` 19 | BytesIn int `json:"bytesIn"` 20 | BytesOut int `json:"bytesOut"` 21 | Ja3 string `json:"ja3"` 22 | } 23 | 24 | // EventsIPRequest contains slice of ip events. 25 | type EventsIPRequest struct { 26 | Entries []*IPEntry 27 | } 28 | 29 | // EventsIPResponse for logs/ip call. 30 | type EventsIPResponse struct { 31 | Received int `json:"received"` 32 | Accepted int `json:"accepted"` 33 | Rejected map[string]int `json:"rejected"` 34 | } 35 | 36 | // EventsIP sends ip events to AlphaSOC engine for analize. 37 | func (c *AlphaSOCClient) EventsIP(req *EventsIPRequest) (*EventsIPResponse, error) { 38 | if c.key == "" { 39 | return nil, ErrNoAPIKey 40 | } 41 | 42 | if req == nil { 43 | return nil, ErrNoRequest 44 | } 45 | 46 | var ( 47 | buffer = &bytes.Buffer{} 48 | enc = json.NewEncoder(buffer) 49 | ) 50 | 51 | for _, entry := range req.Entries { 52 | if err := enc.Encode(entry); err != nil { 53 | return nil, err 54 | } 55 | } 56 | 57 | resp, err := c.post(context.Background(), "events/ip", nil, buffer.Bytes()) 58 | if err != nil { 59 | return nil, err 60 | } 61 | defer resp.Body.Close() 62 | 63 | var r EventsIPResponse 64 | if err := json.NewDecoder(resp.Body).Decode(&r); err != nil { 65 | return nil, err 66 | } 67 | return &r, nil 68 | } 69 | -------------------------------------------------------------------------------- /client/events_ip_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestEventsIP(t *testing.T) { 13 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 14 | checkMethodAndPath(t, r, http.MethodPost, "/events/ip") 15 | json.NewEncoder(w).Encode(&EventsIPResponse{}) 16 | })) 17 | defer ts.Close() 18 | 19 | _, err := New(ts.URL, "test-key").EventsIP(&EventsIPRequest{}) 20 | require.NoError(t, err) 21 | } 22 | 23 | func TestEventsIPFail(t *testing.T) { 24 | _, err := New(internalServerErrorServer.URL, "test-key").EventsIP(nil) 25 | require.Error(t, err) 26 | } 27 | 28 | func TestEventsIPNoKey(t *testing.T) { 29 | _, err := New("", "").EventsIP(nil) 30 | require.Error(t, err) 31 | } 32 | 33 | func TestEventsIPInvalidJSON(t *testing.T) { 34 | _, err := New(noopServer.URL, "test-key").EventsIP(nil) 35 | require.Error(t, err) 36 | } 37 | -------------------------------------------------------------------------------- /client/events_tls.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "net" 8 | "time" 9 | ) 10 | 11 | // TLSEntry is single dns entry for analize. 12 | type TLSEntry struct { 13 | Timestamp time.Time `json:"ts"` 14 | SrcIP net.IP `json:"srcIP"` 15 | SrcPort uint16 `json:"srcPort"` 16 | 17 | DstIP net.IP `json:"destIP,omitempty"` 18 | DstPort uint16 `json:"destPort,omitempty"` 19 | 20 | CertHash string `json:"certHash,omitempty"` 21 | Issuer string `json:"issuer,omitempty"` 22 | Subject string `json:"subject,omitempty"` 23 | ValidFrom time.Time `json:"validFrom,omitempty"` 24 | ValidTo time.Time `json:"validTo,omitempty"` 25 | JA3 string `json:"ja3,omitempty"` 26 | JA3s string `json:"ja3s,omitempty"` 27 | } 28 | 29 | // EventsHTTPResponse represents response for /events/http call. 30 | type EventsTLSResponse struct { 31 | Received int `json:"received"` 32 | Accepted int `json:"accepted"` 33 | Rejected map[string]int `json:"rejected"` 34 | } 35 | 36 | // EventsHTTP sends tls events to AlphaSOC api for analize. 37 | func (c *AlphaSOCClient) EventsTLS(events []*TLSEntry) (*EventsTLSResponse, error) { 38 | if c.key == "" { 39 | return nil, ErrNoAPIKey 40 | } 41 | 42 | if len(events) == 0 { 43 | return nil, ErrNoRequest 44 | } 45 | 46 | var ( 47 | buffer = &bytes.Buffer{} 48 | enc = json.NewEncoder(buffer) 49 | ) 50 | 51 | for _, entry := range events { 52 | if err := enc.Encode(entry); err != nil { 53 | return nil, err 54 | } 55 | } 56 | 57 | resp, err := c.post(context.Background(), "events/tls", nil, buffer.Bytes()) 58 | if err != nil { 59 | return nil, err 60 | } 61 | defer resp.Body.Close() 62 | 63 | var r EventsTLSResponse 64 | if err := json.NewDecoder(resp.Body).Decode(&r); err != nil { 65 | return nil, err 66 | } 67 | return &r, nil 68 | } 69 | -------------------------------------------------------------------------------- /client/key_request.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "runtime" 8 | ) 9 | 10 | // KeyRequestResponse represents response for /key/request call 11 | type KeyRequestResponse struct { 12 | Key string `json:"key"` 13 | } 14 | 15 | // KeyRequestRequest contasin information needed for key registration. 16 | type KeyRequestRequest struct { 17 | Platform struct { 18 | Name string `json:"name"` 19 | } `json:"platform"` 20 | Token string `json:"token"` 21 | } 22 | 23 | // KeyRequest returns new AlphaSOC account key. 24 | func (c *AlphaSOCClient) KeyRequest() (*KeyRequestResponse, error) { 25 | var req KeyRequestRequest 26 | req.Platform.Name = fmt.Sprintf("nfr-%s-%s", runtime.GOOS, runtime.GOARCH) 27 | resp, err := c.post(context.Background(), "key/request", nil, &req) 28 | if err != nil { 29 | return nil, err 30 | } 31 | defer resp.Body.Close() 32 | 33 | var r KeyRequestResponse 34 | if err := json.NewDecoder(resp.Body).Decode(&r); err != nil { 35 | return nil, err 36 | } 37 | return &r, nil 38 | } 39 | -------------------------------------------------------------------------------- /client/key_request_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "net/http/httptest" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestKeyRequest(t *testing.T) { 13 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 14 | checkMethodAndPath(t, r, http.MethodPost, "/key/request") 15 | json.NewEncoder(w).Encode(&KeyRequestResponse{}) 16 | })) 17 | defer ts.Close() 18 | 19 | _, err := New(ts.URL, "").KeyRequest() 20 | require.NoError(t, err) 21 | } 22 | 23 | func TestKeyRequestFail(t *testing.T) { 24 | _, err := New(internalServerErrorServer.URL, "").KeyRequest() 25 | require.Error(t, err) 26 | } 27 | 28 | func TestKeyRequestInvalidJSON(t *testing.T) { 29 | _, err := New(noopServer.URL, "").KeyRequest() 30 | require.Error(t, err) 31 | } 32 | -------------------------------------------------------------------------------- /client/key_reset.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import "context" 4 | 5 | // KeyResetRequest contasin information needed for key reset. 6 | type KeyResetRequest struct { 7 | Email string `json:"email"` 8 | } 9 | 10 | // KeyReset reset AlphaSOC account key. 11 | func (c *AlphaSOCClient) KeyReset(req *KeyResetRequest) error { 12 | resp, err := c.post(context.Background(), "key/reset", nil, req) 13 | if err != nil { 14 | return err 15 | } 16 | resp.Body.Close() 17 | return nil 18 | } 19 | -------------------------------------------------------------------------------- /client/key_reset_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestKeyReset(t *testing.T) { 12 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 13 | checkMethodAndPath(t, r, http.MethodPost, "/key/reset") 14 | })) 15 | defer ts.Close() 16 | 17 | require.NoError(t, New(ts.URL, "").KeyReset(&KeyResetRequest{})) 18 | } 19 | 20 | func TestResetFail(t *testing.T) { 21 | require.Error(t, New(internalServerErrorServer.URL, "").KeyReset(nil)) 22 | } 23 | -------------------------------------------------------------------------------- /client/mock_client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | // MockAlphaSOCClient creates Client for testing. 4 | type MockAlphaSOCClient struct{} 5 | 6 | // NewMock creates new AlphaSOC mock client for testing. 7 | func NewMock() Client { 8 | return &MockAlphaSOCClient{} 9 | } 10 | 11 | // AccountRegister mock. 12 | func (c *MockAlphaSOCClient) AccountRegister(req *AccountRegisterRequest) error { 13 | return nil 14 | } 15 | 16 | // AccountStatus mock. 17 | func (c *MockAlphaSOCClient) AccountStatus() (*AccountStatusResponse, error) { 18 | return &AccountStatusResponse{}, nil 19 | } 20 | 21 | // Alerts mock. 22 | func (c *MockAlphaSOCClient) Alerts(follow string) (*AlertsResponse, error) { 23 | return &AlertsResponse{}, nil 24 | } 25 | 26 | // EventsDNS mock. 27 | func (c *MockAlphaSOCClient) EventsDNS(req *EventsDNSRequest) (*EventsDNSResponse, error) { 28 | return &EventsDNSResponse{}, nil 29 | } 30 | 31 | // EventsIP mock. 32 | func (c *MockAlphaSOCClient) EventsIP(req *EventsIPRequest) (*EventsIPResponse, error) { 33 | return &EventsIPResponse{}, nil 34 | } 35 | 36 | func (c *MockAlphaSOCClient) EventsHTTP(req []*HTTPEntry) (*EventsHTTPResponse, error) { 37 | return &EventsHTTPResponse{}, nil 38 | } 39 | 40 | func (c *MockAlphaSOCClient) EventsTLS(req []*TLSEntry) (*EventsTLSResponse, error) { 41 | return &EventsTLSResponse{}, nil 42 | } 43 | 44 | // KeyRequest mock. 45 | func (c *MockAlphaSOCClient) KeyRequest() (*KeyRequestResponse, error) { 46 | return &KeyRequestResponse{}, nil 47 | } 48 | 49 | // KeyReset mock. 50 | func (c *MockAlphaSOCClient) KeyReset(req *KeyResetRequest) error { 51 | return nil 52 | } 53 | -------------------------------------------------------------------------------- /cmd/account.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import "github.com/spf13/cobra" 4 | 5 | func newAccountCommand() *cobra.Command { 6 | var cmd = &cobra.Command{ 7 | Use: "account", 8 | Short: "Manage AlphaSOC account", 9 | } 10 | cmd.AddCommand(newAccountStatusCommand()) 11 | cmd.AddCommand(newAccountRegisterCommand()) 12 | cmd.AddCommand(newAccountKeyResetCommand()) 13 | return cmd 14 | } 15 | -------------------------------------------------------------------------------- /cmd/account_register.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/alphasoc/nfr/client" 8 | "github.com/alphasoc/nfr/config" 9 | "github.com/alphasoc/nfr/utils" 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | func newAccountRegisterCommand() *cobra.Command { 14 | var ( 15 | key string 16 | host string 17 | cmd = &cobra.Command{ 18 | Use: "register", 19 | Short: "Generate an API key via the licensing server", 20 | Long: "This command provides interactive API key generation and registration.", 21 | RunE: func(cmd *cobra.Command, args []string) error { 22 | cfg := config.NewDefault() 23 | cfg.Engine.Host, cfg.Engine.APIKey = host, key 24 | c := client.New(host, key) 25 | 26 | // do not send error to log output, print on console for user 27 | if err := register(cfg, c); err != nil { 28 | fmt.Fprintf(os.Stderr, "%s\n", err) 29 | os.Exit(1) 30 | } 31 | return nil 32 | }, 33 | } 34 | ) 35 | cmd.Flags().StringVar(&host, "host", "https://api.alphasoc.net", "AlphaSOC Engine host") 36 | cmd.Flags().StringVar(&key, "key", "", "AlphaSOC API key") 37 | return cmd 38 | } 39 | 40 | func register(cfg *config.Config, c *client.AlphaSOCClient) error { 41 | if cfg.Engine.APIKey != "" { 42 | c.SetKey(cfg.Engine.APIKey) 43 | fmt.Printf("Using key %s for registration\n", utils.ShadowKey(cfg.Engine.APIKey)) 44 | } 45 | 46 | if status, err := c.AccountStatus(); err == nil && status.Registered { 47 | return fmt.Errorf("Account is already registered") 48 | } 49 | 50 | fmt.Printf(`Please provide your details to generate an AlphaSOC API key. 51 | A valid email address is required for activation purposes. 52 | 53 | By performing this request you agree to our Terms of Service and Privacy Policy 54 | (https://www.alphasoc.com/terms-of-service) 55 | `) 56 | details, err := utils.GetAccountRegisterDetails() 57 | if err != nil { 58 | return err 59 | } 60 | 61 | if cfg.Engine.APIKey == "" { 62 | keyReq, err2 := c.KeyRequest() 63 | if err2 != nil { 64 | fmt.Fprintln(os.Stderr) 65 | return err2 66 | } 67 | c.SetKey(keyReq.Key) 68 | cfg.Engine.APIKey = keyReq.Key 69 | } 70 | 71 | var errSave = cfg.Save(configPath) 72 | if err != nil { 73 | fmt.Fprintf(os.Stderr, ` 74 | Unable to create /etc/nfr/config.yml. Please manually set up the directory and configuration file. 75 | 76 | alphasoc: 77 | api_key: %s 78 | 79 | `, cfg.Engine.APIKey) 80 | } else { 81 | fmt.Println("\nSuccess! The configuration has been written to /etc/nfr/config.yml") 82 | } 83 | 84 | req := &client.AccountRegisterRequest{Details: struct { 85 | Name string `json:"name"` 86 | Email string `json:"email"` 87 | }{ 88 | Name: details.Name, 89 | Email: details.Email, 90 | }} 91 | if err := c.AccountRegister(req); err != nil { 92 | if errSave != nil { 93 | fmt.Fprintf(os.Stderr, `We were unable to register your account. Please run nfr again with following command: 94 | 95 | $ nfr account register --key %s 96 | `, cfg.Engine.APIKey) 97 | return err 98 | } 99 | 100 | fmt.Fprintf(os.Stderr, `We were unable to register your account. Please run nfr again with following command: 101 | 102 | $ nfr account register 103 | `) 104 | return err 105 | } 106 | 107 | fmt.Println("Next, check your email and click the verification link to activate your API key.") 108 | return nil 109 | } 110 | -------------------------------------------------------------------------------- /cmd/account_reset.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "net/mail" 6 | 7 | "github.com/alphasoc/nfr/client" 8 | "github.com/spf13/cobra" 9 | ) 10 | 11 | func newAccountKeyResetCommand() *cobra.Command { 12 | var cmd = &cobra.Command{ 13 | Use: "reset", 14 | Short: "Reset the API key associated with a given email address", 15 | RunE: func(cmd *cobra.Command, args []string) error { 16 | if len(args) != 1 { 17 | return fmt.Errorf("email is required") 18 | } 19 | 20 | address, err := mail.ParseAddress(args[0]) 21 | if err != nil { 22 | return fmt.Errorf("invalid email %s", args[0]) 23 | } 24 | 25 | _, c, err := createConfigAndClient(false) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | return accountKeyReset(c, address.Address) 31 | }, 32 | } 33 | 34 | return cmd 35 | } 36 | 37 | func accountKeyReset(c client.Client, email string) error { 38 | if err := c.KeyReset(&client.KeyResetRequest{Email: email}); err != nil { 39 | return err 40 | } 41 | 42 | fmt.Println("Check your email and click the reset link to get new API key") 43 | return nil 44 | } 45 | -------------------------------------------------------------------------------- /cmd/account_status.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/cobra" 7 | 8 | "github.com/alphasoc/nfr/client" 9 | ) 10 | 11 | func newAccountStatusCommand() *cobra.Command { 12 | var cmd = &cobra.Command{ 13 | Use: "status", 14 | Short: "Show the status of your AlphaSOC API key and license", 15 | RunE: func(cmd *cobra.Command, args []string) error { 16 | _, c, err := createConfigAndClient(false) 17 | if err != nil { 18 | return err 19 | } 20 | return accountStatus(c) 21 | }, 22 | } 23 | return cmd 24 | } 25 | 26 | func accountStatus(c client.Client) error { 27 | status, err := c.AccountStatus() 28 | if err != nil { 29 | return fmt.Errorf("get account status failed: %s", err) 30 | } 31 | 32 | fmt.Printf("Account registered: %t\n", status.Registered) 33 | fmt.Printf("Account expired: %t\n", status.Expired) 34 | return nil 35 | } 36 | -------------------------------------------------------------------------------- /cmd/read.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | 8 | log "github.com/Sirupsen/logrus" 9 | "github.com/alphasoc/nfr/client" 10 | "github.com/alphasoc/nfr/config" 11 | "github.com/alphasoc/nfr/executor" 12 | "github.com/alphasoc/nfr/utils" 13 | "github.com/spf13/cobra" 14 | ) 15 | 16 | var ( 17 | fileFormats = []string{"bro", "msdns", "pcap", "suricata", "syslog-named", "edge"} 18 | analyzeTypes = []string{"all", "dns", "ip", "http"} 19 | ) 20 | 21 | func newReadCommand() *cobra.Command { 22 | var ( 23 | fileFormat string 24 | fileType string 25 | ) 26 | 27 | var cmd = &cobra.Command{ 28 | Use: "read", 29 | Short: "Process network events stored on disk in known formats", 30 | Long: `Read file in pcap fromat and send DNS queries to AlphaSOC for analyze 31 | The queries could be save to file via tools like tcpdump, bro or suricata. 32 | See nfr read --help for more informations.`, 33 | RunE: func(cmd *cobra.Command, args []string) error { 34 | if fileFormat == "" { 35 | return errors.New("file format required") 36 | } 37 | 38 | if !utils.StringsContains(fileFormats, fileFormat) { 39 | return fmt.Errorf("unknown %s file format", fileFormat) 40 | } 41 | 42 | if len(args) == 0 { 43 | return errors.New("at least 1 file required") 44 | } 45 | 46 | cfg, c, err := createConfigAndClient(true) 47 | if err != nil { 48 | return err 49 | } 50 | return send(cfg, c, fileFormat, fileType, args) 51 | }, 52 | } 53 | cmd.Flags().StringVarP(&fileFormat, "format", "f", "pcap", fmt.Sprintf("One of %s file format", sprintSlice(fileFormats))) 54 | cmd.Flags().StringVar(&fileType, "type", "all", fmt.Sprintf("One of %s type to analyze", sprintSlice(analyzeTypes))) 55 | return cmd 56 | } 57 | 58 | func send(cfg *config.Config, c client.Client, fileFormat, fileType string, files []string) error { 59 | e, err := executor.New(c, cfg) 60 | if err != nil { 61 | return err 62 | } 63 | 64 | for i := range files { 65 | if err := e.Send(files[i], fileFormat, fileType); err != nil { 66 | return err 67 | } 68 | log.Infof("file %s processed\n", files[i]) 69 | } 70 | return nil 71 | } 72 | 73 | // sprintSlice is a helper to pretty print slice in command help. 74 | func sprintSlice(s []string) string { 75 | ret := fmt.Sprintf("%s", s) 76 | return strings.Replace(ret, " ", "|", -1) 77 | } 78 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | "path" 6 | "runtime" 7 | 8 | "github.com/alphasoc/nfr/client" 9 | "github.com/alphasoc/nfr/config" 10 | "github.com/alphasoc/nfr/logger" 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | var ( 15 | configDefaultLocation string // default location for config file. 16 | configPath string // config path flag 17 | ) 18 | 19 | func init() { 20 | base := "/etc/" 21 | if runtime.GOOS == "windows" { 22 | base = os.Getenv("APPDATA") 23 | } 24 | configDefaultLocation = path.Join(base, "nfr", "config.yml") 25 | } 26 | 27 | // NewRootCommand represents the base command when called without any subcommands 28 | func NewRootCommand() *cobra.Command { 29 | var cmd = &cobra.Command{ 30 | Use: "nfr account|listen|read|version", 31 | Short: "nfr is main command used to send dns and ip events to AlphaSOC Engine", 32 | Long: `Network Flight Recorder (NFR) is an application which captures network traffic 33 | and provides deep analysis and alerting of suspicious events, identifying gaps 34 | in your security controls, highlighting targeted attacks and policy violations.`, 35 | SilenceErrors: true, 36 | SilenceUsage: true, 37 | } 38 | 39 | cmd.PersistentFlags().StringVarP(&configPath, "config", "c", configDefaultLocation, "Config path for nfr") 40 | 41 | cmd.AddCommand(newVersionCommand()) 42 | cmd.AddCommand(newAccountCommand()) 43 | cmd.AddCommand(newStartCommand()) 44 | cmd.AddCommand(newReadCommand()) 45 | return cmd 46 | } 47 | 48 | // createConfigAndClient takes one argument to check if key is active. 49 | func createConfigAndClient(checkKey bool) (*config.Config, *client.AlphaSOCClient, error) { 50 | cfg, err := config.New(configPath) 51 | if err != nil { 52 | return nil, nil, err 53 | } 54 | 55 | if err := logger.SetOutput(cfg.Log.File); err != nil { 56 | return nil, nil, err 57 | } 58 | logger.SetLevel(cfg.Log.Level) 59 | 60 | c := client.New(cfg.Engine.Host, cfg.Engine.APIKey) 61 | if checkKey { 62 | if err := c.CheckKey(); err != nil { 63 | return nil, nil, err 64 | } 65 | } 66 | return cfg, c, nil 67 | } 68 | -------------------------------------------------------------------------------- /cmd/start.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/alphasoc/nfr/client" 5 | "github.com/alphasoc/nfr/config" 6 | "github.com/alphasoc/nfr/executor" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | func newStartCommand() *cobra.Command { 11 | var cmd = &cobra.Command{ 12 | Use: "start", 13 | Short: "Start processing network events (inputs defined in config)", 14 | Long: `Start processing network events. API key must be set before calling this mode.`, 15 | RunE: func(cmd *cobra.Command, args []string) error { 16 | cfg, c, err := createConfigAndClient(true) 17 | if err != nil { 18 | return err 19 | } 20 | return start(c, cfg) 21 | }, 22 | } 23 | return cmd 24 | } 25 | 26 | func start(c client.Client, cfg *config.Config) error { 27 | e, err := executor.New(c, cfg) 28 | if err != nil { 29 | return err 30 | } 31 | return e.Start() 32 | } 33 | -------------------------------------------------------------------------------- /cmd/version.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/alphasoc/nfr/version" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | func newVersionCommand() *cobra.Command { 11 | return &cobra.Command{ 12 | Use: "version", 13 | Short: "Show the NFR binary version", 14 | Run: printversion, 15 | } 16 | } 17 | 18 | func printversion(cmd *cobra.Command, args []string) { 19 | fmt.Printf("nfr version %s\n", version.Version) 20 | } 21 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "io/ioutil" 5 | "log" 6 | "os" 7 | "path" 8 | "testing" 9 | "time" 10 | 11 | "github.com/pkg/errors" 12 | "gopkg.in/yaml.v3" 13 | ) 14 | 15 | func checkDefaults(t *testing.T, cfg *Config) { 16 | if cfg.Engine.Host != "https://api.alphasoc.net" { 17 | t.Fatalf("invalid alphasoc host - got %v; expected %v", cfg.Engine.Host, "https://api.alphasoc.net") 18 | } 19 | if !cfg.Engine.Analyze.DNS { 20 | t.Fatalf("analyze dns set to false") 21 | } 22 | if !cfg.Engine.Analyze.IP { 23 | t.Fatalf("analyze ip set to false") 24 | } 25 | if cfg.Inputs.Sniffer.Enabled { 26 | t.Fatalf("sniffer is enabled") 27 | } 28 | if cfg.Outputs.File != "stderr" { 29 | t.Fatalf("invalid output file - got %s; expected %s", cfg.Outputs.File, "stderr") 30 | } 31 | if cfg.Outputs.Syslog.Port != 514 { 32 | t.Fatalf("invalid output syslog port - got %d; expected %d", cfg.Outputs.Syslog.Port, 514) 33 | } 34 | if cfg.Engine.Alerts.PollInterval != 5*time.Minute { 35 | t.Fatalf("invalid events poll interval - got %s; expected %s", cfg.Engine.Alerts.PollInterval, 5*time.Minute) 36 | } 37 | if cfg.Log.File != "stdout" { 38 | t.Fatalf("invalid log file - got %s; expected %s", cfg.Log.File, "stdout") 39 | } 40 | if cfg.Log.Level != "info" { 41 | t.Fatalf("invalid log level - got %s; expected %s", cfg.Log.Level, "info") 42 | } 43 | if cfg.DNSEvents.BufferSize != 65535 { 44 | t.Fatalf("invalid dns queries buffer size - got %d; expected %d", cfg.DNSEvents.BufferSize, 65535) 45 | } 46 | if cfg.DNSEvents.FlushInterval != 30*time.Second { 47 | t.Fatalf("invalid dns queries flush interval - got %s; expected %s", cfg.DNSEvents.FlushInterval, 30*time.Second) 48 | } 49 | if cfg.IPEvents.BufferSize != 65535 { 50 | t.Fatalf("invalid ip events buffer size - got %d; expected %d", cfg.IPEvents.BufferSize, 65535) 51 | } 52 | if cfg.IPEvents.FlushInterval != 30*time.Second { 53 | t.Fatalf("invalid ip events flush interval - got %s; expected %s", cfg.IPEvents.FlushInterval, 30*time.Second) 54 | } 55 | if l := len(cfg.ScopeConfig.Groups); l != 1 { 56 | t.Fatalf("invalid number of scope groups - got %d; expected %d", l, 1) 57 | } 58 | group, ok := cfg.ScopeConfig.Groups["default"] 59 | if !ok { 60 | t.Fatalf("no default scope group") 61 | } 62 | if l := len(group.InScope); l != 4 { 63 | t.Fatalf("invalid number of source networks in default scope group - got %d; expected %d", l, 4) 64 | } 65 | if l := len(group.TrustedIps); l != 10 { 66 | t.Fatalf("invalid number of trusted ips in default scope group - got %d; expected %d", l, 10) 67 | } 68 | if l := len(group.TrustedDomains); l != 4 { 69 | t.Fatalf("invalid number of excluded domains in default scope group - got %d; expected %d", l, 4) 70 | } 71 | } 72 | 73 | func TestDefaultConfig(t *testing.T) { 74 | cfg := NewDefault() 75 | cfg.loadScopeConfig() 76 | checkDefaults(t, cfg) 77 | } 78 | 79 | func TestReadConfig(t *testing.T) { 80 | var content = []byte(` 81 | engine: 82 | host: https://api.alphasoc.net 83 | api_key: test-api-key 84 | alerts: 85 | poll_interval: 5m 86 | log: 87 | file: stdout 88 | level: info 89 | data: 90 | file: nfr.data 91 | outputs: 92 | enabled: true 93 | file: stderr 94 | dns_events: 95 | buffer_size: 65535 96 | flush_interval: 30s 97 | failed: 98 | file: nfr.pcap`) 99 | f, err := ioutil.TempFile("", "nfr-config") 100 | if err != nil { 101 | log.Fatal(err) 102 | } 103 | defer os.Remove(f.Name()) 104 | defer f.Close() 105 | 106 | if _, err = f.Write(content); err != nil { 107 | log.Fatal(err) 108 | } 109 | 110 | cfg, err := New(f.Name()) 111 | if err != nil { 112 | t.Fatal(err) 113 | } 114 | checkDefaults(t, cfg) 115 | if cfg.Engine.APIKey != "test-api-key" { 116 | t.Fatalf("invalid alphasoc api key - got %s; expected %s", cfg.Engine.APIKey, "test-api-key") 117 | } 118 | } 119 | 120 | func TestReadConfigStrict(t *testing.T) { 121 | // apiKey instead of api_key 122 | var content = []byte(` 123 | engine: 124 | host: https://api.alphasoc.net 125 | apiKey: test-api-key`) 126 | 127 | f, err := os.OpenFile(path.Join(t.TempDir(), "nfr-config"), os.O_RDWR|os.O_CREATE, 0755) 128 | if err != nil { 129 | t.Fatal(err) 130 | } 131 | defer f.Close() 132 | 133 | if _, err := f.Write(content); err != nil { 134 | t.Fatal(err) 135 | } 136 | 137 | var want *yaml.TypeError 138 | if _, err := New(f.Name()); !errors.As(err, &want) { 139 | t.Errorf("invalid error type - got %T; expected %T", err, want) 140 | } 141 | } 142 | 143 | func TestReadScope(t *testing.T) { 144 | var content = []byte(` 145 | groups: 146 | private: 147 | in_scope: 148 | - 10.0.0.0/8 149 | out_scope: 150 | - 10.1.0.0/16 151 | trusted_ips: 152 | - 11.0.0.0/8 153 | trusted_domains: 154 | - alphasoc.com 155 | public: 156 | in_scope: 157 | - 0.0.0.0/0 158 | trusted_domains: 159 | - "*.io"`) 160 | f, err := ioutil.TempFile("", "nfr-scope") 161 | if err != nil { 162 | log.Fatal(err) 163 | } 164 | defer os.Remove(f.Name()) 165 | defer f.Close() 166 | 167 | if _, err := f.Write(content); err != nil { 168 | log.Fatal(err) 169 | } 170 | 171 | var cfg Config 172 | cfg.Scope.File = f.Name() 173 | if err := cfg.loadScopeConfig(); err != nil { 174 | t.Fatal(err) 175 | } 176 | 177 | groups := cfg.ScopeConfig.Groups 178 | if l := len(groups); l != 2 { 179 | t.Fatalf("invalid groups length - got %d; exptected %d", l, 2) 180 | } 181 | if _, ok := groups["private"]; !ok { 182 | t.Fatal("no private groups found") 183 | } 184 | if _, ok := groups["public"]; !ok { 185 | t.Fatal("no public groups found") 186 | } 187 | 188 | private := groups["private"] 189 | if len(private.InScope) != 1 { 190 | t.Fatal("invalid private group source network include") 191 | } 192 | if len(private.OutScope) != 1 { 193 | t.Fatal("invalid private group source network exclude") 194 | } 195 | if len(private.TrustedIps) != 1 { 196 | t.Fatal("invalid private group destination network include") 197 | } 198 | if len(private.TrustedDomains) != 1 { 199 | t.Fatal("invalid private group domains exclude") 200 | } 201 | } 202 | 203 | func TestReadScopeStrict(t *testing.T) { 204 | // group instead of groups 205 | var content = []byte(` 206 | group: 207 | private: 208 | in_scope: 209 | - 10.0.0.0/8 210 | out_scope: 211 | - 10.1.0.0/16 212 | trusted_ips: 213 | - 11.0.0.0/8 214 | trusted_domains: 215 | - alphasoc.com 216 | public: 217 | in_scope: 218 | - 0.0.0.0/0 219 | trusted_domains: 220 | - "*.io"`) 221 | 222 | f, err := os.OpenFile(path.Join(t.TempDir(), "nfr-scope"), os.O_RDWR|os.O_CREATE, 0755) 223 | if err != nil { 224 | t.Fatal(err) 225 | } 226 | defer f.Close() 227 | 228 | if _, err := f.Write(content); err != nil { 229 | t.Fatal(err) 230 | } 231 | 232 | var cfg Config 233 | cfg.Scope.File = f.Name() 234 | 235 | var want *yaml.TypeError 236 | if err := cfg.loadScopeConfig(); !errors.As(err, &want) { 237 | t.Errorf("invalid error type - got %T; expected %T", err, want) 238 | } 239 | } 240 | -------------------------------------------------------------------------------- /elastic/client.go: -------------------------------------------------------------------------------- 1 | package elastic 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "time" 9 | 10 | "github.com/alphasoc/nfr/client" 11 | "github.com/cenkalti/backoff/v4" 12 | es7 "github.com/elastic/go-elasticsearch/v7" 13 | "github.com/pkg/errors" 14 | ) 15 | 16 | // SupportedEventTypes is a list of event types supported by the module. 17 | var SupportedEventTypes = []client.EventType{ 18 | client.EventTypeDNS, 19 | client.EventTypeIP, 20 | client.EventTypeHTTP, 21 | client.EventTypeTLS, 22 | } 23 | 24 | // Client is an elasticsearch client capable of pulling telemetry data 25 | // from and pushing alphasoc threats to an es instance. It also has all 26 | // necessary methods to set up an index and field mappings for sending 27 | // threats compatible with ECS. 28 | type Client struct { 29 | opts *Config 30 | c *es7.Client 31 | 32 | retryBackoff *backoff.ExponentialBackOff 33 | } 34 | 35 | // NewClient creates a new Client. 36 | func NewClient(opts *Config) (*Client, error) { 37 | if opts == nil { 38 | return nil, errors.New("client options must not be null") 39 | } 40 | 41 | c := &Client{ 42 | opts: opts, 43 | retryBackoff: backoff.NewExponentialBackOff(), 44 | } 45 | c.retryBackoff.MaxElapsedTime = 0 46 | 47 | cfg := es7.Config{ 48 | CloudID: c.opts.CloudID, 49 | APIKey: c.opts.APIKey, 50 | Addresses: c.opts.Hosts, 51 | Username: c.opts.Username, 52 | Password: c.opts.Password, 53 | Transport: &fastTransport{}, 54 | 55 | // Retry on too many requests as well 56 | RetryOnStatus: []int{502, 503, 504, 429}, 57 | 58 | RetryBackoff: func(i int) time.Duration { 59 | if i == 1 { 60 | c.retryBackoff.Reset() 61 | } 62 | return c.retryBackoff.NextBackOff() 63 | }, 64 | 65 | // Retry up to 5 attempts 66 | MaxRetries: 5, 67 | } 68 | 69 | var err error 70 | c.c, err = es7.NewClient(cfg) 71 | if err != nil { 72 | return nil, errors.Wrap(err, "creating es client") 73 | } 74 | 75 | return c, nil 76 | } 77 | 78 | // Fetch opens a new Point-In-Time for given indices and returns a cursor 79 | // used to retrieve events. 80 | func (c *Client) Fetch(ctx context.Context, search *SearchConfig, from time.Time) (*EventsCursor, error) { 81 | if search == nil { 82 | return nil, errors.New("search config must not be null") 83 | } 84 | 85 | ec := &EventsCursor{client: c.c, search: search, newestIngested: from} 86 | return ec, nil 87 | } 88 | 89 | // OpenPIT opens a Point-in-time transaction for given indices. 90 | func (c *Client) OpenPIT(ctx context.Context, indices []string, keepAlive time.Duration) (*PointInTime, error) { 91 | var ka int 92 | if keepAlive > 0 { 93 | ka = int(keepAlive.Seconds()) 94 | } else { 95 | ka = int(DefaultPITKeepAlive.Seconds()) 96 | } 97 | 98 | res, err := c.c.OpenPointInTime( 99 | c.c.OpenPointInTime.WithContext(ctx), 100 | c.c.OpenPointInTime.WithIndex(indices...), 101 | c.c.OpenPointInTime.WithKeepAlive(fmt.Sprintf("%vs", ka)), 102 | ) 103 | if err != nil { 104 | return nil, err 105 | } 106 | defer res.Body.Close() 107 | 108 | if err := IsAPIError(res); err != nil { 109 | return nil, err 110 | } 111 | 112 | data, err := ioutil.ReadAll(res.Body) 113 | if err != nil { 114 | return nil, err 115 | } 116 | 117 | pit := PointInTime{client: c.c} 118 | err = json.Unmarshal(data, &pit) 119 | return &pit, err 120 | } 121 | -------------------------------------------------------------------------------- /elastic/config_test.go: -------------------------------------------------------------------------------- 1 | package elastic 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/alphasoc/nfr/client" 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestEmptyFieldNamesConfig(t *testing.T) { 11 | fnc := FieldNamesConfig{} 12 | require.True(t, fnc.IsEmpty(), "expected fnc to be empty") 13 | 14 | fnc.QType = []string{"text"} 15 | require.False(t, fnc.IsEmpty(), "expected fnc to be not empty") 16 | } 17 | 18 | func TestInvalidConfig(t *testing.T) { 19 | searchcfg := &SearchConfig{ 20 | EventType: "dns", 21 | Indices: []string{"filebeat-*"}, 22 | SearchTerm: "{}", 23 | IndexSchema: "ecs", 24 | } 25 | 26 | // Create valid config 27 | cfg := &Config{ 28 | Enabled: true, 29 | Searches: []*SearchConfig{searchcfg}, 30 | } 31 | 32 | cfg.CloudID = "randomstring" 33 | cfg.APIKey = "foobar" 34 | 35 | require.NoError(t, cfg.Validate()) 36 | 37 | // Remove elastic address and see if it fails 38 | cfg.CloudID = "" 39 | require.Error(t, cfg.Validate(), "empty cloud_id and hosts shouldn't validate") 40 | 41 | // Set hosts and it should validate. It doesn't matter it's not a valid FQDN -- 42 | // this is taken care of by elasticsearch client. 43 | cfg.Hosts = []string{"randomstring"} 44 | require.NoError(t, cfg.Validate(), "host is set, expected valid config") 45 | 46 | cfg.APIKey = "" 47 | require.NoError(t, cfg.Validate(), "both api key and username are empty, which is valid") 48 | 49 | cfg.Username = "bob" 50 | require.NoError(t, cfg.Validate(), "username is set, expected valid config") 51 | 52 | cfg.Searches = nil 53 | require.Error(t, cfg.Validate(), "no defined searches should be invalid") 54 | 55 | // Append invalid search config. It should not validate 56 | emptysc := &SearchConfig{} 57 | cfg.Searches = []*SearchConfig{emptysc} 58 | require.Error(t, cfg.Validate(), "it should not validate with an invalid search config") 59 | } 60 | 61 | func TestInvalidSearchConfig(t *testing.T) { 62 | cfg := SearchConfig{ 63 | EventType: "dns", 64 | Indices: []string{"filebeat-*"}, 65 | SearchTerm: "", 66 | IndexSchema: "ecs", 67 | } 68 | 69 | require.NoError(t, cfg.Validate(), "it should validate") 70 | 71 | cfg.EventType = "fsdfkjsdf" 72 | require.Error(t, cfg.Validate(), "it should have invalid event type") 73 | 74 | cfg.EventType = "" 75 | require.Error(t, cfg.Validate(), "event type should be required") 76 | 77 | cfg.EventType = client.EventTypeDNS 78 | require.NoError(t, cfg.Validate(), "it should validate") 79 | 80 | cfg.Indices = nil 81 | require.Error(t, cfg.Validate(), "index name is required") 82 | 83 | cfg.Indices = []string{"filebeat-*"} 84 | cfg.IndexSchema = "sfhsdjfhwer" 85 | require.Error(t, cfg.Validate(), "invalid field schema") 86 | 87 | // Set index schema 88 | cfg.IndexSchema = IndexSchemaECS 89 | require.NoError(t, cfg.Validate(), "it should validate with correct schema") 90 | 91 | // It should use default search terms for ecs schema 92 | st := cfg.FinalSearchTerm() 93 | mhf := cfg.FinalMustHaveFields() 94 | require.False(t, st == "" && len(mhf) == 0, "it should have either search term or must have fields for given schema") 95 | 96 | cfg.SearchTerm = "{}" 97 | require.Equal(t, cfg.SearchTerm, cfg.FinalSearchTerm(), "custom search term should override default") 98 | require.NoError(t, cfg.Validate(), "it should validate with correct schema") 99 | 100 | cfg.IndexSchema = "" 101 | cfg.SearchTerm = "" 102 | require.Empty(t, cfg.FinalSearchTerm(), "custom schema should have no default search term") 103 | require.Error(t, cfg.Validate(), "custom schema should have no default search term") 104 | 105 | cfg.SearchTerm = ":342==_-@!" 106 | require.Error(t, cfg.Validate(), "search term should be valid json") 107 | 108 | cfg.SearchTerm = "" 109 | cfg.IndexSchema = IndexSchemaECS 110 | require.NoError(t, cfg.Validate(), "it should validate") 111 | 112 | require.NoError(t, cfg.evaluateFieldNames()) 113 | ffn := cfg.FinalFieldNames() 114 | require.Equal(t, defaultFieldNames[client.EventTypeDNS][IndexSchemaECS].SrcIP, 115 | ffn.SrcIP, "src_ip field name should be equal to the default one") 116 | 117 | // Override field name 118 | cfg.FieldNames = &FieldNamesConfig{SrcIP: []string{"foobar"}} 119 | require.NoError(t, cfg.evaluateFieldNames()) 120 | ffn = cfg.FinalFieldNames() 121 | require.Equal(t, cfg.FieldNames.SrcIP, 122 | ffn.SrcIP, "src_ip field name should be equal to the overriden one") 123 | } 124 | 125 | func TestFieldNamesConfig(t *testing.T) { 126 | sc := &SearchConfig{EventType: client.EventTypeDNS, FieldNames: &FieldNamesConfig{}} 127 | 128 | require.NoError(t, sc.evaluateFieldNames()) 129 | ffs := sc.FinalFieldNames() 130 | 131 | require.Error(t, ffs.Validate(sc.EventType), "required fields should be missing") 132 | 133 | // Add required fields one by one. Validation should fail until all required 134 | // fields are present. 135 | sc.FieldNames.EventIngested = "event.ingested" 136 | 137 | require.NoError(t, sc.evaluateFieldNames()) 138 | ffs = sc.FinalFieldNames() 139 | require.Error(t, ffs.Validate(sc.EventType), "required fields should be missing") 140 | 141 | sc.FieldNames.Timestamp = "@timestamp" 142 | require.NoError(t, sc.evaluateFieldNames()) 143 | ffs = sc.FinalFieldNames() 144 | require.Error(t, ffs.Validate(sc.EventType), "required fields should be missing") 145 | 146 | sc.FieldNames.SrcIP = []string{"src", "ip"} 147 | require.NoError(t, sc.evaluateFieldNames()) 148 | ffs = sc.FinalFieldNames() 149 | require.Error(t, ffs.Validate(sc.EventType), "required fields should be missing") 150 | 151 | // This is the last remaining required fields for EventTypeDNS 152 | sc.FieldNames.Query = []string{"dns", "question", "name"} 153 | require.NoError(t, sc.evaluateFieldNames()) 154 | ffs = sc.FinalFieldNames() 155 | require.NoError(t, ffs.Validate(sc.EventType), "final field names should return no error") 156 | 157 | sc.FieldNames.Query = nil 158 | require.NoError(t, sc.evaluateFieldNames()) 159 | ffs = sc.FinalFieldNames() 160 | require.Error(t, ffs.Validate(sc.EventType), "a required field is missing") 161 | 162 | sc.EventType = client.EventTypeIP 163 | sc.FieldNames.DstIP = []string{"destination", "ip"} 164 | require.NoError(t, sc.evaluateFieldNames()) 165 | ffs = sc.FinalFieldNames() 166 | require.NoError(t, ffs.Validate(sc.EventType), "final field names should return no error") 167 | } 168 | -------------------------------------------------------------------------------- /elastic/cursor.go: -------------------------------------------------------------------------------- 1 | package elastic 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "os" 8 | "strconv" 9 | "time" 10 | 11 | es7 "github.com/elastic/go-elasticsearch/v7" 12 | "github.com/elastic/go-elasticsearch/v7/esapi" 13 | "github.com/pkg/errors" 14 | ) 15 | 16 | // EventsCursor is an iterator which downloads paginated search results 17 | // returned within an open Point-In-Time. 18 | type EventsCursor struct { 19 | client *es7.Client 20 | search *SearchConfig 21 | searchAfter []interface{} 22 | pit *PointInTime // can be used from 7.10 with X-Pack instead of scrollID 23 | scrollCursor []byte // used for pagination 24 | newestIngested time.Time 25 | 26 | // Save search query for debug purposes 27 | lastSearchQuery []byte 28 | } 29 | 30 | // Next retrieves the next page of events. If (nil, nil) is returned, 31 | // there are no new events. 32 | func (ec *EventsCursor) Next(ctx context.Context) ([]Hit, error) { 33 | var res *esapi.Response 34 | var err error 35 | 36 | if ec.scrollCursor != nil { 37 | // Use scroll id to retrieve paginated results 38 | res, err = ec.client.Scroll( 39 | ec.client.Scroll.WithContext(ctx), 40 | ec.client.Scroll.WithBody(bytes.NewReader(ec.scrollCursor))) 41 | } else { 42 | // Scroll ID is empty, create a search with scroll. 43 | ec.lastSearchQuery, err = ec.searchQuery() 44 | if err != nil { 45 | return nil, errors.Wrap(err, "creating search query") 46 | } 47 | 48 | res, err = ec.client.Search( 49 | ec.client.Search.WithContext(ctx), 50 | ec.client.Search.WithIndex(ec.search.Indices...), 51 | ec.client.Search.WithBody(bytes.NewReader(ec.lastSearchQuery)), 52 | ec.client.Search.WithScroll(time.Duration(ec.search.PITKeepAlive)*time.Second)) 53 | } 54 | 55 | if err != nil { 56 | return nil, errors.Wrap(err, "doing search") 57 | } 58 | defer res.Body.Close() 59 | 60 | if err := IsAPIError(res); err != nil { 61 | return nil, errors.Wrap(err, "search api error") 62 | } 63 | 64 | var answer SearchResult 65 | if err := json.NewDecoder(res.Body).Decode(&answer); err != nil { 66 | return nil, errors.Wrap(err, "decoding search response") 67 | } 68 | 69 | if answer.TimedOut { 70 | return nil, ErrQueryTimeout 71 | } 72 | 73 | hits := answer.Hits.Hits 74 | 75 | if len(hits) == 0 { 76 | // No more hits 77 | return nil, nil 78 | } 79 | 80 | lastHit := hits[len(hits)-1] 81 | 82 | if answer.ScrollID != "" { 83 | d := time.Duration(ec.search.PITKeepAlive) * time.Second 84 | 85 | var scroll string 86 | if d < time.Millisecond { 87 | scroll = strconv.FormatInt(int64(d), 10) + "nanos" 88 | } else { 89 | scroll = strconv.FormatInt(int64(d)/int64(time.Millisecond), 10) + "ms" 90 | } 91 | 92 | marshaled, _ := json.Marshal(ScrollSearch{Scroll: scroll, ScrollID: answer.ScrollID}) 93 | ec.scrollCursor = marshaled 94 | } 95 | 96 | // Update searchAfter for the next page. 97 | ec.searchAfter = lastHit.Sort 98 | 99 | // And save the timestamp of the most recent ingested event. 100 | ec.newestIngested, err = lastHit.timestamp(ec.search.FinalFieldNames().EventIngested) 101 | if err != nil { 102 | return nil, errors.Wrap(err, "get event.ingested") 103 | } 104 | 105 | return hits, nil 106 | } 107 | 108 | // NewestIngested returns a timestamp of the most recently ingested event 109 | // retrieved by the search. 110 | func (ec *EventsCursor) NewestIngested() time.Time { 111 | return ec.newestIngested 112 | } 113 | 114 | // SearchConfig returns the underlying search config. 115 | func (ec *EventsCursor) SearchConfig() *SearchConfig { 116 | return ec.search 117 | } 118 | 119 | // Close closes the point-in-time transaction, if it was open. 120 | func (ec *EventsCursor) Close() error { 121 | if ec.pit != nil { 122 | err := ec.pit.Close() 123 | ec.pit = nil 124 | return err 125 | } 126 | 127 | return nil 128 | } 129 | 130 | func (ec *EventsCursor) DumpLastSearchQuery(fname string) error { 131 | f, err := os.Create(fname) 132 | if err != nil { 133 | return err 134 | } 135 | defer f.Close() 136 | 137 | var out bytes.Buffer 138 | json.Indent(&out, ec.lastSearchQuery, "", " ") 139 | out.WriteTo(f) 140 | 141 | return nil 142 | } 143 | -------------------------------------------------------------------------------- /elastic/errors.go: -------------------------------------------------------------------------------- 1 | package elastic 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/elastic/go-elasticsearch/v7/esapi" 9 | ) 10 | 11 | // ErrQueryTimeout is returned when the search query has timed out. 12 | var ErrQueryTimeout = errors.New("query timeout") 13 | 14 | // IsAPIError checks if an es response contains an error. 15 | // If so, it decodes the body and returns the error. 16 | func IsAPIError(res *esapi.Response) error { 17 | if !res.IsError() { 18 | return nil 19 | } 20 | 21 | // read body and return as error 22 | errMsg := make([]byte, 1024) 23 | errMsgSize, _ := io.ReadFull(res.Body, errMsg) 24 | return fmt.Errorf("[%s] %s", res.Status(), errMsg[:errMsgSize]) 25 | } 26 | -------------------------------------------------------------------------------- /elastic/pit.go: -------------------------------------------------------------------------------- 1 | package elastic 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "time" 7 | 8 | es7 "github.com/elastic/go-elasticsearch/v7" 9 | ) 10 | 11 | // DefaultPITKeepAlive defines a default keepalive for point-in-time 12 | // if the provided paramter is zero. 13 | const DefaultPITKeepAlive = 3 * time.Minute // 3 minutes 14 | 15 | // PointInTime is something like a SQL's transaction. It guarantees that 16 | // returned documents are unchanged during the retrieval. 17 | type PointInTime struct { 18 | client *es7.Client 19 | 20 | ID string `json:"id"` 21 | KeepAlive string `json:"keep_alive,omitempty"` 22 | } 23 | 24 | // ScrollSearch is a deprecated way of retrieving 25 | // paginated search results. It is replaced by PointInTime (supported from es 7.10) 26 | type ScrollSearch struct { 27 | Scroll string `json:"scroll"` 28 | ScrollID string `json:"scroll_id"` 29 | } 30 | 31 | // Close closes an open Point-in-time. 32 | func (p *PointInTime) Close() error { 33 | res, err := p.client.ClosePointInTime( 34 | p.client.ClosePointInTime.WithBody(strings.NewReader(fmt.Sprintf(`{"id":"%v"}`, p.ID))), 35 | ) 36 | if err != nil { 37 | return err 38 | } 39 | defer res.Body.Close() 40 | 41 | err = IsAPIError(res) 42 | return err 43 | } 44 | -------------------------------------------------------------------------------- /elastic/search_query.go: -------------------------------------------------------------------------------- 1 | package elastic 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | ) 7 | 8 | // DocRangeField is used in DocRange 9 | type DocRangeField struct { 10 | Gt string `json:"gt"` 11 | Format string `json:"format,omitempty"` 12 | } 13 | 14 | // DocRange is used in SearchQuery. 15 | type DocRange struct { 16 | Range map[string]DocRangeField `json:"range"` 17 | } 18 | 19 | // DocValueField as defined by the es Search API 20 | type DocValueField struct { 21 | Field string `json:"field"` 22 | Format string `json:"format"` 23 | } 24 | 25 | // SearchQuery is a JSON object passed to es instance 26 | // when doing a search. 27 | type SearchQuery struct { 28 | // Return timestamp in docvalue_fields, as we want to force their format 29 | DocValueFields []DocValueField `json:"docvalue_fields"` 30 | 31 | // Return other fields in _source 32 | Source []string `json:"_source"` 33 | Size int `json:"size"` 34 | Query struct { 35 | Bool struct { 36 | Must []json.RawMessage `json:"must,omitempty"` 37 | Filter []json.RawMessage `json:"filter"` 38 | } `json:"bool"` 39 | } `json:"query"` 40 | Sort []map[string]string `json:"sort"` 41 | PIT *PointInTime `json:"pit,omitempty"` 42 | SearchAfter []interface{} `json:"search_after,omitempty"` 43 | } 44 | 45 | func (ec *EventsCursor) searchQuery() ([]byte, error) { 46 | fn := ec.search.FinalFieldNames() 47 | 48 | sq := SearchQuery{} 49 | sq.Size = ec.search.BatchSize 50 | sq.Sort = []map[string]string{ 51 | {fn.EventIngested: "asc"}, 52 | } 53 | sq.PIT = ec.pit 54 | sq.SearchAfter = ec.searchAfter 55 | 56 | var err error 57 | 58 | sq.DocValueFields = []DocValueField{{Field: fn.Timestamp, Format: "strict_date_time"}} 59 | 60 | if fn.Timestamp != fn.EventIngested { 61 | sq.DocValueFields = append(sq.DocValueFields, 62 | DocValueField{Field: fn.EventIngested, Format: "strict_date_time"}) 63 | } 64 | 65 | // Get the document field names to retrieve (except ts and event.ingested) 66 | sq.Source, err = ec.search.EventFields() 67 | if err != nil { 68 | return nil, err 69 | } 70 | 71 | // Build a time range of events we want to retrieve. If no events where 72 | // retrieved ever, let's just ingest the last 10 minutes. Otherwise, 73 | // retrieve all events since NewestIngested from the pull job. 74 | docrange := DocRange{Range: make(map[string]DocRangeField)} 75 | 76 | if ec.newestIngested.IsZero() { 77 | docrange.Range[fn.EventIngested] = DocRangeField{Gt: "now-5m"} 78 | } else { 79 | drf := DocRangeField{Gt: ec.newestIngested.Format(time.RFC3339Nano)} 80 | // Add a timestamp format to the query if configured. Added in response to a 81 | // case where the ingested timestamp lacked milliseconds. The search query, 82 | // to prevent a date field parse error, needed an explicitly set timestamp 83 | // format of 'strict_date_time_no_millis'. Thus it was decided to make this 84 | // configurable. 85 | if ec.search.TimestampFormat != "" { 86 | drf.Format = ec.search.TimestampFormat 87 | } 88 | docrange.Range[fn.EventIngested] = drf 89 | } 90 | 91 | drjson, _ := json.Marshal(docrange) 92 | 93 | // Define search filter terms. Use defaults or custom, if available. 94 | terms := ec.search.FinalSearchTerm() 95 | if terms != "" { 96 | sq.Query.Bool.Filter = append(sq.Query.Bool.Filter, json.RawMessage(terms)) 97 | } 98 | 99 | sq.Query.Bool.Filter = append(sq.Query.Bool.Filter, drjson) 100 | 101 | sq.Query.Bool.Must, err = mustExistFields(ec.search.FinalMustHaveFields()) 102 | if err != nil { 103 | return nil, err 104 | } 105 | 106 | return json.Marshal(sq) 107 | } 108 | 109 | // mustExistFields constructs an Elastic Query DSL fragment used in the 110 | // search query. 111 | func mustExistFields(fields []string) ([]json.RawMessage, error) { 112 | type query struct { 113 | Exists struct { 114 | Field string `json:"field"` 115 | } `json:"exists"` 116 | } 117 | 118 | var ret []json.RawMessage 119 | for _, f := range fields { 120 | q := query{} 121 | q.Exists.Field = f 122 | data, err := json.Marshal(q) 123 | if err != nil { 124 | return nil, err 125 | } 126 | ret = append(ret, data) 127 | } 128 | 129 | return ret, nil 130 | } 131 | -------------------------------------------------------------------------------- /elastic/search_test.go: -------------------------------------------------------------------------------- 1 | package elastic 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func ipProtoParse(val string) string { 9 | src := fmt.Sprintf( 10 | "{\"_source\":{\"device_version\":\"v7.0.1\",\"device_product\":\"Foogate\",\"FTNTFGTmastersrcmac\":\"00:00:00:00:00:00\",\"proto\":%v,\"FTNTFGTpoluuid\":\"00000000-0000-0000-0000-000000000000\"}}", 11 | val) 12 | h := Hit{ 13 | ID: "eFVpenwBwORpcXQ7osoi", 14 | Source: []byte(src), 15 | } 16 | return h.sourceProtocol([]string{"_source", "proto"}) 17 | } 18 | 19 | func TestIPProtoParsing(t *testing.T) { 20 | valMap := make(map[string]string) 21 | valMap["\"6\""] = "tcp" 22 | valMap["6"] = "tcp" 23 | valMap["\"253\""] = "253" 24 | valMap["253"] = "253" 25 | valMap["not_a_json_string"] = "" 26 | valMap["\"udp\""] = "udp" 27 | valMap["\"foo\""] = "foo" 28 | valMap["256"] = "" 29 | valMap["0"] = "" 30 | valMap["-1"] = "" 31 | 32 | for k, v := range valMap { 33 | got := ipProtoParse(k) 34 | want := v 35 | if got != want { 36 | t.Errorf("from %q got %q, wanted %q", k, got, want) 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /elastic/transport.go: -------------------------------------------------------------------------------- 1 | package elastic 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http" 6 | "strings" 7 | 8 | "github.com/valyala/fasthttp" 9 | ) 10 | 11 | // Transport implements the estransport interface with 12 | // the github.com/valyala/fasthttp HTTP client. 13 | type fastTransport struct{} 14 | 15 | // RoundTrip performs the request and returns a response or error 16 | func (t *fastTransport) RoundTrip(req *http.Request) (*http.Response, error) { 17 | freq := fasthttp.AcquireRequest() 18 | defer fasthttp.ReleaseRequest(freq) 19 | 20 | fres := fasthttp.AcquireResponse() 21 | defer fasthttp.ReleaseResponse(fres) 22 | 23 | t.copyRequest(freq, req) 24 | 25 | err := fasthttp.Do(freq, fres) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | res := &http.Response{Header: make(http.Header)} 31 | t.copyResponse(res, fres) 32 | 33 | return res, nil 34 | } 35 | 36 | // copyRequest converts a http.Request to fasthttp.Request 37 | func (t *fastTransport) copyRequest(dst *fasthttp.Request, src *http.Request) *fasthttp.Request { 38 | if src.Method == "GET" && src.Body != nil { 39 | src.Method = "POST" 40 | } 41 | 42 | dst.SetHost(src.Host) 43 | dst.SetRequestURI(src.URL.String()) 44 | 45 | dst.Header.SetRequestURI(src.URL.String()) 46 | dst.Header.SetMethod(src.Method) 47 | 48 | for k, vv := range src.Header { 49 | for _, v := range vv { 50 | dst.Header.Set(k, v) 51 | } 52 | } 53 | 54 | if src.Body != nil { 55 | dst.SetBodyStream(src.Body, -1) 56 | } 57 | 58 | return dst 59 | } 60 | 61 | // copyResponse converts a http.Response to fasthttp.Response 62 | func (t *fastTransport) copyResponse(dst *http.Response, src *fasthttp.Response) *http.Response { 63 | dst.StatusCode = src.StatusCode() 64 | 65 | src.Header.VisitAll(func(k, v []byte) { 66 | dst.Header.Set(string(k), string(v)) 67 | }) 68 | 69 | // Cast to a string to make a copy seeing as src.Body() won't 70 | // be valid after the response is released back to the pool (fasthttp.ReleaseResponse). 71 | dst.Body = ioutil.NopCloser(strings.NewReader(string(src.Body()))) 72 | 73 | return dst 74 | } 75 | -------------------------------------------------------------------------------- /gelf/gelf.go: -------------------------------------------------------------------------------- 1 | package gelf 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net" 7 | 8 | "github.com/Jeffail/gabs" 9 | ) 10 | 11 | // Gelf client. 12 | type Gelf struct { 13 | conn net.Conn 14 | } 15 | 16 | // Message for graylog server. 17 | type Message struct { 18 | Version string `json:"version"` 19 | Host string `json:"host"` 20 | ShortMessage string `json:"short_message"` 21 | FullMessage string `json:"full_message"` 22 | Timestamp int64 `json:"timestamp"` 23 | Level int `json:"level"` 24 | 25 | Extra map[string]interface{} `json:"-"` 26 | } 27 | 28 | // New returns GELF client. 29 | func NewConnected(scheme, uri string) (*Gelf, error) { 30 | conn, err := net.Dial(scheme, uri) 31 | if err != nil { 32 | return nil, err 33 | 34 | } 35 | return &Gelf{conn: conn}, nil 36 | } 37 | 38 | // Close the connection to the server. 39 | func (g *Gelf) Close() error { 40 | if g.conn != nil { 41 | return g.conn.Close() 42 | } 43 | return nil 44 | } 45 | 46 | // Send message to the server. 47 | func (g *Gelf) Send(m *Message) error { 48 | b, err := json.Marshal(m) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | c, err := gabs.ParseJSON(b) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | for k, v := range m.Extra { 59 | _, err = c.Set(v, fmt.Sprintf("_%s", k)) 60 | if err != nil { 61 | return err 62 | } 63 | } 64 | 65 | _, err = g.conn.Write(append(c.Bytes(), '\n', 0)) 66 | return err 67 | } 68 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/alphasoc/nfr 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/Jeffail/gabs v1.0.1-0.20171015111430-44cbc2713851 7 | github.com/Sirupsen/logrus v0.11.6-0.20170512101114-62f94013e586 8 | github.com/buger/jsonparser v1.1.1 9 | github.com/cenkalti/backoff/v4 v4.1.0 10 | github.com/elastic/go-elasticsearch/v7 v7.11.0 11 | github.com/google/go-cmp v0.5.2 12 | github.com/google/gopacket v1.1.18-0.20190912173203-2d7fab0d91d6 13 | github.com/hpcloud/tail v1.0.1-0.20170814160653-37f427138745 14 | github.com/imdario/mergo v0.3.11 15 | github.com/pkg/errors v0.9.1 16 | github.com/spf13/cobra v1.1.1 17 | github.com/stretchr/testify v1.4.0 18 | github.com/twmb/murmur3 v1.1.5 19 | github.com/valyala/fasthttp v1.34.0 20 | github.com/xoebus/ceflog v0.0.0-20180302015320-9cb6ad8a040b 21 | golang.org/x/net v0.15.0 22 | gopkg.in/yaml.v3 v3.0.1 23 | ) 24 | 25 | require ( 26 | github.com/andybalholm/brotli v1.0.4 // indirect 27 | github.com/davecgh/go-spew v1.1.1 // indirect 28 | github.com/fsnotify/fsnotify v1.4.9 // indirect 29 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 30 | github.com/klauspost/compress v1.15.0 // indirect 31 | github.com/pmezard/go-difflib v1.0.0 // indirect 32 | github.com/spf13/pflag v1.0.5 // indirect 33 | github.com/valyala/bytebufferpool v1.0.0 // indirect 34 | golang.org/x/sys v0.12.0 // indirect 35 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 36 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 37 | gopkg.in/fsnotify.v1 v1.4.7 // indirect 38 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect 39 | gopkg.in/yaml.v2 v2.3.0 // indirect 40 | ) 41 | -------------------------------------------------------------------------------- /gopacket/ssl/ssl.go: -------------------------------------------------------------------------------- 1 | // Package tls parse TCP tls data. 2 | package ssl 3 | 4 | import ( 5 | "crypto/tls" 6 | ) 7 | 8 | const ( 9 | VersionTLS13 = 0x304 10 | ) 11 | 12 | // SSL Message type 13 | const ( 14 | TLS_HANDSHAKE = 22 15 | TLS_CLIENT_HELLO = 1 16 | ) 17 | 18 | const ( 19 | TLS_CLIENT_HELLO_RANDOM_LEN = 32 20 | ) 21 | 22 | // TLSGreaseCiperSiutes table ref: https://tools.ietf.org/html/draft-davidben-tls-grease-00 23 | var TLSGreaseCiperSiutes = map[uint16]bool{ 24 | 0x0a0a: true, 25 | 0x1a1a: true, 26 | 0x2a2a: true, 27 | 0x3a3a: true, 28 | 0x4a4a: true, 29 | 0x5a5a: true, 30 | 0x6a6a: true, 31 | 0x7a7a: true, 32 | 0x8a8a: true, 33 | 0x9a9a: true, 34 | 0xaaaa: true, 35 | 0xbaba: true, 36 | 0xcaca: true, 37 | 0xdada: true, 38 | 0xeaea: true, 39 | 0xfafa: true, 40 | } 41 | 42 | const TLSRecordHeaderLength = 5 43 | 44 | type TLSRecord struct { 45 | Type uint8 46 | Version uint16 47 | Length uint16 48 | Data []byte 49 | } 50 | 51 | type TLSExtension struct { 52 | Type uint16 53 | Data []byte 54 | } 55 | 56 | type TLSClientHello struct { 57 | Type uint8 58 | Length uint32 59 | Version uint16 60 | Random []byte 61 | SessionIDLen uint8 62 | SessionID []byte 63 | CipherSuitesLen uint16 64 | CipherSuites []byte 65 | NumCompressMethods uint8 66 | CompressMethods []uint8 67 | ExtensionsLen uint16 68 | Extensions []TLSExtension 69 | } 70 | 71 | func newTLSRecord(buf []byte) *TLSRecord { 72 | if len(buf) < TLSRecordHeaderLength { 73 | return nil 74 | } 75 | 76 | var record = TLSRecord{ 77 | Type: uint8(buf[0]), 78 | Version: uint16(buf[1])<<8 | uint16(buf[2]), 79 | Length: uint16(buf[3])<<8 | uint16(buf[4]), 80 | } 81 | 82 | if len(buf) < int(record.Length+TLSRecordHeaderLength) { 83 | return nil 84 | } 85 | 86 | record.Data = buf[TLSRecordHeaderLength : TLSRecordHeaderLength+record.Length] 87 | return &record 88 | } 89 | 90 | func GetTLSRecord(buf []byte) *TLSRecord { 91 | version := uint16(buf[1])<<8 | uint16(buf[2]) 92 | if version < tls.VersionSSL30 || version > VersionTLS13 { 93 | return nil 94 | } 95 | 96 | return newTLSRecord(buf) 97 | } 98 | 99 | func (r *TLSRecord) TLSClientHello() *TLSClientHello { 100 | var ( 101 | clientHello TLSClientHello 102 | buf = r.Data 103 | ) 104 | 105 | if len(buf) < 6+TLS_CLIENT_HELLO_RANDOM_LEN { 106 | return nil 107 | } 108 | if clientHello.Type = uint8(buf[0]); clientHello.Type != TLS_CLIENT_HELLO { 109 | return nil 110 | } 111 | 112 | clientHello.Length = uint32(buf[1])<<16 | uint32(buf[2])<<8 | uint32(buf[3]) 113 | clientHello.Version = uint16(buf[4])<<8 | uint16(buf[5]) 114 | clientHello.Random = buf[6 : 6+TLS_CLIENT_HELLO_RANDOM_LEN] 115 | buf = buf[6+TLS_CLIENT_HELLO_RANDOM_LEN:] 116 | 117 | if len(buf) < 1 { 118 | return nil 119 | } 120 | clientHello.SessionIDLen = uint8(buf[0]) 121 | buf = buf[1:] 122 | 123 | if len(buf) < int(clientHello.SessionIDLen) { 124 | return nil 125 | } 126 | clientHello.SessionID = buf[:clientHello.SessionIDLen] 127 | buf = buf[clientHello.SessionIDLen:] 128 | 129 | if len(buf) < 2 { 130 | return nil 131 | } 132 | 133 | clientHello.CipherSuitesLen = (uint16(buf[0])<<8 | uint16(buf[1])) / 2 134 | buf = buf[2:] 135 | 136 | if len(buf) < int(clientHello.CipherSuitesLen) { 137 | return nil 138 | } 139 | 140 | clientHello.CipherSuites = make([]byte, clientHello.CipherSuitesLen*2) 141 | for i := 0; i < int(clientHello.CipherSuitesLen*2); i++ { 142 | clientHello.CipherSuites[i] = buf[i] 143 | } 144 | buf = buf[clientHello.CipherSuitesLen*2:] 145 | 146 | if len(buf) < 1 { 147 | return nil 148 | } 149 | 150 | clientHello.NumCompressMethods = uint8(buf[0]) 151 | if len(buf) < int(clientHello.NumCompressMethods) { 152 | return nil 153 | } 154 | buf = buf[1:] 155 | 156 | clientHello.CompressMethods = make([]uint8, clientHello.NumCompressMethods) 157 | for i := 0; i < int(clientHello.NumCompressMethods); i++ { 158 | clientHello.CompressMethods[i] = uint8(buf[i]) 159 | } 160 | buf = buf[clientHello.NumCompressMethods:] 161 | 162 | if len(buf) > 6 { 163 | clientHello.Extensions = make([]TLSExtension, 0) 164 | clientHello.ExtensionsLen = uint16(buf[0])<<8 | uint16(buf[1]) 165 | buf = buf[2:] 166 | 167 | for len(buf) > 0 { 168 | extType := uint16(buf[0])<<8 | uint16(buf[1]) 169 | buf = buf[2:] 170 | l := uint16(buf[0])<<8 | uint16(buf[1]) 171 | buf = buf[2:] 172 | 173 | extData := buf[:l] 174 | clientHello.Extensions = append(clientHello.Extensions, TLSExtension{Type: extType, Data: extData}) 175 | buf = buf[l:] 176 | } 177 | } 178 | 179 | return &clientHello 180 | } 181 | -------------------------------------------------------------------------------- /groups/groups.go: -------------------------------------------------------------------------------- 1 | package groups 2 | 3 | import ( 4 | "net" 5 | "regexp" 6 | "strings" 7 | 8 | "github.com/alphasoc/nfr/matchers" 9 | ) 10 | 11 | // httpRegexp extracts scheme, hostname and port 12 | var httpRegexp = regexp.MustCompile(`^(?:(.*?)://)?([-a-zA-Z0-9_.]+)(?:[:](\d+))?[^?&#]*`) 13 | 14 | // Group is a definition used for whitelising ip and dns traffic. 15 | type Group struct { 16 | Name string 17 | Label string 18 | 19 | // source ip/cidr 20 | SrcIncludes []string 21 | SrcExcludes []string 22 | // destination ip/cidr 23 | DstIncludes []string 24 | DstExcludes []string 25 | 26 | // only used for dns query whitelist 27 | ExcludedDomains []string 28 | } 29 | 30 | // matcher type for single group 31 | type matcher struct { 32 | dm *matchers.Domain 33 | nm *matchers.Network 34 | } 35 | 36 | // Groups is a set of group definition used for whitelisting ip and dns traffic. 37 | type Groups struct { 38 | ms map[string]matcher 39 | 40 | gs map[string]*Group 41 | } 42 | 43 | // New creates new Groups. 44 | func New() *Groups { 45 | return &Groups{ 46 | ms: make(map[string]matcher), 47 | gs: make(map[string]*Group), 48 | } 49 | } 50 | 51 | // Add adds whitelist group. 52 | func (g *Groups) Add(group *Group) error { 53 | dm, err := matchers.NewDomain(group.ExcludedDomains) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | nm, err := matchers.NewNetwork(group.SrcIncludes, group.SrcExcludes, group.DstIncludes, group.DstExcludes) 59 | if err != nil { 60 | return err 61 | } 62 | 63 | g.ms[group.Name] = matcher{dm, nm} 64 | g.gs[group.Name] = group 65 | return nil 66 | } 67 | 68 | // IsIPWhitelisted returns true if ip packet doesn't match any of a groups. 69 | func (g *Groups) IsIPWhitelisted(srcIP, dstIP net.IP) (string, bool) { 70 | // if there is no groups, then every query is whitelisted 71 | if g == nil || len(g.ms) == 0 { 72 | return "", true 73 | } 74 | 75 | if srcIP == nil || dstIP == nil { 76 | return "", false 77 | } 78 | 79 | // ip must be included in at least 1 group, while 80 | // being not excluded from others groups. 81 | ok := false 82 | for name, matcher := range g.ms { 83 | matched, excluded := matcher.nm.Match(srcIP, dstIP) 84 | if !matched { 85 | continue 86 | } 87 | if excluded { 88 | return name, false 89 | } 90 | ok = true 91 | } 92 | 93 | return "", ok 94 | } 95 | 96 | // IsDNSQueryWhitelisted returns true if dns query doesn't match any of groups. 97 | func (g *Groups) IsDNSQueryWhitelisted(domain string, srcIP, dstIP net.IP) (string, bool) { 98 | // if there is no group, then every query is whitelisted 99 | if g == nil || len(g.ms) == 0 { 100 | return "", true 101 | } 102 | 103 | if domain == "" || srcIP == nil { 104 | return "", false 105 | } 106 | 107 | // ip must be included in at least 1 matcher, while 108 | // being not excluded from others groups. 109 | // At the same time domain can't be included in 110 | // groups excluded domains. 111 | var ( 112 | ok = false 113 | matched, excluded bool 114 | ) 115 | for name, matcher := range g.ms { 116 | // allow dstIP to be null, some logs format dosen't track dst ip. 117 | if dstIP != nil { 118 | matched, excluded = matcher.nm.Match(srcIP, dstIP) 119 | } else { 120 | matched, excluded = matcher.nm.MatchSrcIP(srcIP) 121 | } 122 | if !matched { 123 | continue 124 | } 125 | if excluded { 126 | return name, false 127 | } 128 | 129 | // ip is matched, now check if domain is not excluded 130 | if matcher.dm.Match(strings.ToLower(domain)) { 131 | return name, false 132 | } 133 | 134 | // in case of success do not break, because the ip/domain 135 | // could be on other lists. 136 | ok = true 137 | } 138 | 139 | return "", ok 140 | } 141 | 142 | // IsHTTPQueryWhitelisted returns true if dns query doesn't match any of groups. 143 | func (g *Groups) IsHTTPQueryWhitelisted(url string, srcIP net.IP) (string, bool) { 144 | // if there is no group, then every query is whitelisted 145 | if g == nil || len(g.ms) == 0 { 146 | return "", true 147 | } 148 | 149 | if url == "" || srcIP == nil { 150 | return "", false 151 | } 152 | 153 | var domain string 154 | if p := httpRegexp.FindStringSubmatch(url); p != nil { 155 | domain = strings.ToLower(p[2]) 156 | } 157 | 158 | // ip must be included in at least 1 matcher, while 159 | // being not excluded from others groups. 160 | // At the same time domain can't be included in 161 | // groups excluded domains. 162 | var ( 163 | ok = false 164 | matched, excluded bool 165 | ) 166 | for name, matcher := range g.ms { 167 | matched, excluded = matcher.nm.MatchSrcIP(srcIP) 168 | if !matched { 169 | continue 170 | } 171 | if excluded { 172 | return name, false 173 | } 174 | 175 | // ip is matched, now check if domain is not excluded 176 | if domain != "" && matcher.dm.Match(domain) { 177 | return name, false 178 | } 179 | 180 | // in case of success do not break, because the ip/domain 181 | // could be on other lists. 182 | ok = true 183 | } 184 | 185 | return "", ok 186 | } 187 | 188 | // FindGroupsBySrcIP finds first group src ip belongs to. 189 | func (g *Groups) FindGroupsBySrcIP(srcIP net.IP) (groups []*Group) { 190 | for name, matcher := range g.ms { 191 | if matched, excluded := matcher.nm.MatchSrcIP(srcIP); matched && !excluded { 192 | groups = append(groups, g.gs[name]) 193 | } 194 | } 195 | return 196 | } 197 | -------------------------------------------------------------------------------- /groups/groups_test.go: -------------------------------------------------------------------------------- 1 | package groups 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | ) 7 | 8 | func TestIsDNSQueryWhitelisted(t *testing.T) { 9 | var testsGroups = []struct { 10 | name string 11 | groups []*Group 12 | 13 | domains []string 14 | srcIps []net.IP 15 | dstIps []net.IP 16 | expected []bool 17 | }{ 18 | { 19 | "allow any ips", 20 | []*Group{ 21 | { 22 | Name: "allow any", 23 | SrcIncludes: []string{"0.0.0.0/0"}, 24 | }, 25 | }, 26 | []string{"a"}, 27 | []net.IP{net.IPv4(10, 0, 0, 0)}, 28 | []net.IP{net.IPv4(10, 0, 0, 0)}, 29 | []bool{true}, 30 | }, 31 | { 32 | "allow private networks", 33 | []*Group{ 34 | { 35 | Name: "private network 1", 36 | SrcIncludes: []string{"10.0.0.0/8"}, 37 | }, 38 | { 39 | Name: "private network 2", 40 | SrcIncludes: []string{"192.168.0.0/16"}, 41 | }, 42 | }, 43 | []string{"a", "a", "a"}, 44 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(192, 168, 0, 0), net.IPv4(11, 0, 0, 0)}, 45 | []net.IP{net.IPv4(0, 0, 0, 0), net.IPv4(0, 0, 0, 0), net.IPv4(0, 0, 0, 0)}, 46 | []bool{true, true, false}, 47 | }, 48 | { 49 | "allow private networks with excludes", 50 | []*Group{ 51 | { 52 | Name: "private network 1", 53 | SrcIncludes: []string{"10.0.0.0/8"}, 54 | SrcExcludes: []string{"10.1.0.0/16"}, 55 | }, 56 | { 57 | Name: "private network 2", 58 | SrcIncludes: []string{"192.168.0.0/16"}, 59 | SrcExcludes: []string{"10.2.0.0/16"}, 60 | }, 61 | }, 62 | []string{"a", "a", "a", "a"}, 63 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(192, 168, 0, 0), net.IPv4(10, 1, 0, 0), net.IPv4(10, 2, 0, 0)}, 64 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(192, 168, 0, 0), net.IPv4(10, 1, 0, 0), net.IPv4(10, 2, 0, 0)}, 65 | []bool{true, true, false, true}, 66 | }, 67 | { 68 | "include in one group then exclude in next group", 69 | []*Group{ 70 | { 71 | Name: "private network 1", 72 | SrcIncludes: []string{"10.0.0.0/8"}, 73 | }, 74 | { 75 | Name: "private network 2", 76 | SrcIncludes: []string{"10.1.0.0/16"}, 77 | SrcExcludes: []string{"10.1.1.0/24"}, 78 | }, 79 | }, 80 | []string{"a", "a"}, 81 | []net.IP{net.IPv4(10, 1, 0, 0), net.IPv4(10, 1, 1, 0)}, 82 | []net.IP{net.IPv4(10, 1, 0, 0), net.IPv4(10, 1, 1, 0)}, 83 | []bool{true, false}, 84 | }, 85 | { 86 | "exclude domain in multiple groups", 87 | []*Group{ 88 | { 89 | Name: "private network 1", 90 | SrcIncludes: []string{"10.0.0.0/16"}, 91 | ExcludedDomains: []string{"a"}, 92 | }, 93 | { 94 | Name: "private network 2", 95 | SrcIncludes: []string{"10.1.0.0/16"}, 96 | ExcludedDomains: []string{"b"}, 97 | }, 98 | }, 99 | []string{"a", "b", "a", "B"}, 100 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(10, 0, 0, 0), net.IPv4(10, 1, 0, 0), net.IPv4(10, 1, 0, 0)}, 101 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(10, 0, 0, 0), net.IPv4(10, 1, 0, 0), net.IPv4(10, 1, 0, 0)}, 102 | []bool{false, true, true, false}, 103 | }, 104 | { 105 | "mix", 106 | []*Group{ 107 | { 108 | Name: "private network", 109 | SrcIncludes: []string{"10.0.0.0/24", "10.1.0.0/24"}, 110 | SrcExcludes: []string{"10.0.0.1", "10.1.0.1"}, 111 | DstIncludes: []string{"11.0.0.0/24", "11.1.0.0/24"}, 112 | DstExcludes: []string{"11.0.0.1", "11.1.0.1"}, 113 | }, 114 | }, 115 | []string{"a", "a", "a", "a", "a", "a"}, 116 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(10, 0, 0, 0), net.IPv4(10, 1, 0, 0), net.IPv4(10, 1, 0, 0), net.IPv4(10, 1, 0, 1), net.IPv4(10, 1, 0, 0)}, 117 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(11, 0, 0, 0), net.IPv4(10, 1, 0, 0), net.IPv4(11, 1, 0, 0), net.IPv4(11, 0, 0, 0), net.IPv4(11, 1, 0, 1)}, 118 | []bool{false, true, false, true, false, false}, 119 | }, 120 | { 121 | "public", 122 | []*Group{ 123 | { 124 | Name: "default", 125 | SrcIncludes: []string{"0.0.0.0/0"}, 126 | SrcExcludes: []string{}, 127 | DstIncludes: []string{"0.0.0.0/0", "::/0"}, 128 | DstExcludes: []string{"10.0.0.0/8", "192.168.0.0/16", "172.16.0.0/12", "fc00::/7"}, 129 | ExcludedDomains: []string{"*.arpa", "*.lan", "*.local", "*.internal"}, 130 | }, 131 | }, 132 | []string{"alphasoc.com"}, 133 | []net.IP{net.IPv4(172, 31, 84, 103)}, 134 | []net.IP{net.IPv4(31, 13, 65, 56)}, 135 | []bool{true}, 136 | }, 137 | } 138 | 139 | for _, tt := range testsGroups { 140 | g := New() 141 | for _, group := range tt.groups { 142 | if err := g.Add(group); err != nil { 143 | t.Fatal(tt.name, err) 144 | } 145 | } 146 | for i := range tt.domains { 147 | if name, b := g.IsDNSQueryWhitelisted(tt.domains[i], tt.srcIps[i], tt.dstIps[i]); b != tt.expected[i] { 148 | t.Fatalf("%s IsDNSQueryWhitelisted(%s, %s, %s) %s %t; expected %t", tt.name, 149 | tt.domains[i], tt.srcIps[i], tt.dstIps[i], name, b, tt.expected[i]) 150 | } 151 | } 152 | } 153 | } 154 | 155 | func TestIsIPWhitelisted(t *testing.T) { 156 | var testsGroups = []struct { 157 | name string 158 | groups []*Group 159 | 160 | srcIps []net.IP 161 | dstIps []net.IP 162 | expected []bool 163 | }{ 164 | { 165 | "mix", 166 | []*Group{ 167 | { 168 | Name: "private network", 169 | SrcIncludes: []string{"10.0.0.0/24", "10.1.0.0/24"}, 170 | SrcExcludes: []string{"10.0.0.1", "10.1.0.1"}, 171 | DstIncludes: []string{"11.0.0.0/24", "11.1.0.0/24"}, 172 | DstExcludes: []string{"11.0.0.1", "11.1.0.1"}, 173 | }, 174 | }, 175 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(10, 0, 0, 0), net.IPv4(10, 1, 0, 0), net.IPv4(10, 1, 0, 0), net.IPv4(10, 1, 0, 1), net.IPv4(10, 1, 0, 0)}, 176 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(11, 0, 0, 0), net.IPv4(10, 1, 0, 0), net.IPv4(11, 1, 0, 0), net.IPv4(11, 0, 0, 0), net.IPv4(11, 1, 0, 1)}, 177 | []bool{false, true, false, true, false, false}, 178 | }, 179 | } 180 | 181 | for _, tt := range testsGroups { 182 | g := New() 183 | for _, group := range tt.groups { 184 | if err := g.Add(group); err != nil { 185 | t.Fatal(err) 186 | } 187 | } 188 | for i := range tt.srcIps { 189 | if _, b := g.IsIPWhitelisted(tt.srcIps[i], tt.dstIps[i]); b != tt.expected[i] { 190 | t.Fatalf("IsIPWhitelisted(%s, %s) got %t; expected %t", tt.srcIps[i], tt.dstIps[i], b, tt.expected[i]) 191 | } 192 | } 193 | } 194 | } 195 | 196 | func TestIsHTTPQueryWhitelisted(t *testing.T) { 197 | var testsGroups = []struct { 198 | name string 199 | groups []*Group 200 | 201 | urls []string 202 | srcIps []net.IP 203 | expected []bool 204 | }{ 205 | { 206 | "exclude domain in url", 207 | []*Group{ 208 | { 209 | Name: "private network 1", 210 | SrcIncludes: []string{"10.0.0.0/16"}, 211 | ExcludedDomains: []string{"*.com"}, 212 | }, 213 | { 214 | Name: "private network 2", 215 | SrcIncludes: []string{"10.1.0.0/16"}, 216 | ExcludedDomains: []string{"*.net"}, 217 | }, 218 | }, 219 | []string{"x.com/p", "https://x.net", "y.com/xyz", "http://y.net/a?x=z"}, 220 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(10, 0, 0, 0), net.IPv4(10, 1, 0, 0), net.IPv4(10, 1, 0, 0)}, 221 | []bool{false, true, true, false}, 222 | }, 223 | } 224 | 225 | for _, tt := range testsGroups { 226 | g := New() 227 | for _, group := range tt.groups { 228 | if err := g.Add(group); err != nil { 229 | t.Fatal(tt.name, err) 230 | } 231 | } 232 | for i := range tt.urls { 233 | if name, b := g.IsHTTPQueryWhitelisted(tt.urls[i], tt.srcIps[i]); b != tt.expected[i] { 234 | t.Errorf("%s IsDNSQueryWhitelisted(%s, %s) %s %t; expected %t", tt.name, 235 | tt.urls[i], tt.srcIps[i], name, b, tt.expected[i]) 236 | } 237 | } 238 | } 239 | } 240 | 241 | func TestEmptyGroup(t *testing.T) { 242 | var g *Groups 243 | if _, b := g.IsDNSQueryWhitelisted("a", net.IPv4(10, 0, 0, 0), net.IPv4(10, 0, 0, 0)); !b { 244 | t.Fatalf("nil groups must whitelist domain") 245 | } 246 | g = New() 247 | if _, b := g.IsDNSQueryWhitelisted("a", net.IPv4(10, 0, 0, 0), net.IPv4(10, 0, 0, 0)); !b { 248 | t.Fatalf("no groups must whitelist domain") 249 | } 250 | } 251 | -------------------------------------------------------------------------------- /ja3/ja3.go: -------------------------------------------------------------------------------- 1 | package ja3 2 | 3 | import ( 4 | "crypto/md5" 5 | "encoding/hex" 6 | "errors" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/alphasoc/nfr/gopacket/ssl" 11 | "github.com/google/gopacket" 12 | "github.com/google/gopacket/layers" 13 | ) 14 | 15 | // Convert converts raw packet to ja3 digest. 16 | // It retruns empty string if packet is not convertable to ja3. 17 | func Convert(raw gopacket.Packet) string { 18 | _, digest := convert(raw) 19 | return digest 20 | } 21 | 22 | func convert(raw gopacket.Packet) (string, string) { 23 | var transportLayer = raw.TransportLayer() 24 | 25 | tcp, ok := transportLayer.(gopacket.Layer).(*layers.TCP) 26 | if !ok { 27 | return "", "" 28 | } 29 | 30 | payload := tcp.LayerPayload() 31 | if len(payload) == 0 || uint8(payload[0]) != ssl.TLS_HANDSHAKE { 32 | return "", "" 33 | } 34 | 35 | // FIXME: panic if len(tcp.LayerPayload()) < 2; fix here or in alphasoc/nfr/gopacket/ssl 36 | record := ssl.GetTLSRecord(tcp.LayerPayload()) 37 | if record == nil { 38 | return "", "" 39 | } 40 | 41 | if record.Type != ssl.TLS_HANDSHAKE || record.Length == 0 { 42 | return "", "" 43 | } 44 | 45 | clientHello := record.TLSClientHello() 46 | if clientHello == nil { 47 | return "", "" 48 | } 49 | 50 | var ja3 []string 51 | ja3 = append(ja3, strconv.FormatInt(int64(clientHello.Version), 10)) 52 | 53 | seg, err := convertToJa3Segment(clientHello.CipherSuites, 2) 54 | if err != nil { 55 | return "", "" 56 | } 57 | ja3 = append(ja3, seg) 58 | 59 | exts, err := processExtensions(clientHello) 60 | if err != nil { 61 | return "", "" 62 | } 63 | 64 | ja3 = append(ja3, exts...) 65 | 66 | ja3Str := strings.Join(ja3, ",") 67 | ja3Hash := md5.Sum([]byte(ja3Str)) 68 | return ja3Str, hex.EncodeToString(ja3Hash[:]) 69 | } 70 | 71 | func convertToJa3Segment(buf []byte, elemWidth int) (string, error) { 72 | var vals []string 73 | if len(buf)%elemWidth != 0 { 74 | return "", errors.New("len(buf) not multiple") 75 | } 76 | 77 | for i := 0; i < len(buf); i += elemWidth { 78 | var element uint16 79 | if elemWidth == 1 { 80 | element = uint16(uint8(buf[i])) 81 | } else { 82 | element = uint16(buf[i])<<8 | uint16(buf[i+1]) 83 | } 84 | if !ssl.TLSGreaseCiperSiutes[element] { 85 | vals = append(vals, strconv.FormatInt(int64(element), 10)) 86 | } 87 | } 88 | return strings.Join(vals, "-"), nil 89 | } 90 | 91 | func processExtensions(clientHello *ssl.TLSClientHello) (exts []string, err error) { 92 | if len(clientHello.Extensions) == 0 { 93 | // Needed to preserve commas on the join 94 | return []string{"", "", ""}, nil 95 | } 96 | 97 | var ( 98 | ellipticCurve = "" 99 | ellipticCurvePointFormat = "" 100 | ) 101 | 102 | for _, ext := range clientHello.Extensions { 103 | if !ssl.TLSGreaseCiperSiutes[ext.Type] { 104 | exts = append(exts, strconv.FormatInt(int64(ext.Type), 10)) 105 | } 106 | if ext.Type == 0x0a { 107 | l := uint16(ext.Data[0])<<8 | uint16(ext.Data[1]) 108 | ellipticCurve, err = convertToJa3Segment(ext.Data[2:l+2], 2) 109 | if err != nil { 110 | return nil, err 111 | } 112 | } else if ext.Type == 0x0b { 113 | l := uint8(ext.Data[0]) 114 | ellipticCurvePointFormat, err = convertToJa3Segment(ext.Data[1:l+1], 1) 115 | if err != nil { 116 | return nil, err 117 | } 118 | } 119 | } 120 | 121 | return []string{strings.Join(exts, "-"), ellipticCurve, ellipticCurvePointFormat}, nil 122 | } 123 | -------------------------------------------------------------------------------- /ja3/ja3_test.go: -------------------------------------------------------------------------------- 1 | package ja3 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/gopacket" 7 | "github.com/google/gopacket/layers" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | // testPacketPacket0 is the packet: 12 | // 04:10:16.589867 IP 10.0.14.129.49206 > 185.174.175.14.443: Flags [P.], seq 2134588155:2134588313, ack 135605544, win 64240, length 158 13 | // 0x0000: 0002 1647 96ef 001a 9206 5c7b 0800 4500 ...G......\{..E. 14 | // 0x0010: 00c6 07f5 4000 8006 70ff 0a00 0e81 b9ae ....@...p....... 15 | // 0x0020: af0e c036 01bb 7f3b 3afb 0815 2d28 5018 ...6...;:...-(P. 16 | // 0x0030: faf0 68f2 0000 1603 0300 9901 0000 9503 ..h............. 17 | // 0x0040: 035a f4fb 7795 5f4f fb01 23b7 4f0e a49b .Z..w._O..#.O... 18 | // 0x0050: 26b8 f407 a99a 98d3 40a0 2516 be06 43b0 &.......@.%...C. 19 | // 0x0060: b800 002a 003c 002f 003d 0035 0005 000a ...*.<./.=.5.... 20 | // 0x0070: c027 c013 c014 c02b c023 c02c c024 c009 .'.....+.#.,.$.. 21 | // 0x0080: c00a 0040 0032 006a 0038 0013 0004 0100 ...@.2.j.8...... 22 | // 0x0090: 0042 ff01 0001 0000 0000 1500 1300 0010 .B.............. 23 | // 0x00a0: 726f 6277 6173 736f 7464 696e 742e 7275 robwassotdint.ru 24 | // 0x00b0: 000a 0006 0004 0017 0018 000b 0002 0100 ................ 25 | // 0x00c0: 000d 0010 000e 0401 0501 0201 0403 0503 ................ 26 | // 0x00d0: 0203 0202 .... 27 | var testTLSPacketPacket = []byte{ 28 | 0x00, 0x02, 0x16, 0x47, 0x96, 0xef, 0x00, 0x1a, 0x92, 0x06, 0x5c, 0x7b, 0x08, 0x00, 0x45, 0x00, 29 | 0x00, 0xc6, 0x07, 0xf5, 0x40, 0x00, 0x80, 0x06, 0x70, 0xff, 0x0a, 0x00, 0x0e, 0x81, 0xb9, 0xae, 30 | 0xaf, 0x0e, 0xc0, 0x36, 0x01, 0xbb, 0x7f, 0x3b, 0x3a, 0xfb, 0x08, 0x15, 0x2d, 0x28, 0x50, 0x18, 31 | 0xfa, 0xf0, 0x68, 0xf2, 0x00, 0x00, 0x16, 0x03, 0x03, 0x00, 0x99, 0x01, 0x00, 0x00, 0x95, 0x03, 32 | 0x03, 0x5a, 0xf4, 0xfb, 0x77, 0x95, 0x5f, 0x4f, 0xfb, 0x01, 0x23, 0xb7, 0x4f, 0x0e, 0xa4, 0x9b, 33 | 0x26, 0xb8, 0xf4, 0x07, 0xa9, 0x9a, 0x98, 0xd3, 0x40, 0xa0, 0x25, 0x16, 0xbe, 0x06, 0x43, 0xb0, 34 | 0xb8, 0x00, 0x00, 0x2a, 0x00, 0x3c, 0x00, 0x2f, 0x00, 0x3d, 0x00, 0x35, 0x00, 0x05, 0x00, 0x0a, 35 | 0xc0, 0x27, 0xc0, 0x13, 0xc0, 0x14, 0xc0, 0x2b, 0xc0, 0x23, 0xc0, 0x2c, 0xc0, 0x24, 0xc0, 0x09, 36 | 0xc0, 0x0a, 0x00, 0x40, 0x00, 0x32, 0x00, 0x6a, 0x00, 0x38, 0x00, 0x13, 0x00, 0x04, 0x01, 0x00, 37 | 0x00, 0x42, 0xff, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x13, 0x00, 0x00, 0x10, 38 | 0x72, 0x6f, 0x62, 0x77, 0x61, 0x73, 0x73, 0x6f, 0x74, 0x64, 0x69, 0x6e, 0x74, 0x2e, 0x72, 0x75, 39 | 0x00, 0x0a, 0x00, 0x06, 0x00, 0x04, 0x00, 0x17, 0x00, 0x18, 0x00, 0x0b, 0x00, 0x02, 0x01, 0x00, 40 | 0x00, 0x0d, 0x00, 0x10, 0x00, 0x0e, 0x04, 0x01, 0x05, 0x01, 0x02, 0x01, 0x04, 0x03, 0x05, 0x03, 41 | 0x02, 0x03, 0x02, 0x02, 42 | } 43 | 44 | func TestConvert(t *testing.T) { 45 | p := gopacket.NewPacket(testTLSPacketPacket, layers.LinkTypeEthernet, gopacket.Default) 46 | if p.ErrorLayer() != nil { 47 | t.Error("failed to decode packet:", p.ErrorLayer().Error()) 48 | } 49 | 50 | ja3, ja3Hash := convert(p) 51 | require.Equal(t, "771,60-47-61-53-5-10-49191-49171-49172-49195-49187-49196-49188-49161-49162-64-50-106-56-19-4,65281-0-10-11-13,23-24,0", ja3, "invalid ja3") 52 | require.Equal(t, "4d7a28d6f2263ed61de88ca66eb011e3", ja3Hash, "invalid ja3 hash") 53 | } 54 | -------------------------------------------------------------------------------- /logger/logger.go: -------------------------------------------------------------------------------- 1 | // Package logger provides functions to configure global logger behaviour. 2 | package logger 3 | 4 | import ( 5 | "fmt" 6 | "os" 7 | 8 | log "github.com/Sirupsen/logrus" 9 | ) 10 | 11 | // SetOutput sets output for global logger. 12 | func SetOutput(file string) error { 13 | log.SetFormatter(&log.TextFormatter{ 14 | FullTimestamp: true, 15 | }) 16 | switch file { 17 | case "stdout": 18 | log.SetOutput(os.Stdout) 19 | case "stderr": 20 | log.SetOutput(os.Stderr) 21 | default: 22 | f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) 23 | if err != nil { 24 | return fmt.Errorf("can't set logger output: %s", err) 25 | } 26 | log.SetOutput(f) 27 | } 28 | return nil 29 | } 30 | 31 | // SetLevel sets the standard logger level. 32 | func SetLevel(level string) { 33 | switch level { 34 | case "debug": 35 | log.SetLevel(log.DebugLevel) 36 | case "info": 37 | log.SetLevel(log.InfoLevel) 38 | case "warn": 39 | log.SetLevel(log.WarnLevel) 40 | case "error": 41 | log.SetLevel(log.ErrorLevel) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /logs/bro/parser_test.go: -------------------------------------------------------------------------------- 1 | package bro 2 | 3 | import ( 4 | "io/ioutil" 5 | "net" 6 | "os" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestReaderReadDNS(t *testing.T) { 12 | const ( 13 | filename = "dns.log" 14 | logcontent = `#separator \x09 15 | #set_separator , 16 | #empty_field (empty) 17 | #unset_field - 18 | #path dns 19 | #open 2017-01-01-00-00-00 20 | #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id rtt query qclass qclass_name qtype qtype_name rcode rcode_name AA TC RD RA Z answers TTLs rejected 21 | #types time string addr port addr port enum count interval string count string count string count string bool bool bool bool count vector[string] vector[interval] bool 22 | 1483228800.000000 COSwep1PLjkOcNQdoa 10.0.0.1 52213 10.0.0.1 53 udp 53 - alphasoc.com - - - A 0 NOERROR F F F T 0 35.196.211.126 t0.000000 F 23 | 24 | 1483228800.000000 CDx0B32ubObBNO6lUk 10.0.0.1 52214 10.0.0.1 53 udp 53 - alphasoc.net - - - - 0 NOERROR F F F T 0 35.196.211.126 50.000000 F 25 | #close 2017-01-01-00-00-00 26 | ` 27 | ) 28 | 29 | if _, err := NewFileParser("non-existing-log.json"); err == nil { 30 | t.Fatal("file parser should return error") 31 | } 32 | 33 | if err := ioutil.WriteFile(filename, []byte(logcontent), os.ModePerm); err != nil { 34 | t.Fatalf("write bro log file failed - %s", err) 35 | } 36 | defer os.Remove(filename) 37 | 38 | r, err := NewFileParser(filename) 39 | if err != nil { 40 | t.Fatalf("create sucricata parser failed - %s", err) 41 | } 42 | defer r.Close() 43 | 44 | packets, err := r.ReadDNS() 45 | if err != nil { 46 | t.Fatalf("reading bro log failed - %s", err) 47 | } 48 | 49 | if len(packets) != 2 { 50 | t.Fatalf("reading bro dns package failed - want: 2, got: %d", len(packets)) 51 | } 52 | 53 | tc := time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC) 54 | if !(packets[0].DstPort == 53 && 55 | packets[0].SrcPort == 52213 && 56 | packets[0].Protocol == "udp" && 57 | packets[0].Timestamp.Equal(tc) && 58 | packets[0].RecordType == "A" && 59 | packets[0].FQDN == "alphasoc.com") { 60 | t.Fatal("invalid 1st packet", packets[0]) 61 | } 62 | 63 | if !(packets[1].DstPort == 53 && 64 | packets[1].SrcPort == 52214 && 65 | packets[1].Protocol == "udp" && 66 | packets[1].Timestamp.Equal(tc) && 67 | packets[1].SrcIP.Equal(net.IPv4(10, 0, 0, 1)) && 68 | packets[1].RecordType == "" && 69 | packets[1].FQDN == "alphasoc.net") { 70 | t.Fatal("invalid 2st packet", packets[1]) 71 | } 72 | } 73 | 74 | func TestReaderReadIP(t *testing.T) { 75 | const ( 76 | filename = "ip.log" 77 | logcontent = `#separator \x09 78 | #set_separator , 79 | #empty_field (empty) 80 | #unset_field - 81 | #path conn 82 | #open 2018-01-24-01-01-01 83 | #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents 84 | #types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] 85 | 1483228800.000000 1 10.0.0.1 5021 10.0.0.2 22 udp ssh - - - S0 - - 0 D 1 10 0 0 (empty) 86 | 1483228800.000000 2 10.0.0.1 3210 10.0.0.2 22 tcp ssh - - 20 S1 - - 0 D 1 0 0 0 (empty)` 87 | ) 88 | 89 | if _, err := NewFileParser("non-existing-log.json"); err == nil { 90 | t.Fatal("file parser should return error") 91 | } 92 | 93 | if err := ioutil.WriteFile(filename, []byte(logcontent), os.ModePerm); err != nil { 94 | t.Fatalf("write bro log file failed - %s", err) 95 | } 96 | defer os.Remove(filename) 97 | 98 | r, err := NewFileParser(filename) 99 | if err != nil { 100 | t.Fatalf("create bro parser failed - %s", err) 101 | } 102 | defer r.Close() 103 | 104 | packets, err := r.ReadIP() 105 | if err != nil { 106 | t.Fatalf("reading bro log failed - %s", err) 107 | } 108 | 109 | if len(packets) != 2 { 110 | t.Fatalf("reading bro dns package failed - want: 2, got: %d", len(packets)) 111 | } 112 | 113 | tc := time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC) 114 | if !(packets[0].SrcIP.Equal(net.IPv4(10, 0, 0, 1)) && 115 | packets[0].SrcPort == 5021 && 116 | packets[0].DstIP.Equal(net.IPv4(10, 0, 0, 2)) && 117 | packets[0].DstPort == 22 && 118 | packets[0].Protocol == "udp" && 119 | packets[0].Timestamp.Equal(tc) && 120 | packets[0].BytesCount == 10) { 121 | t.Fatal("invalid 1st packet") 122 | } 123 | 124 | if !(packets[1].SrcIP.Equal(net.IPv4(10, 0, 0, 1)) && 125 | packets[1].SrcPort == 3210 && 126 | packets[1].DstIP.Equal(net.IPv4(10, 0, 0, 2)) && 127 | packets[1].DstPort == 22 && 128 | packets[1].Protocol == "tcp" && 129 | packets[1].Timestamp.Equal(tc) && 130 | packets[1].BytesCount == 20) { 131 | t.Fatal("invalid 2nd packet") 132 | } 133 | } 134 | 135 | func TestReaderReadHTTP(t *testing.T) { 136 | const ( 137 | filename = "http.log" 138 | logcontent = `#separator \x09 139 | #set_separator , 140 | #empty_field (empty) 141 | #unset_field - 142 | #path http 143 | #open 2019-10-01-16-08-28 144 | #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer version user_agent request_body_len response_body_len status_code status_msg info_code info_msg tags username password proxied orig_fuids orig_filenames orig_mime_types resp_fuids resp_filenames resp_mime_types 145 | #types time string addr port addr port count string string string string string string count count count string count string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] vector[string] vector[string] 146 | 1569938908.384253 C2YBbB4oCIHnYJQbhh 10.0.0.1 50434 17.253.107.201 80 1 GET ocsp.apple.com /ocsp-devid01/ME4wTKADAgEAMEUwQzBBMAkGBSsOAwIaBQAEFDOB0e/baLCFIU0u76+MSmlkPCpsBBRXF+2iz9x8mKEQ4Py+hy0s8uMXVAIIFelDYw2P904= - 1.1 com.apple.trustd/2.0 0 3698 200 OK - - (empty) - - - - - - FlUaB7nbVNzuhJpgh - application/ocsp-response 147 | 1569938911.839230 CPgxUR3KvtWXSHXpRl 10.0.0.2 50450 80.252.0.235 80 1 GET gazeta.hit.gemius.pl /gemius.js http://wyborcza.pl/0,0.html 1.1 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Safari/605.1.15 0 0 304 Not Modified - - (empty) - - - - - - - - - 148 | ` 149 | ) 150 | 151 | if _, err := NewFileParser("non-existing-log.json"); err == nil { 152 | t.Fatal("file parser should return error") 153 | } 154 | 155 | if err := ioutil.WriteFile(filename, []byte(logcontent), os.ModePerm); err != nil { 156 | t.Fatalf("write bro log file failed - %s", err) 157 | } 158 | defer os.Remove(filename) 159 | 160 | r, err := NewFileParser(filename) 161 | if err != nil { 162 | t.Fatalf("create bro parser failed - %s", err) 163 | } 164 | defer r.Close() 165 | 166 | packets, err := r.ReadHTTP() 167 | if err != nil { 168 | t.Fatalf("reading bro log failed - %s", err) 169 | } 170 | 171 | if len(packets) != 2 { 172 | t.Fatalf("reading bro dns package failed - want: 2, got: %d", len(packets)) 173 | } 174 | 175 | if !(packets[0].Timestamp.Equal(time.Date(2019, 10, 1, 14, 8, 28, 384253, time.UTC)) && 176 | packets[0].SrcIP.Equal(net.IPv4(10, 0, 0, 1)) && 177 | packets[0].SrcPort == 50434 && 178 | packets[0].URL == "http://ocsp.apple.com/ocsp-devid01/ME4wTKADAgEAMEUwQzBBMAkGBSsOAwIaBQAEFDOB0e/baLCFIU0u76+MSmlkPCpsBBRXF+2iz9x8mKEQ4Py+hy0s8uMXVAIIFelDYw2P904=" && 179 | packets[0].Method == "GET" && 180 | packets[0].Referrer == "" && 181 | packets[0].Status == 200 && 182 | packets[0].BytesOut == 0 && 183 | packets[0].BytesIn == 3698 && 184 | packets[0].ContentType == "application/ocsp-response" && 185 | packets[0].UserAgent == "com.apple.trustd/2.0") { 186 | t.Errorf("invalid 1st packet: %+v", packets[0]) 187 | } 188 | 189 | if !(packets[1].Timestamp.Equal(time.Date(2019, 10, 1, 14, 8, 31, 839230, time.UTC)) && 190 | packets[1].SrcIP.Equal(net.IPv4(10, 0, 0, 2)) && 191 | packets[1].SrcPort == 50450 && 192 | packets[1].URL == "http://gazeta.hit.gemius.pl/gemius.js" && 193 | packets[1].Method == "GET" && 194 | packets[1].Referrer == "http://wyborcza.pl/0,0.html" && 195 | packets[1].Status == 304 && 196 | packets[1].BytesOut == 0 && 197 | packets[1].BytesIn == 0 && 198 | packets[1].ContentType == "" && 199 | packets[1].UserAgent == "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Safari/605.1.15") { 200 | t.Errorf("invalid 2nd packet: %+v", packets[1]) 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /logs/bro/timestamp.go: -------------------------------------------------------------------------------- 1 | package bro 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | "time" 8 | ) 9 | 10 | func parseEpochTime(t string) (time.Time, error) { 11 | s := strings.Split(t, ".") 12 | if len(s) != 2 { 13 | return time.Time{}, fmt.Errorf("invalid timestamp %s", t) 14 | } 15 | 16 | sec, err := strconv.ParseInt(s[0], 10, 64) 17 | if err != nil { 18 | return time.Time{}, fmt.Errorf("invalid timestamp %s", t) 19 | } 20 | 21 | nsec, err := strconv.ParseInt(s[1], 10, 64) 22 | if err != nil { 23 | return time.Time{}, fmt.Errorf("invalid timestamp %s", t) 24 | } 25 | 26 | return time.Unix(sec, nsec), nil 27 | } 28 | -------------------------------------------------------------------------------- /logs/edge/parser.go: -------------------------------------------------------------------------------- 1 | package edge 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "net" 8 | "os" 9 | "strings" 10 | "time" 11 | 12 | "github.com/alphasoc/nfr/client" 13 | "github.com/alphasoc/nfr/packet" 14 | ) 15 | 16 | // dnsLog represents single DNS log entry 17 | type dnsLog struct { 18 | Time int64 `json:"time"` 19 | Source string `json:"source"` 20 | Query string `json:"query"` 21 | QueryType string `json:"queryType"` 22 | Protocol string `json:"queryProtocol"` 23 | } 24 | 25 | func (l *dnsLog) toPacket() (*packet.DNSPacket, error) { 26 | if l.Time <= 0 { 27 | return nil, fmt.Errorf("invalid time: %d", l.Time) 28 | } 29 | srcIP := net.ParseIP(l.Source) 30 | if srcIP == nil { 31 | return nil, fmt.Errorf("invalid source (must be valid IP): %s", l.Source) 32 | } 33 | 34 | return &packet.DNSPacket{ 35 | Timestamp: time.Unix(l.Time/1000, (l.Time%1000)*int64(time.Millisecond)), 36 | SrcIP: srcIP, 37 | Protocol: strings.ToLower(l.Protocol), 38 | FQDN: l.Query, 39 | RecordType: l.QueryType, 40 | }, nil 41 | } 42 | 43 | // A Parser parses and reads network events from edge logs. 44 | type Parser struct { 45 | r io.ReadCloser 46 | } 47 | 48 | // NewParser creates new edge parser. 49 | func NewParser() *Parser { 50 | return &Parser{} 51 | } 52 | 53 | // NewFileParser creates new edge reader from given file. 54 | func NewFileParser(filename string) (*Parser, error) { 55 | f, err := os.Open(filename) 56 | if err != nil { 57 | return nil, err 58 | } 59 | 60 | return &Parser{r: f}, nil 61 | } 62 | 63 | // ReadDNS reads all dns packets from the file. 64 | func (p *Parser) ReadDNS() ([]*packet.DNSPacket, error) { 65 | if p.r == nil { 66 | return nil, fmt.Errorf("parser must be created with file reader") 67 | } 68 | 69 | var entries []dnsLog 70 | dec := json.NewDecoder(p.r) 71 | if err := dec.Decode(&entries); err != nil { 72 | return nil, err 73 | } 74 | 75 | res := make([]*packet.DNSPacket, 0, len(entries)) 76 | for n := range entries { 77 | p, err := entries[n].toPacket() 78 | if err != nil { 79 | return nil, fmt.Errorf("failed to parse event %d: %s", n, err) 80 | } 81 | res = append(res, p) 82 | } 83 | 84 | return res, nil 85 | } 86 | 87 | // ReadIP reads all ip packets from the file. 88 | func (*Parser) ReadIP() ([]*packet.IPPacket, error) { 89 | return nil, nil 90 | } 91 | 92 | // ParseLineDNS parse single log line with dns data. 93 | func (*Parser) ParseLineDNS(line string) (*packet.DNSPacket, error) { 94 | return nil, nil 95 | } 96 | 97 | // ParseLineIP reads all ip packets from the file. 98 | func (*Parser) ParseLineIP(line string) (*packet.IPPacket, error) { 99 | return nil, nil 100 | } 101 | 102 | func (*Parser) ReadHTTP() ([]*client.HTTPEntry, error) { 103 | return nil, nil 104 | } 105 | 106 | func (*Parser) ParseLineHTTP(line string) (*client.HTTPEntry, error) { 107 | return nil, nil 108 | } 109 | 110 | // Close underlying log file. 111 | func (p *Parser) Close() error { 112 | return p.r.Close() 113 | } 114 | -------------------------------------------------------------------------------- /logs/edge/parser_test.go: -------------------------------------------------------------------------------- 1 | package edge 2 | 3 | import ( 4 | "io/ioutil" 5 | "net" 6 | "os" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestReaderReadDNS(t *testing.T) { 12 | const ( 13 | filename = "edge.json" 14 | logcontent = `[ 15 | {"time":1529603514134,"source":"10.0.0.1","query":"google.com.","queryType":"A","queryProtocol":"UDP"}, 16 | {"time":1529603514134,"source":"10.0.0.2","query":"api.google.com.","queryType":"AAAA","queryProtocol":"TCP"} 17 | ]` 18 | ) 19 | 20 | if _, err := NewFileParser("non-existing-log.json"); err == nil { 21 | t.Fatal("NewFileParser should return error") 22 | } 23 | 24 | if err := ioutil.WriteFile(filename, []byte(logcontent), os.ModePerm); err != nil { 25 | t.Fatalf("write log file failed - %s", err) 26 | } 27 | defer os.Remove(filename) 28 | 29 | r, err := NewFileParser(filename) 30 | if err != nil { 31 | t.Fatalf("create reader failed - %s", err) 32 | } 33 | defer r.Close() 34 | 35 | packets, err := r.ReadDNS() 36 | if err != nil { 37 | t.Fatalf("reading log failed - %s", err) 38 | } 39 | 40 | if len(packets) != 2 { 41 | t.Fatalf("expected 2 packets, got %d", len(packets)) 42 | } 43 | 44 | tc := time.Date(2018, 6, 21, 17, 51, 54, 134000000, time.UTC) 45 | if !(packets[0].Protocol == "udp" && 46 | packets[0].Timestamp.Equal(tc) && 47 | packets[0].SrcIP.Equal(net.IPv4(10, 0, 0, 1)) && 48 | packets[0].RecordType == "A" && 49 | packets[0].FQDN == "google.com.") { 50 | t.Fatalf("invalid 1st packet %+q", packets[0]) 51 | } 52 | 53 | if !(packets[1].Protocol == "tcp" && 54 | packets[1].Timestamp.Equal(tc) && 55 | packets[1].SrcIP.Equal(net.IPv4(10, 0, 0, 2)) && 56 | packets[1].RecordType == "AAAA" && 57 | packets[1].FQDN == "api.google.com.") { 58 | t.Fatalf("invalid 2nd packet %+q", packets[1]) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /logs/msdns/parser.go: -------------------------------------------------------------------------------- 1 | package msdns 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "net" 8 | "os" 9 | "regexp" 10 | "strings" 11 | "time" 12 | 13 | "github.com/alphasoc/nfr/client" 14 | "github.com/alphasoc/nfr/packet" 15 | ) 16 | 17 | // A Parser parses and reads network events from msdns logs. 18 | type Parser struct { 19 | // TimeFormat used for parsing timestamp. 20 | // If not set, will accept the following format: 21 | // 2006-01-02 3:04:05 PM 22 | // 2006/01/02 3:04:05 PM 23 | // 2006-01-02 15:04:05 24 | // 2006/01/02 15:04:05 25 | TimeFormat string 26 | 27 | r io.ReadCloser 28 | } 29 | 30 | // NewParser creates new msdns parser. 31 | func NewParser() *Parser { 32 | return &Parser{} 33 | } 34 | 35 | // NewFileParser creates new msdns reader from given file. 36 | func NewFileParser(filename string) (*Parser, error) { 37 | f, err := os.Open(filename) 38 | if err != nil { 39 | return nil, err 40 | } 41 | 42 | return &Parser{r: f}, nil 43 | } 44 | 45 | // ReadDNS reads all dns packets from the file. 46 | func (p *Parser) ReadDNS() ([]*packet.DNSPacket, error) { 47 | if p.r == nil { 48 | return nil, fmt.Errorf("msdns parser must be created with file reader") 49 | } 50 | 51 | var packets []*packet.DNSPacket 52 | 53 | s := bufio.NewScanner(p.r) 54 | for s.Scan() { 55 | dnspacket, err := p.ParseLineDNS(string(s.Bytes())) 56 | if err != nil { 57 | return nil, err 58 | } 59 | if dnspacket != nil { 60 | packets = append(packets, dnspacket) 61 | } 62 | } 63 | 64 | if err := s.Err(); err != nil { 65 | return nil, err 66 | } 67 | return packets, nil 68 | } 69 | 70 | var numToDotRe = regexp.MustCompile(`\(\d+\)`) 71 | 72 | func guessTimeFormat(s string) string { 73 | if len(s) < 10 { 74 | return "" 75 | } 76 | 77 | sep := s[4] // date separator 78 | ampm := (s[len(s)-1] == 'M') // use 12h clock (AM/PM) 79 | 80 | switch true { 81 | case sep == '-' && ampm: 82 | return "2006-01-02 3:04:05 PM" 83 | case sep == '-' && !ampm: 84 | return "2006-01-02 15:04:05" 85 | case sep == '/' && ampm: 86 | return "2006/01/02 3:04:05 PM" 87 | case sep == '/' && !ampm: 88 | return "2006/01/02 15:04:05" 89 | } 90 | 91 | return "" 92 | } 93 | 94 | // ParseLineDNS parse single log line with dns data. 95 | func (p *Parser) ParseLineDNS(line string) (*packet.DNSPacket, error) { 96 | s := strings.Fields(line) 97 | 98 | if len(s) < 15 { 99 | return nil, nil 100 | } 101 | 102 | // find Context field, which can be 4th or 5th field, 103 | // depends if timestamp consists of 2 or 3 fields. 104 | contextIdx := 0 105 | switch "PACKET" { 106 | case s[3]: 107 | contextIdx = 3 108 | case s[4]: 109 | contextIdx = 4 110 | default: 111 | return nil, nil 112 | } 113 | 114 | ts := strings.Join(s[:contextIdx-1], " ") 115 | s = s[contextIdx:] 116 | 117 | // fields are now starting with Context, expect 12 fields 118 | if len(s) != 12 || s[3] != "Rcv" || s[6] != "Q" { 119 | return nil, nil 120 | } 121 | 122 | timeFormat := p.TimeFormat 123 | if timeFormat == "" { 124 | timeFormat = guessTimeFormat(ts) 125 | } 126 | if timeFormat == "" { 127 | return nil, fmt.Errorf("Unknown time format for timestamp: %s", ts) 128 | } 129 | 130 | timestamp, err := time.ParseInLocation(timeFormat, ts, time.Local) 131 | if err != nil { 132 | return nil, err 133 | } 134 | 135 | srcIP := net.ParseIP(s[4]) 136 | if err != nil { 137 | return nil, fmt.Errorf("invalid source ip at line %s", line) 138 | } 139 | 140 | return &packet.DNSPacket{ 141 | DstPort: 0, 142 | Protocol: strings.ToLower(s[2]), 143 | Timestamp: timestamp, 144 | SrcIP: srcIP, 145 | RecordType: s[10], 146 | FQDN: strings.Trim(numToDotRe.ReplaceAllString(s[11], "."), "."), 147 | }, nil 148 | } 149 | 150 | // ReadIP reads all ip packets from the file. 151 | func (*Parser) ReadIP() ([]*packet.IPPacket, error) { 152 | return nil, nil 153 | } 154 | 155 | // ParseLineIP reads all ip packets from the file. 156 | func (*Parser) ParseLineIP(line string) (*packet.IPPacket, error) { 157 | return nil, nil 158 | } 159 | 160 | func (*Parser) ReadHTTP() ([]*client.HTTPEntry, error) { 161 | return nil, nil 162 | } 163 | 164 | func (*Parser) ParseLineHTTP(line string) (*client.HTTPEntry, error) { 165 | return nil, nil 166 | } 167 | 168 | // Close underlying log file. 169 | func (p *Parser) Close() error { 170 | return p.r.Close() 171 | } 172 | -------------------------------------------------------------------------------- /logs/msdns/parser_test.go: -------------------------------------------------------------------------------- 1 | package msdns 2 | 3 | import ( 4 | "io/ioutil" 5 | "net" 6 | "os" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestReaderReadDNSNonExistingFiel(t *testing.T) { 12 | if _, err := NewFileParser("non-existing-log.json"); err == nil { 13 | t.Fatal("NewFileParser should return error") 14 | } 15 | } 16 | 17 | func TestReaderReadDNS(t *testing.T) { 18 | const ( 19 | filename = "msdns.log" 20 | logcontent = ` 21 | DNS Server log file creation at 2017-01-01 00:00:00 22 | Message logging key (for packets - other items use a subset of these fields): 23 | Field # Information Values 24 | ------- ----------- ------ 25 | 1 Date 26 | 2 Time 27 | 3 Thread ID 28 | 4 Context 29 | 5 Internal packet identifier 30 | 6 UDP/TCP indicator 31 | 7 Send/Receive indicator 32 | 8 Remote IP 33 | 9 Xid (hex) 34 | 10 Query/Response R = Response 35 | blank = Query 36 | 11 Opcode Q = Standard Query 37 | N = Notify 38 | U = Update 39 | ? = Unknown 40 | 12 [ Flags (hex) 41 | 13 Flags (char codes) A = Authoritative Answer 42 | T = Truncated Response 43 | D = Recursion Desired 44 | R = Recursion Available 45 | 14 ResponseCode ] 46 | 15 Question Type 47 | 16 Question Name 48 | 2017-01-02 00:00:00 01A0 EVENT The DNS server did not detect any zones of either primary or secondary type during initialization. It will not be authoritative for any zones, and it will run as a caching-only server until a zone is loaded manually or by Active Directory replication. For more information, see the online Help. 49 | 2017-01-02 00:00:00 01A0 EVENT The DNS server has started. 50 | 2017-01-02 21:00:00 0DB8 PACKET 0000000001962BB0 UDP Rcv 10.0.0.1 0030 Q [0001 D NOERROR] A (8)alphasoc(3)com(0) 51 | 2017-01-02 00:00:00 0DB8 PACKET 0000000001962BB0 UDP Snd 127.0.0.1 0030 Q [0001 D NOERROR] A (8)alphasoc(3)com(0) 52 | 2017/01/02 09:00:00 PM 0DB8 PACKET 0000000001962BB0 TCP Rcv 10.0.0.2 0030 Q [0001 D NOERROR] AAAA (8)alphasoc(3)net(0) 53 | ` 54 | ) 55 | 56 | if err := ioutil.WriteFile(filename, []byte(logcontent), os.ModePerm); err != nil { 57 | t.Fatalf("write msdns log file failed - %s", err) 58 | } 59 | defer os.Remove(filename) 60 | 61 | r, err := NewFileParser(filename) 62 | if err != nil { 63 | t.Fatalf("create msdns reader failed - %s", err) 64 | } 65 | defer r.Close() 66 | 67 | packets, err := r.ReadDNS() 68 | if err != nil { 69 | t.Fatalf("reading msdns log failed - %s", err) 70 | } 71 | 72 | if len(packets) != 2 { 73 | t.Fatalf("reading msdns dns package failed - want: 2, got: %d", len(packets)) 74 | } 75 | 76 | tc := time.Date(2017, 1, 2, 21, 0, 0, 0, time.Local) 77 | if !(packets[0].DstPort == 0 && 78 | packets[0].Protocol == "udp" && 79 | packets[0].Timestamp.Equal(tc) && 80 | packets[0].SrcIP.Equal(net.IPv4(10, 0, 0, 1)) && 81 | packets[0].RecordType == "A" && 82 | packets[0].FQDN == "alphasoc.com") { 83 | t.Errorf("invalid 1st packet %+q", packets[0]) 84 | } 85 | 86 | if !(packets[1].DstPort == 0 && 87 | packets[1].Protocol == "tcp" && 88 | packets[1].Timestamp.Equal(tc) && 89 | packets[1].SrcIP.Equal(net.IPv4(10, 0, 0, 2)) && 90 | packets[1].RecordType == "AAAA" && 91 | packets[1].FQDN == "alphasoc.net") { 92 | t.Errorf("invalid 2nd packet %+q", packets[1]) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /logs/parser.go: -------------------------------------------------------------------------------- 1 | package logs 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/alphasoc/nfr/client" 7 | "github.com/alphasoc/nfr/packet" 8 | ) 9 | 10 | // FileParser is the interface what wraps ip and dns parser. 11 | type FileParser interface { 12 | ReadDNS() ([]*packet.DNSPacket, error) 13 | ReadIP() ([]*packet.IPPacket, error) 14 | ReadHTTP() ([]*client.HTTPEntry, error) 15 | io.Closer 16 | } 17 | 18 | // Parser is the interface what wraps ip and dns parser. 19 | type Parser interface { 20 | ParseLineDNS(line string) (*packet.DNSPacket, error) 21 | ParseLineIP(line string) (*packet.IPPacket, error) 22 | ParseLineHTTP(line string) (*client.HTTPEntry, error) 23 | } 24 | -------------------------------------------------------------------------------- /logs/pcap/pcap.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphasoc/nfr/03d7c4b8b628ba2c68a22c60a42b5a6329e09e44/logs/pcap/pcap.log -------------------------------------------------------------------------------- /logs/pcap/reader.go: -------------------------------------------------------------------------------- 1 | package pcap 2 | 3 | import ( 4 | "github.com/alphasoc/nfr/client" 5 | "github.com/alphasoc/nfr/packet" 6 | "github.com/google/gopacket" 7 | "github.com/google/gopacket/pcap" 8 | ) 9 | 10 | // A Reader reads network events from pcap logs. 11 | type Reader struct { 12 | handle *pcap.Handle 13 | } 14 | 15 | // NewReader creates new pcap reader. Logs will be read from given file. 16 | func NewReader(filename string) (*Reader, error) { 17 | handle, err := pcap.OpenOffline(filename) 18 | if err != nil { 19 | return nil, err 20 | } 21 | 22 | return &Reader{handle: handle}, nil 23 | } 24 | 25 | // ReadDNS reads all dns packets from the file. 26 | func (r *Reader) ReadDNS() ([]*packet.DNSPacket, error) { 27 | var packets []*packet.DNSPacket 28 | 29 | source := gopacket.NewPacketSource(r.handle, r.handle.LinkType()) 30 | for raw := range source.Packets() { 31 | if dnspacket := packet.NewDNSPacket(raw); dnspacket != nil { 32 | packets = append(packets, dnspacket) 33 | } 34 | } 35 | return packets, nil 36 | } 37 | 38 | // ReadIP reads all ip packets from the file. 39 | func (r *Reader) ReadIP() ([]*packet.IPPacket, error) { 40 | var packets []*packet.IPPacket 41 | 42 | source := gopacket.NewPacketSource(r.handle, r.handle.LinkType()) 43 | for raw := range source.Packets() { 44 | if ippacket := packet.NewIPPacket(raw); ippacket != nil { 45 | packets = append(packets, ippacket) 46 | } 47 | } 48 | return packets, nil 49 | } 50 | 51 | func (*Reader) ReadHTTP() ([]*client.HTTPEntry, error) { 52 | return nil, nil 53 | } 54 | 55 | // Close underlying log file. 56 | func (r *Reader) Close() error { 57 | r.handle.Close() 58 | return nil 59 | } 60 | -------------------------------------------------------------------------------- /logs/pcap/reader_test.go: -------------------------------------------------------------------------------- 1 | package pcap 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestReader(t *testing.T) { 8 | if _, err := NewReader("no.data"); err == nil { 9 | t.Fatal("sniffer create without error for non existing file") 10 | } 11 | } 12 | 13 | func TestReadDNS(t *testing.T) { 14 | r, err := NewReader("pcap.log") 15 | if err != nil { 16 | t.Fatal(err) 17 | } 18 | defer r.Close() 19 | 20 | packets, err := r.ReadDNS() 21 | if err != nil { 22 | t.Fatal(err) 23 | } 24 | 25 | if len(packets) != 1 { 26 | t.Errorf("invalid packet count - got: %d, expected: 1", len(packets)) 27 | } 28 | } 29 | 30 | func TestReadIP(t *testing.T) { 31 | r, err := NewReader("pcap.log") 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | defer r.Close() 36 | 37 | packets, err := r.ReadIP() 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | 42 | if len(packets) != 2 { 43 | t.Errorf("invalid packet count - got: %d, expected: 1", len(packets)) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /logs/suricata/parser.go: -------------------------------------------------------------------------------- 1 | package suricata 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net" 9 | "os" 10 | "path" 11 | "strings" 12 | "time" 13 | 14 | "github.com/alphasoc/nfr/client" 15 | "github.com/alphasoc/nfr/packet" 16 | ) 17 | 18 | // logEntry represents single log file in used in surciata eve output. 19 | type logEntry struct { 20 | Timestamp timestamp `json:"timestamp"` 21 | SrcIP string `json:"src_ip"` 22 | SrcPort uint16 `json:"src_port"` 23 | DestPort int `json:"dest_port"` 24 | Proto string `json:"proto"` 25 | DNS struct { 26 | Type string `json:"type"` 27 | Rrname string `json:"rrname"` 28 | Rrtype string `json:"rrtype"` 29 | } `json:"dns"` 30 | HTTP struct { 31 | Hostname string `json:"hostname"` 32 | URL string `json:"url"` 33 | HTTPUserAgent string `json:"http_user_agent"` 34 | HTTPContentType string `json:"http_content_type"` 35 | HTTPRefer string `json:"http_refer"` 36 | HTTPMethod string `json:"http_method"` 37 | Protocol string `json:"protocol"` 38 | Status int `json:"status"` 39 | Length int `json:"length"` 40 | } `json:"http"` 41 | TLS struct { 42 | SessionResumed bool `json:"session_resumed"` 43 | Sni string `json:"sni"` 44 | Version string `json:"version"` 45 | Ja3 struct { 46 | Hash string `json:"hash"` 47 | String string `json:"string"` 48 | } `json:"ja3"` 49 | } `json:"tls"` 50 | } 51 | 52 | // A Parser parses and reads network events from suricata logs. 53 | type Parser struct { 54 | r io.ReadCloser 55 | } 56 | 57 | // NewParser creates new suricata parser. 58 | func NewParser() *Parser { 59 | return &Parser{} 60 | } 61 | 62 | // NewFileParser creates new suricata reader from given file. 63 | func NewFileParser(filename string) (*Parser, error) { 64 | f, err := os.Open(filename) 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | return &Parser{r: f}, nil 70 | } 71 | 72 | // ReadDNS reads all dns packets from the file. 73 | func (p *Parser) ReadDNS() ([]*packet.DNSPacket, error) { 74 | if p.r == nil { 75 | return nil, fmt.Errorf("suricata parser must be created with file reader") 76 | } 77 | 78 | var packets []*packet.DNSPacket 79 | 80 | s := bufio.NewScanner(p.r) 81 | for s.Scan() { 82 | dnspacket, err := p.ParseLineDNS(string(s.Bytes())) 83 | if err != nil { 84 | return nil, err 85 | } 86 | if dnspacket != nil { 87 | packets = append(packets, dnspacket) 88 | } 89 | } 90 | 91 | if err := s.Err(); err != nil { 92 | return nil, err 93 | } 94 | return packets, nil 95 | } 96 | 97 | // ReadIP reads all ip packets from the file. 98 | func (*Parser) ReadIP() ([]*packet.IPPacket, error) { 99 | return nil, nil 100 | } 101 | 102 | // ParseLineDNS parse single log line with dns data. 103 | func (*Parser) ParseLineDNS(line string) (*packet.DNSPacket, error) { 104 | if len(line) == 0 { 105 | return nil, nil 106 | } 107 | 108 | var entry logEntry 109 | if err := json.Unmarshal([]byte(line), &entry); err != nil { 110 | return nil, fmt.Errorf("suricata %s", err) 111 | } 112 | 113 | if entry.DNS.Type != "query" { 114 | return nil, nil 115 | } 116 | 117 | return &packet.DNSPacket{ 118 | DstPort: entry.DestPort, 119 | Protocol: strings.ToLower(entry.Proto), 120 | Timestamp: time.Time(entry.Timestamp), 121 | SrcIP: net.ParseIP(entry.SrcIP), 122 | RecordType: entry.DNS.Rrtype, 123 | FQDN: entry.DNS.Rrname, 124 | }, nil 125 | } 126 | 127 | // ParseLineIP reads all ip packets from the file. 128 | func (*Parser) ParseLineIP(line string) (*packet.IPPacket, error) { 129 | return nil, nil 130 | } 131 | 132 | func (p *Parser) ReadHTTP() ([]*client.HTTPEntry, error) { 133 | if p.r == nil { 134 | return nil, fmt.Errorf("suricata parser must be created with file reader") 135 | } 136 | 137 | var packets []*client.HTTPEntry 138 | 139 | s := bufio.NewScanner(p.r) 140 | for s.Scan() { 141 | dnspacket, err := p.ParseLineHTTP(string(s.Bytes())) 142 | if err != nil { 143 | return nil, err 144 | } 145 | if dnspacket != nil { 146 | packets = append(packets, dnspacket) 147 | } 148 | } 149 | 150 | if err := s.Err(); err != nil { 151 | return nil, err 152 | } 153 | return packets, nil 154 | 155 | } 156 | 157 | func (p *Parser) ParseLineHTTP(line string) (*client.HTTPEntry, error) { 158 | if len(line) == 0 { 159 | return nil, nil 160 | } 161 | 162 | var entry logEntry 163 | if err := json.Unmarshal([]byte(line), &entry); err != nil { 164 | return nil, fmt.Errorf("suricata %s", err) 165 | } 166 | 167 | if entry.HTTP.Hostname == "" { 168 | return nil, nil 169 | } 170 | 171 | // TODO: if different port then attach to URL? 172 | schema := "" 173 | switch entry.DestPort { 174 | case 80: 175 | schema = "http://" 176 | case 443: 177 | schema = "https://" 178 | } 179 | 180 | return &client.HTTPEntry{ 181 | Timestamp: time.Time(entry.Timestamp), 182 | SrcIP: net.ParseIP(entry.SrcIP), 183 | SrcPort: entry.SrcPort, 184 | 185 | URL: schema + path.Join(entry.HTTP.Hostname, entry.HTTP.URL), 186 | Method: entry.HTTP.HTTPMethod, 187 | Status: entry.HTTP.Status, 188 | ContentType: entry.HTTP.HTTPContentType, 189 | Referrer: entry.HTTP.HTTPRefer, 190 | UserAgent: entry.HTTP.HTTPUserAgent, 191 | 192 | // Action: 193 | // BytesIn: 194 | // BytesOut: entry.HTTP.Length ? 195 | }, nil 196 | } 197 | 198 | // Close underlying log file. 199 | func (p *Parser) Close() error { 200 | return p.r.Close() 201 | } 202 | -------------------------------------------------------------------------------- /logs/suricata/parser_test.go: -------------------------------------------------------------------------------- 1 | package suricata 2 | 3 | import ( 4 | "io/ioutil" 5 | "net" 6 | "os" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestReaderReadDNS(t *testing.T) { 12 | const ( 13 | filename = "suricata-eve.json" 14 | logcontent = ` 15 | {"timestamp":"2017-01-01T00:00:00.000000+0000","flow_id":1,"in_iface":"eth0","event_type":"dns","src_ip":"10.0.0.1","dest_ip":"10.0.0.1","dest_port":53,"proto":"UDP","dns":{"type":"query","id":1,"rrname":"alphasoc.com","rrtype":"A","tx_id":0}} 16 | {"timestamp":"2017-01-01T00:00:00.000000+0000","flow_id":1,"in_iface":"eth0","event_type":"dns","src_ip":"10.0.0.2","dest_ip":"10.0.0.1","dest_port":1053,"proto":"TCP","dns":{"type":"answer","id":1,"rcode":"NOERROR","rrname":"alphasoc.com","rrtype":"AAAA","ttl":300,"rdata":"35.196.211.126"}} 17 | {"timestamp":"2017-01-01T00:00:00.000000+0000","flow_id":2,"in_iface":"eth0","event_type":"dns","src_ip":"10.0.0.2","dest_ip":"10.0.0.1","dest_port":1053,"proto":"TCP","dns":{"type":"query","id":1,"rrname":"alphasoc.net","rrtype":"AAAA","tx_id":0}} 18 | ` 19 | ) 20 | 21 | if _, err := NewFileParser("non-existing-log.json"); err == nil { 22 | t.Fatal("NewFileParser should return error") 23 | } 24 | 25 | if err := ioutil.WriteFile(filename, []byte(logcontent), os.ModePerm); err != nil { 26 | t.Fatalf("write suricata log file failed - %s", err) 27 | } 28 | defer os.Remove(filename) 29 | 30 | r, err := NewFileParser(filename) 31 | if err != nil { 32 | t.Fatalf("create suricata reader failed - %s", err) 33 | } 34 | defer r.Close() 35 | 36 | packets, err := r.ReadDNS() 37 | if err != nil { 38 | t.Fatalf("reading suricata log failed - %s", err) 39 | } 40 | 41 | if len(packets) != 2 { 42 | t.Fatalf("reading suricata dns package failed - want: 2, got: %d", len(packets)) 43 | } 44 | 45 | tc := time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC) 46 | if !(packets[0].DstPort == 53 && 47 | packets[0].Protocol == "udp" && 48 | packets[0].Timestamp.Equal(tc) && 49 | packets[0].SrcIP.Equal(net.IPv4(10, 0, 0, 1)) && 50 | packets[0].RecordType == "A" && 51 | packets[0].FQDN == "alphasoc.com") { 52 | t.Fatalf("invalid 1st packet %+q", packets[0]) 53 | } 54 | 55 | if !(packets[1].DstPort == 1053 && 56 | packets[1].Protocol == "tcp" && 57 | packets[1].Timestamp.Equal(tc) && 58 | packets[1].SrcIP.Equal(net.IPv4(10, 0, 0, 2)) && 59 | packets[1].RecordType == "AAAA" && 60 | packets[1].FQDN == "alphasoc.net") { 61 | t.Fatalf("invalid 2nd packet %+q", packets[1]) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /logs/suricata/timestamp.go: -------------------------------------------------------------------------------- 1 | package suricata 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | "time" 7 | ) 8 | 9 | // timestamp type to use when parse eve log entries. 10 | type timestamp time.Time 11 | 12 | // time format used in suricata eve logs. 13 | const timestampFormat = "2006-01-02T15:04:05.999999999-0700" 14 | 15 | func (t *timestamp) UnmarshalJSON(b []byte) error { 16 | s := strings.Trim(string(b), "\"") 17 | _t, err := time.Parse(timestampFormat, s) 18 | if err != nil { 19 | return err 20 | } 21 | *t = timestamp(_t) 22 | return nil 23 | 24 | } 25 | 26 | func (t *timestamp) MarshalJSON() ([]byte, error) { 27 | return json.Marshal(t) 28 | } 29 | -------------------------------------------------------------------------------- /logs/syslognamed/parser.go: -------------------------------------------------------------------------------- 1 | package syslognamed 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "net" 8 | "os" 9 | "regexp" 10 | "strconv" 11 | "time" 12 | 13 | "github.com/alphasoc/nfr/client" 14 | "github.com/alphasoc/nfr/packet" 15 | ) 16 | 17 | // A Parser parses and reads network events from syslog-named logs. 18 | type Parser struct { 19 | r io.ReadCloser 20 | } 21 | 22 | // NewParser creates new syslog-named parser. 23 | func NewParser() *Parser { 24 | return &Parser{} 25 | } 26 | 27 | // NewFileParser creates new syslog-named reader from given file. 28 | func NewFileParser(filename string) (*Parser, error) { 29 | f, err := os.Open(filename) 30 | if err != nil { 31 | return nil, err 32 | } 33 | 34 | return &Parser{r: f}, nil 35 | } 36 | 37 | // ReadDNS reads all dns packets from the file. 38 | func (p *Parser) ReadDNS() ([]*packet.DNSPacket, error) { 39 | if p.r == nil { 40 | return nil, fmt.Errorf("syslog-named parser must be created with file reader") 41 | } 42 | 43 | var packets []*packet.DNSPacket 44 | 45 | s := bufio.NewScanner(p.r) 46 | for s.Scan() { 47 | dnspacket, err := p.ParseLineDNS(string(s.Bytes())) 48 | if err != nil { 49 | return nil, err 50 | } 51 | if dnspacket != nil { 52 | packets = append(packets, dnspacket) 53 | } 54 | } 55 | 56 | if err := s.Err(); err != nil { 57 | return nil, err 58 | } 59 | return packets, nil 60 | } 61 | 62 | // ReadIP reads all ip packets from the file. 63 | func (*Parser) ReadIP() ([]*packet.IPPacket, error) { 64 | return nil, nil 65 | } 66 | 67 | var re = regexp.MustCompile(`(\d+).*named\[\d+\]: queries: info: client (.*)#\d+.*query: (.*) IN (.*) \+`) 68 | 69 | // ParseLineDNS parse single log line with dns data. 70 | func (*Parser) ParseLineDNS(line string) (*packet.DNSPacket, error) { 71 | m := re.FindStringSubmatch(line) 72 | if len(m) != 5 { 73 | return nil, nil 74 | } 75 | 76 | srcIP := net.ParseIP(m[2]) 77 | if srcIP == nil { 78 | return nil, fmt.Errorf("syslog-named: invalid ip: %s", line) 79 | } 80 | 81 | sec, err := strconv.ParseInt(m[1], 10, 64) 82 | if err != nil { 83 | return nil, fmt.Errorf("syslog-named: invalid timestamp: %s", line) 84 | } 85 | 86 | return &packet.DNSPacket{ 87 | DstPort: 0, 88 | Protocol: "udp", 89 | Timestamp: time.Unix(sec, 0), 90 | SrcIP: srcIP, 91 | RecordType: m[4], 92 | FQDN: m[3], 93 | }, nil 94 | } 95 | 96 | // ParseLineIP reads all ip packets from the file. 97 | func (*Parser) ParseLineIP(line string) (*packet.IPPacket, error) { 98 | return nil, nil 99 | } 100 | 101 | func (*Parser) ReadHTTP() ([]*client.HTTPEntry, error) { 102 | return nil, nil 103 | } 104 | 105 | func (*Parser) ParseLineHTTP(line string) (*client.HTTPEntry, error) { 106 | return nil, nil 107 | } 108 | 109 | // Close underlying log file. 110 | func (p *Parser) Close() error { 111 | return p.r.Close() 112 | } 113 | -------------------------------------------------------------------------------- /logs/syslognamed/parser_test.go: -------------------------------------------------------------------------------- 1 | package syslognamed 2 | 3 | import ( 4 | "io/ioutil" 5 | "net" 6 | "os" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | func TestReaderReadDNS(t *testing.T) { 12 | const ( 13 | filename = "syslog-named.log" 14 | logcontent = ` 15 | 1483228800 Jan 1 00:00:00 localhost named[100]: queries: info: client 10.0.0.1#10000 (alphasoc.com): query: alphasoc.com IN A +ED (10.0.0.1) 16 | 1483228800 Jan 1 00:00:00 localhost named[100]: queries: info: client 10.0.0.2#10001 (alphasoc.net): query: alphasoc.net IN AAAA +T (10.205.40.193) 17 | ` 18 | ) 19 | 20 | if _, err := NewFileParser("non-existing-log"); err == nil { 21 | t.Fatal("NewFileParser should return error") 22 | } 23 | 24 | if err := ioutil.WriteFile(filename, []byte(logcontent), os.ModePerm); err != nil { 25 | t.Fatalf("write syslog-named log file failed - %s", err) 26 | } 27 | defer os.Remove(filename) 28 | 29 | r, err := NewFileParser(filename) 30 | if err != nil { 31 | t.Fatalf("create syslog-named reader failed - %s", err) 32 | } 33 | defer r.Close() 34 | 35 | packets, err := r.ReadDNS() 36 | if err != nil { 37 | t.Fatalf("reading syslog-named log failed - %s", err) 38 | } 39 | 40 | if len(packets) != 2 { 41 | t.Fatalf("reading syslog-named dns package failed - want: 2, got: %d", len(packets)) 42 | } 43 | 44 | tc := time.Date(2017, 1, 1, 0, 0, 0, 0, time.UTC) 45 | if !(packets[0].DstPort == 0 && 46 | packets[0].Protocol == "udp" && 47 | packets[0].Timestamp.Equal(tc) && 48 | packets[0].SrcIP.Equal(net.IPv4(10, 0, 0, 1)) && 49 | packets[0].RecordType == "A" && 50 | packets[0].FQDN == "alphasoc.com") { 51 | t.Fatalf("invalid 1st packet %+q", packets[0]) 52 | } 53 | 54 | if !(packets[1].DstPort == 0 && 55 | packets[1].Protocol == "udp" && 56 | packets[1].Timestamp.Equal(tc) && 57 | packets[1].SrcIP.Equal(net.IPv4(10, 0, 0, 2)) && 58 | packets[1].RecordType == "AAAA" && 59 | packets[1].FQDN == "alphasoc.net") { 60 | t.Fatalf("invalid 2nd packet %+q", packets[1]) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/alphasoc/nfr/cmd" 8 | ) 9 | 10 | func main() { 11 | if err := cmd.NewRootCommand().Execute(); err != nil { 12 | fmt.Fprintf(os.Stderr, "%s\n", err) 13 | os.Exit(1) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /matchers/domains.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/alphasoc/nfr/utils" 8 | ) 9 | 10 | // Domain matches domain on blacklist. 11 | type Domain struct { 12 | snames map[string]bool // strict domain names 13 | mnames map[string]bool // multimatch domain names 14 | maxLabels int // maxLabels is max count of '.' char in multimatch domains map 15 | } 16 | 17 | // NewDomain creates Domain macher for given domains list. 18 | // It retrus errors if any of domain has invalid format. 19 | // Domains could be strict domain like alphasoc.com or 20 | // multimatch domain with prefix *. like *.alphasoc.com. 21 | func NewDomain(domains []string) (*Domain, error) { 22 | d := &Domain{ 23 | snames: make(map[string]bool), 24 | mnames: make(map[string]bool), 25 | } 26 | return d, d.add(domains) 27 | } 28 | 29 | // add adds domains to Domain matcher. 30 | func (m *Domain) add(domains []string) error { 31 | for _, domain := range domains { 32 | // check if it's valid strict or multimatch domain 33 | if !utils.IsDomainName(domain) && 34 | !utils.IsDomainName(strings.TrimPrefix(domain, "*.")) { 35 | // Do not add invalid domains 36 | return fmt.Errorf("%s is not valid domain name", domain) 37 | } 38 | 39 | // if it's strict domain then put it on list 40 | if !strings.HasPrefix(domain, "*") { 41 | domain = strings.Trim(domain, ".") 42 | m.snames[domain] = true 43 | continue 44 | } 45 | 46 | // otherwise domain must be multimatch domain 47 | domain = strings.TrimPrefix(domain, "*") 48 | domain = strings.Trim(domain, ".") 49 | m.mnames[domain] = true 50 | 51 | if labels := strings.Count(domain, ".") + 1; labels > m.maxLabels { 52 | m.maxLabels = labels 53 | } 54 | } 55 | return nil 56 | } 57 | 58 | // Match matches domain and check if it's on any domain blacklist. 59 | func (m *Domain) Match(domain string) bool { 60 | if domain == "" { 61 | return false 62 | } 63 | 64 | if m.snames[domain] { 65 | return true 66 | } 67 | 68 | if len(m.mnames) == 0 { 69 | return false 70 | } 71 | 72 | // shrink to longest suffix, used to search in map 73 | dot := len(domain) 74 | for n := m.maxLabels; n > 0 && dot > 0; n-- { 75 | dot = strings.LastIndexByte(domain[:dot], '.') 76 | } 77 | domain = domain[dot+1:] 78 | 79 | // find matching suffix 80 | for len(domain) > 0 { 81 | if _, ok := m.mnames[domain]; ok { 82 | return true 83 | } 84 | 85 | // remove first subdomain and check if the 86 | // rest of domain is in map, for example: 87 | // a.b.c becomes b.c and is checked in blacklist map. 88 | dot := strings.IndexByte(domain, '.') 89 | if dot < 0 { 90 | return false 91 | } 92 | domain = domain[dot+1:] 93 | } 94 | 95 | return false 96 | } 97 | -------------------------------------------------------------------------------- /matchers/domains_test.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import "testing" 4 | 5 | func TestDomain(t *testing.T) { 6 | domainsTests := []struct { 7 | name string 8 | domains []string 9 | cases []string 10 | expected []bool 11 | }{ 12 | { 13 | "empty domain", 14 | []string{}, 15 | []string{""}, 16 | []bool{false}, 17 | }, 18 | { 19 | "strict domain", 20 | []string{"a.b"}, 21 | []string{"a.b", "b"}, 22 | []bool{true, false}, 23 | }, 24 | { 25 | "multimatch domains", 26 | []string{"*.a"}, 27 | []string{"a", "b.a", "c.b.a", "a.b"}, 28 | []bool{true, true, true, false}, 29 | }, 30 | { 31 | "strict and multimatch domains", 32 | []string{"*.b.a", "*.a", "c"}, 33 | []string{"c.b.a", "c.a", "b.a", "c", "d.c.d."}, 34 | []bool{true, true, true, true, false}, 35 | }, 36 | } 37 | 38 | for _, tt := range domainsTests { 39 | matcher, err := NewDomain(tt.domains) 40 | if err != nil { 41 | t.Fatalf("%s %s", tt.name, err) 42 | } 43 | for i := range tt.cases { 44 | if matcher.Match(tt.cases[i]) != tt.expected[i] { 45 | t.Fatalf("test %s - match(%s) = %t; want %t", tt.name, tt.cases[i], !tt.expected[i], tt.expected[i]) 46 | } 47 | 48 | } 49 | } 50 | } 51 | 52 | func TestInvalidDomain(t *testing.T) { 53 | if _, err := NewDomain([]string{"$invalid_domain$"}); err == nil { 54 | t.Fatalf("got %s; expected ", err) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /matchers/networks.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | ) 7 | 8 | // Network matches network based on src and dst IP. 9 | // Checks if the ip is included and not excluded from list at the same time. 10 | type Network struct { 11 | srcIncludes []*net.IPNet 12 | srcExcludes []*net.IPNet 13 | srcExcludesIps map[string]bool // compare ip as string 14 | 15 | dstIncludes []*net.IPNet 16 | dstExcludes []*net.IPNet 17 | dstExcludesIps map[string]bool // compare ip as string 18 | } 19 | 20 | // NewNetwork creates Network matcher for given includes and excludes netowrks. 21 | // The excludes network acceptable format is cidr and ip. 22 | func NewNetwork(srcIncludes, srcExcludes, dstIncludes, dstExcludes []string) (*Network, error) { 23 | if len(srcIncludes) == 0 { 24 | srcIncludes = []string{"0.0.0.0/0", "::/0"} 25 | } 26 | if len(dstIncludes) == 0 { 27 | dstIncludes = []string{"0.0.0.0/0", "::/0"} 28 | } 29 | 30 | m := &Network{ 31 | srcExcludesIps: make(map[string]bool), 32 | dstExcludesIps: make(map[string]bool), 33 | } 34 | 35 | for i := range srcIncludes { 36 | _, ipnet, err := net.ParseCIDR(srcIncludes[i]) 37 | if err != nil { 38 | return nil, err 39 | } 40 | m.srcIncludes = append(m.srcIncludes, ipnet) 41 | } 42 | 43 | for i := range dstIncludes { 44 | _, ipnet, err := net.ParseCIDR(dstIncludes[i]) 45 | if err != nil { 46 | return nil, err 47 | } 48 | m.dstIncludes = append(m.dstIncludes, ipnet) 49 | } 50 | 51 | for i := range srcExcludes { 52 | ip := net.ParseIP(srcExcludes[i]) 53 | _, ipnet, err := net.ParseCIDR(srcExcludes[i]) 54 | if ip == nil && err != nil { 55 | return nil, fmt.Errorf("%s is not cidr or ip", srcExcludes[i]) 56 | } 57 | 58 | if ip != nil { 59 | m.srcExcludesIps[srcExcludes[i]] = true 60 | } else { 61 | if isIpnetIP(ipnet) { 62 | m.srcExcludesIps[ipnet.IP.String()] = true 63 | continue 64 | } 65 | m.srcExcludes = append(m.srcExcludes, ipnet) 66 | } 67 | } 68 | 69 | for i := range dstExcludes { 70 | ip := net.ParseIP(dstExcludes[i]) 71 | _, ipnet, err := net.ParseCIDR(dstExcludes[i]) 72 | if ip == nil && err != nil { 73 | return nil, fmt.Errorf("%s is not cidr or ip", dstExcludes[i]) 74 | } 75 | 76 | if ip != nil { 77 | m.dstExcludesIps[dstExcludes[i]] = true 78 | } else { 79 | if isIpnetIP(ipnet) { 80 | m.dstExcludesIps[ipnet.IP.String()] = true 81 | continue 82 | } 83 | m.dstExcludes = append(m.dstExcludes, ipnet) 84 | } 85 | } 86 | 87 | return m, nil 88 | } 89 | 90 | // MatchSrcIP matches src ip to check if it's on networks included list 91 | // at the same time checking if it's not in any excluded list. 92 | func (m *Network) MatchSrcIP(srcIP net.IP) (bool, bool) { 93 | if srcIP == nil { 94 | return false, false 95 | } 96 | 97 | // check if the ip is included in source networks 98 | ok := false 99 | for _, n := range m.srcIncludes { 100 | if n.Contains(srcIP) { 101 | ok = true 102 | break 103 | } 104 | } 105 | 106 | // if the ip is not in any networks, it means that ip is not matched 107 | if !ok { 108 | return false, false 109 | } 110 | 111 | // check ip exclusion 112 | if m.srcExcludesIps[srcIP.String()] { 113 | return true, true 114 | } 115 | 116 | // if the ip is within any excluded source network 117 | for _, n := range m.srcExcludes { 118 | if n.Contains(srcIP) { 119 | return true, true 120 | } 121 | } 122 | 123 | return true, false 124 | } 125 | 126 | // MatchDstIP matches dest ips and check if it's on networks included list 127 | // at the same time checking if it's not in any excluded list. 128 | func (m *Network) MatchDstIP(dstIP net.IP) (bool, bool) { 129 | if dstIP == nil { 130 | return false, false 131 | } 132 | 133 | // check if the ip is included in destination networks 134 | ok := false 135 | for _, n := range m.dstIncludes { 136 | if n.Contains(dstIP) { 137 | ok = true 138 | break 139 | } 140 | } 141 | 142 | if !ok { 143 | return false, false 144 | } 145 | 146 | // check ip exclusion 147 | if m.dstExcludesIps[dstIP.String()] { 148 | return true, true 149 | } 150 | 151 | // if the ip is within any excluded destination network 152 | for _, n := range m.dstExcludes { 153 | if n.Contains(dstIP) { 154 | return true, true 155 | } 156 | } 157 | 158 | return true, false 159 | } 160 | 161 | // Match matches src and dest ips and check if it's on networks included list 162 | // at the same time checking if it's not in any excluded list. 163 | func (m *Network) Match(srcIP, dstIP net.IP) (bool, bool) { 164 | matched, excluded := m.MatchSrcIP(srcIP) 165 | if !matched || (matched && excluded) { 166 | return matched, excluded 167 | } 168 | 169 | return m.MatchDstIP(dstIP) 170 | } 171 | 172 | func isIpnetIP(ipnet *net.IPNet) bool { 173 | once, _ := ipnet.Mask.Size() 174 | return (ipnet.IP.To4() != nil && once == 32) || once == 128 175 | } 176 | -------------------------------------------------------------------------------- /matchers/networks_test.go: -------------------------------------------------------------------------------- 1 | package matchers 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | ) 7 | 8 | func TestNetwork(t *testing.T) { 9 | networksTests := []struct { 10 | name string 11 | srcincludes []string 12 | srcexcludes []string 13 | dstincludes []string 14 | dstexcludes []string 15 | srcIP []net.IP 16 | dstIP []net.IP 17 | expected []bool // two bool for one test 18 | }{ 19 | { 20 | "no network", 21 | nil, 22 | nil, 23 | nil, 24 | nil, 25 | []net.IP{net.IPv4(10, 0, 0, 0)}, 26 | []net.IP{net.IPv4(0, 0, 0, 0)}, 27 | []bool{true, false, false, false}, 28 | }, 29 | { 30 | "multiple source network", 31 | []string{"10.0.0.0/16", "10.1.0.0/16"}, 32 | nil, 33 | nil, 34 | nil, 35 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(10, 1, 0, 0), net.IPv4(10, 2, 0, 0)}, 36 | []net.IP{net.IPv4(0, 0, 0, 0), net.IPv4(0, 0, 0, 0), net.IPv4(0, 0, 0, 0)}, 37 | []bool{true, false, true, false, false, false}, 38 | }, 39 | { 40 | "exclude source network", 41 | []string{"0.0.0.0/0"}, 42 | []string{"10.0.0.0/16"}, 43 | nil, 44 | nil, 45 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(10, 1, 0, 0)}, 46 | []net.IP{net.IPv4(0, 0, 0, 0), net.IPv4(0, 0, 0, 0)}, 47 | []bool{true, true, true, false}, 48 | }, 49 | { 50 | "many different source networks", 51 | []string{"10.0.0.0/16", "10.1.0.0/16"}, 52 | []string{"10.0.0.0/24", "10.1.0.0"}, 53 | nil, 54 | nil, 55 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(10, 1, 0, 0), net.IPv4(10, 0, 1, 0), net.IPv4(10, 1, 1, 0)}, 56 | []net.IP{net.IPv4(0, 0, 0, 0), net.IPv4(0, 0, 0, 0), net.IPv4(0, 0, 0, 0), net.IPv4(0, 0, 0, 0)}, 57 | []bool{true, true, true, true, true, false, true, false}, 58 | }, 59 | { 60 | "max mask size source networks", 61 | []string{"10.0.0.0/32", "10.0.0.1/32"}, 62 | []string{"10.0.0.1/32", "::1/128"}, 63 | nil, 64 | nil, 65 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(10, 0, 0, 1)}, 66 | []net.IP{net.IPv4(0, 0, 0, 0), net.IPv4(0, 0, 0, 0)}, 67 | []bool{true, false, true, true}, 68 | }, 69 | 70 | { 71 | "multiple destination network", 72 | nil, 73 | nil, 74 | []string{"10.0.0.0/16", "10.1.0.0/16"}, 75 | nil, 76 | []net.IP{net.IPv4(0, 0, 0, 0), net.IPv4(0, 0, 0, 0), net.IPv4(0, 0, 0, 0)}, 77 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(10, 1, 0, 0), net.IPv4(10, 2, 0, 0)}, 78 | []bool{true, false, true, false, false, false}, 79 | }, 80 | { 81 | "exclude destination network", 82 | nil, 83 | nil, 84 | []string{"0.0.0.0/0"}, 85 | []string{"10.0.0.0/16"}, 86 | []net.IP{net.IPv4(0, 0, 0, 0), net.IPv4(0, 0, 0, 0)}, 87 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(10, 1, 0, 0)}, 88 | []bool{true, true, true, false}, 89 | }, 90 | { 91 | "many different destination networks", 92 | nil, 93 | nil, 94 | []string{"10.0.0.0/16", "10.1.0.0/16"}, 95 | []string{"10.0.0.0/24", "10.1.0.0"}, 96 | []net.IP{net.IPv4(0, 0, 0, 0), net.IPv4(0, 0, 0, 0), net.IPv4(0, 0, 0, 0), net.IPv4(0, 0, 0, 0)}, 97 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(10, 1, 0, 0), net.IPv4(10, 0, 1, 0), net.IPv4(10, 1, 1, 0)}, 98 | []bool{true, true, true, true, true, false, true, false}, 99 | }, 100 | { 101 | "max mask size destination networks", 102 | nil, 103 | nil, 104 | []string{"10.0.0.0/32", "10.0.0.1/32"}, 105 | []string{"10.0.0.1/32", "::1/128"}, 106 | []net.IP{net.IPv4(0, 0, 0, 0), net.IPv4(0, 0, 0, 0)}, 107 | []net.IP{net.IPv4(10, 0, 0, 0), net.IPv4(10, 0, 0, 1)}, 108 | []bool{true, false, true, true}, 109 | }, 110 | } 111 | 112 | for _, tt := range networksTests { 113 | matcher, err := NewNetwork(tt.srcincludes, tt.srcexcludes, tt.dstincludes, tt.dstexcludes) 114 | if err != nil { 115 | t.Fatalf("%s %s", tt.name, err) 116 | } 117 | 118 | for i := range tt.srcIP { 119 | included, excluded := matcher.Match(tt.srcIP[i], tt.dstIP[i]) 120 | if included != tt.expected[i*2] || excluded != tt.expected[i*2+1] { 121 | t.Fatalf("test %s - match(%s, %s) = %t, %t; expected %t, %t", 122 | tt.name, tt.srcIP[i], tt.dstIP[i], included, excluded, 123 | tt.expected[i*2], tt.expected[i*2+1]) 124 | } 125 | } 126 | } 127 | } 128 | 129 | func TestInvalidNetwork(t *testing.T) { 130 | if _, err := NewNetwork([]string{"10.0.0.0"}, nil, nil, nil); err == nil { 131 | t.Fatalf("got %s; expected ", err) 132 | } 133 | if _, err := NewNetwork([]string{"10.0.0.0/8"}, []string{"bad_ip_address"}, nil, nil); err == nil { 134 | t.Fatalf("got %s; expected ", err) 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /nfr.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=AlphaSOC Network Flight Recorder 3 | After=network.target 4 | 5 | [Service] 6 | Type=simple 7 | ExecStart=/usr/bin/nfr start 8 | Restart=always 9 | 10 | [Install] 11 | WantedBy=multi-user.target 12 | -------------------------------------------------------------------------------- /packet/dns_packet_buffer.go: -------------------------------------------------------------------------------- 1 | package packet 2 | 3 | // A DNSPacketBuffer holds slice of packets. 4 | type DNSPacketBuffer struct { 5 | packets []*DNSPacket 6 | } 7 | 8 | // NewDNSPacketBuffer initializes a new DNSPacketBuffer. 9 | func NewDNSPacketBuffer() *DNSPacketBuffer { 10 | return &DNSPacketBuffer{} 11 | } 12 | 13 | // Writes single dns packet to the buffer. 14 | // Returns number of packets added to the buffer and length of the buffer. 15 | func (b *DNSPacketBuffer) Write(packets ...*DNSPacket) { 16 | // do not write packets that was duplicated recentrly 17 | // checks 8 packets back. 18 | l := b.Len() 19 | pos := l - 8 20 | if pos < 0 { 21 | pos = 0 22 | } 23 | 24 | packetLoop: 25 | for i := range packets { 26 | for j := pos; j < l; j++ { 27 | if b.packets[j].Equal(packets[i]) { 28 | continue packetLoop 29 | } 30 | } 31 | b.packets = append(b.packets, packets[i]) 32 | pos++ 33 | } 34 | } 35 | 36 | // Packets returns slice of packets and reset the buffer. 37 | func (b *DNSPacketBuffer) Packets() []*DNSPacket { 38 | packets := make([]*DNSPacket, len(b.packets)) 39 | copy(packets, b.packets) 40 | b.packets = b.packets[:0] 41 | return packets 42 | } 43 | 44 | // Len returns the number of packets in the buffer. 45 | func (b *DNSPacketBuffer) Len() int { 46 | return len(b.packets) 47 | } 48 | -------------------------------------------------------------------------------- /packet/http_packet_buffer.go: -------------------------------------------------------------------------------- 1 | package packet 2 | 3 | import ( 4 | "github.com/alphasoc/nfr/client" 5 | ) 6 | 7 | // A HTTPPacketBuffer holds slice of packets. 8 | type HTTPPacketBuffer struct { 9 | packets []*client.HTTPEntry 10 | } 11 | 12 | // NewHTTPPacketBuffer initializes a new HTTPPacketBuffer. 13 | func NewHTTPPacketBuffer() *HTTPPacketBuffer { 14 | return &HTTPPacketBuffer{} 15 | } 16 | 17 | // Writes HTTP packets to the buffer. 18 | func (b *HTTPPacketBuffer) Write(packets ...*client.HTTPEntry) { 19 | b.packets = append(b.packets, packets...) 20 | } 21 | 22 | // Packets returns slice of packets and reset the buffer. 23 | func (b *HTTPPacketBuffer) Packets() []*client.HTTPEntry { 24 | packets := make([]*client.HTTPEntry, len(b.packets)) 25 | copy(packets, b.packets) 26 | b.packets = b.packets[:0] 27 | return packets 28 | } 29 | 30 | // Len returns the number of packets in the buffer. 31 | func (b *HTTPPacketBuffer) Len() int { 32 | return len(b.packets) 33 | } 34 | -------------------------------------------------------------------------------- /packet/ip_packet_buffer.go: -------------------------------------------------------------------------------- 1 | package packet 2 | 3 | // A IPPacketBuffer holds slice of packets. 4 | type IPPacketBuffer struct { 5 | packets []*IPPacket 6 | } 7 | 8 | // NewIPPacketBuffer initializes a new IPPacketBuffer. 9 | func NewIPPacketBuffer() *IPPacketBuffer { 10 | return &IPPacketBuffer{packets: make([]*IPPacket, 0, 1024)} 11 | } 12 | 13 | // Writes single ip packet to the buffer. 14 | func (b *IPPacketBuffer) Write(packets ...*IPPacket) { 15 | b.packets = append(b.packets, packets...) 16 | } 17 | 18 | // Packets returns slice of packets and reset the buffer. 19 | func (b *IPPacketBuffer) Packets() []*IPPacket { 20 | packets := make([]*IPPacket, len(b.packets)) 21 | copy(packets, b.packets) 22 | b.packets = b.packets[:0] 23 | return packets 24 | } 25 | 26 | // Len returns the number of packets in the buffer. 27 | func (b *IPPacketBuffer) Len() int { 28 | return len(b.packets) 29 | } 30 | 31 | // reset resets the buffer to be empty. 32 | func (b *IPPacketBuffer) reset() { 33 | b.packets = b.packets[:0] 34 | } 35 | -------------------------------------------------------------------------------- /packet/packet.go: -------------------------------------------------------------------------------- 1 | package packet 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net" 7 | "time" 8 | 9 | "github.com/google/gopacket" 10 | "github.com/google/gopacket/layers" 11 | ) 12 | 13 | // RawPacket is an interface that wraps method for raw packet 14 | type RawPacket interface { 15 | Raw() gopacket.Packet 16 | } 17 | 18 | // Direction of the packet. 19 | type Direction int 20 | 21 | // List of all packet directions 22 | const ( 23 | DirectionUnknown Direction = 0 24 | DirectionIn Direction = 1 25 | DirectionOut Direction = 2 26 | ) 27 | 28 | // IPPacket represents single dns query that could be 29 | // converted to feed AlphaSOC Engine. 30 | type IPPacket struct { 31 | raw gopacket.Packet 32 | srcMAC net.HardwareAddr 33 | 34 | Timestamp time.Time 35 | Protocol string 36 | SrcIP net.IP 37 | SrcPort int 38 | DstIP net.IP 39 | DstPort int 40 | BytesCount int 41 | Direction Direction 42 | Ja3 string 43 | } 44 | 45 | // NewIPPacket creates IPPacket from raw packet. 46 | func NewIPPacket(raw gopacket.Packet) *IPPacket { 47 | linkLayer := raw.LinkLayer() 48 | networkLayer := raw.NetworkLayer() 49 | transportLayer := raw.TransportLayer() 50 | metadata := raw.Metadata() 51 | 52 | if linkLayer == nil || networkLayer == nil || 53 | transportLayer == nil || metadata == nil { 54 | return nil 55 | } 56 | 57 | ethernet, ok := linkLayer.(gopacket.Layer).(*layers.Ethernet) 58 | if !ok { 59 | return nil 60 | } 61 | 62 | var ippacket = &IPPacket{ 63 | raw: raw, 64 | srcMAC: ethernet.SrcMAC, 65 | Timestamp: metadata.Timestamp, 66 | BytesCount: len(raw.Data()), 67 | // Ja3: ja3.Convert(raw), 68 | } 69 | if lipv4, ok := networkLayer.(gopacket.Layer).(*layers.IPv4); ok { 70 | ippacket.SrcIP = lipv4.SrcIP 71 | ippacket.DstIP = lipv4.DstIP 72 | } else if lipv6, ok := networkLayer.(gopacket.Layer).(*layers.IPv6); ok { 73 | ippacket.SrcIP = lipv6.SrcIP 74 | ippacket.DstIP = lipv6.DstIP 75 | } else { 76 | return nil 77 | } 78 | 79 | if tcp, ok := transportLayer.(gopacket.Layer).(*layers.TCP); ok { 80 | ippacket.SrcPort = int(tcp.SrcPort) 81 | ippacket.DstPort = int(tcp.DstPort) 82 | ippacket.Protocol = "tcp" 83 | } else if udp, ok := transportLayer.(gopacket.Layer).(*layers.UDP); ok { 84 | ippacket.SrcPort = int(udp.SrcPort) 85 | ippacket.DstPort = int(udp.DstPort) 86 | ippacket.Protocol = "udp" 87 | } else { 88 | return nil 89 | } 90 | 91 | return ippacket 92 | } 93 | 94 | // Raw returns raw packet. 95 | func (p *IPPacket) Raw() gopacket.Packet { 96 | return p.raw 97 | } 98 | 99 | // DetermineDirection determines packet direciton based on interface mac address. 100 | func (p *IPPacket) DetermineDirection(ifaceMac net.HardwareAddr) { 101 | if bytes.Equal(p.srcMAC, ifaceMac) { 102 | p.Direction = DirectionOut 103 | } else { 104 | p.Direction = DirectionIn 105 | } 106 | } 107 | 108 | // DNSPacket represents single dns query that could be 109 | // converted to feed AlphaSOC Engine. 110 | type DNSPacket struct { 111 | raw gopacket.Packet 112 | 113 | Timestamp time.Time 114 | Protocol string 115 | SrcPort int 116 | DstPort int 117 | SrcIP net.IP 118 | DstIP net.IP 119 | FQDN string 120 | RecordType string 121 | } 122 | 123 | // NewDNSPacket creates new dns packet from raw packet. 124 | func NewDNSPacket(raw gopacket.Packet) *DNSPacket { 125 | var ( 126 | metadata = raw.Metadata() 127 | networkLayer = raw.NetworkLayer() 128 | transportLayer = raw.TransportLayer() 129 | applicationLayer = raw.ApplicationLayer() 130 | ) 131 | 132 | if metadata == nil || networkLayer == nil || transportLayer == nil || applicationLayer == nil { 133 | return nil 134 | } 135 | 136 | dns, ok := applicationLayer.(gopacket.Layer).(*layers.DNS) 137 | if !ok || dns.QR || len(dns.Questions) == 0 { 138 | return nil 139 | } 140 | 141 | var dnspacket = &DNSPacket{ 142 | raw: raw, 143 | Timestamp: metadata.Timestamp, 144 | RecordType: dns.Questions[0].Type.String(), 145 | FQDN: string(dns.Questions[0].Name), 146 | } 147 | 148 | if lipv4, ok := networkLayer.(gopacket.Layer).(*layers.IPv4); ok { 149 | dnspacket.SrcIP = lipv4.SrcIP 150 | dnspacket.DstIP = lipv4.DstIP 151 | } else if lipv6, ok := networkLayer.(gopacket.Layer).(*layers.IPv6); ok { 152 | dnspacket.SrcIP = lipv6.SrcIP 153 | dnspacket.DstIP = lipv6.DstIP 154 | } else { 155 | return nil 156 | } 157 | 158 | if tcp, ok := transportLayer.(gopacket.Layer).(*layers.TCP); ok { 159 | dnspacket.SrcPort = int(tcp.SrcPort) 160 | dnspacket.DstPort = int(tcp.DstPort) 161 | dnspacket.Protocol = "tcp" 162 | } else if udp, ok := transportLayer.(gopacket.Layer).(*layers.UDP); ok { 163 | dnspacket.SrcPort = int(udp.SrcPort) 164 | dnspacket.DstPort = int(udp.DstPort) 165 | dnspacket.Protocol = "udp" 166 | } else { 167 | return nil 168 | } 169 | 170 | return dnspacket 171 | } 172 | 173 | func (p *DNSPacket) String() string { 174 | return fmt.Sprintf("%s %s from %s to %s", p.FQDN, p.RecordType, p.SrcIP, p.DstIP) 175 | } 176 | 177 | // Equal checks if two packets are equal. 178 | func (p *DNSPacket) Equal(p1 *DNSPacket) bool { 179 | if p == nil || p1 == nil { 180 | return false 181 | } 182 | return p.SrcIP.Equal(p1.SrcIP) && 183 | p.DstIP.Equal(p1.DstIP) && 184 | p.RecordType == p1.RecordType && 185 | p.FQDN == p1.FQDN 186 | } 187 | 188 | // Raw returns raw packet. 189 | func (p *DNSPacket) Raw() gopacket.Packet { 190 | return p.raw 191 | } 192 | -------------------------------------------------------------------------------- /packet/packet_buffer_test.go: -------------------------------------------------------------------------------- 1 | package packet 2 | 3 | import ( 4 | "net" 5 | 6 | "testing" 7 | ) 8 | 9 | func TestDNSBufferWrite(t *testing.T) { 10 | b := NewDNSPacketBuffer() 11 | 12 | p1 := &DNSPacket{SrcIP: net.IP{1, 1, 1, 1}} 13 | p2 := &DNSPacket{SrcIP: net.IP{2, 2, 2, 2}} 14 | p3 := &DNSPacket{SrcIP: net.IP{3, 3, 3, 3}} 15 | 16 | b.Write(p1) 17 | b.Write(p2) 18 | b.Write(p1) // should be deduplicated 19 | 20 | packets := b.Packets() 21 | if len(packets) != 2 { 22 | t.Fatalf("invalid packet length: %d", len(packets)) 23 | } 24 | 25 | if packets[0] != p1 { 26 | t.Fatalf("invalid packet at 0: %v", packets[0]) 27 | } 28 | if packets[1] != p2 { 29 | t.Fatalf("invalid packet at 1: %v", packets[1]) 30 | } 31 | 32 | if l := b.Len(); l != 0 { 33 | t.Fatalf("invalid buffer length - got %d; expected %d", l, 0) 34 | } 35 | 36 | // write to buffer after reset, make sure previous packets 37 | // are not overwritten. 38 | b.Write(p3) 39 | if packets[0] != p1 { 40 | t.Fatalf("invalid packet at 0 after reset: %v", packets[0]) 41 | } 42 | } 43 | 44 | func TestIPBufferWrite(t *testing.T) { 45 | b := NewIPPacketBuffer() 46 | 47 | p1 := &IPPacket{SrcIP: net.IP{1, 1, 1, 1}} 48 | p2 := &IPPacket{SrcIP: net.IP{2, 2, 2, 2}} 49 | p3 := &IPPacket{SrcIP: net.IP{3, 3, 3, 3}} 50 | 51 | b.Write(p1) 52 | b.Write(p2) 53 | b.Write(p1) // not deduplicated for IPs 54 | 55 | packets := b.Packets() 56 | if len(packets) != 3 { 57 | t.Fatalf("invalid packet length: %d", len(packets)) 58 | } 59 | 60 | if packets[0] != p1 { 61 | t.Fatalf("invalid packet at 0: %v", packets[0]) 62 | } 63 | if packets[1] != p2 { 64 | t.Fatalf("invalid packet at 1: %v", packets[1]) 65 | } 66 | if packets[2] != p1 { 67 | t.Fatalf("invalid packet at 1: %v", packets[2]) 68 | } 69 | 70 | if l := b.Len(); l != 0 { 71 | t.Fatalf("invalid buffer length - got %d; expected %d", l, 0) 72 | } 73 | 74 | // write to buffer after reset, make sure previous packets 75 | // are not overwritten. 76 | b.Write(p3) 77 | if packets[0] != p1 { 78 | t.Fatalf("invalid packet at 0 after reset: %v", packets[0]) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /packet/packet_test.go: -------------------------------------------------------------------------------- 1 | package packet 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | 7 | "github.com/google/gopacket" 8 | "github.com/google/gopacket/layers" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | // testPacketPacket0 is the packet: 13 | // 13:35:56.456790 IP 10.0.2.15.13705 > 8.8.8.8.53: 59721+ [1au] A? api.alphasoc.net. (45) 14 | // 0x0000: 5254 0012 3502 0800 27b1 891d 0800 4500 RT..5...'.....E. 15 | // 0x0010: 0049 734b 4000 4011 ab3a 0a00 020f 0808 .IsK@.@..:...... 16 | // 0x0020: 0808 3589 0035 0035 1c65 e949 0120 0001 ..5..5.5.e.I.... 17 | // 0x0030: 0000 0000 0001 0361 7069 0861 6c70 6861 .......api.alpha 18 | // 0x0040: 736f 6303 6e65 7400 0001 0001 0000 2910 soc.net.......). 19 | // 0x0050: 0000 0000 0000 00 ....... 20 | var testPacketDNSQuery = []byte{ 21 | 0x52, 0x54, 0x00, 0x12, 0x35, 0x02, 0x08, 0x00, 0x27, 0xb1, 0x89, 0x1d, 0x08, 0x00, 0x45, 0x00, 22 | 0x00, 0x49, 0x73, 0x4b, 0x40, 0x00, 0x40, 0x11, 0xab, 0x3a, 0x0a, 0x00, 0x02, 0x0f, 0x08, 0x08, 23 | 0x08, 0x08, 0x35, 0x89, 0x00, 0x35, 0x00, 0x35, 0x1c, 0x65, 0xe9, 0x49, 0x01, 0x20, 0x00, 0x01, 24 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x03, 0x61, 0x70, 0x69, 0x08, 0x61, 0x6c, 0x70, 0x68, 0x61, 25 | 0x73, 0x6f, 0x63, 0x03, 0x6e, 0x65, 0x74, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00, 0x29, 0x10, 26 | 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 27 | } 28 | 29 | const testPacketDNSQueryLenght = 87 30 | 31 | func TestNewDNSPacket(t *testing.T) { 32 | rawPacket := gopacket.NewPacket(testPacketDNSQuery, layers.LinkTypeEthernet, gopacket.Default) 33 | packet := NewDNSPacket(rawPacket) 34 | require.NotNil(t, packet) 35 | require.Equal(t, "api.alphasoc.net", packet.FQDN, "invalid fqdn") 36 | require.Equal(t, "A", packet.RecordType, "invalid record type") 37 | require.True(t, net.IPv4(10, 0, 2, 15).Equal(packet.SrcIP), "invalid soruce ip") 38 | } 39 | 40 | func TestDNSPacketEqual(t *testing.T) { 41 | packet := NewDNSPacket(gopacket.NewPacket(testPacketDNSQuery, layers.LinkTypeEthernet, gopacket.Default)) 42 | require.False(t, packet.Equal(nil), "equal with nil must return false") 43 | require.True(t, packet.Equal(packet), "not equal with itself") 44 | } 45 | 46 | func TestNewIPPacket(t *testing.T) { 47 | rawPacket := gopacket.NewPacket(testPacketDNSQuery, layers.LinkTypeEthernet, gopacket.Default) 48 | packet := NewIPPacket(rawPacket) 49 | require.NotNil(t, packet) 50 | 51 | packet.DetermineDirection(net.HardwareAddr{0x8, 0x0, 0x27, 0xb1, 0x89, 0x1d}) 52 | require.Equal(t, "udp", packet.Protocol) 53 | require.True(t, packet.SrcIP.Equal(net.IPv4(10, 0, 2, 15))) 54 | require.Equal(t, 13705, packet.SrcPort) 55 | require.True(t, packet.DstIP.Equal(net.IPv4(8, 8, 8, 8))) 56 | require.Equal(t, 53, packet.DstPort) 57 | require.Equal(t, DirectionOut, packet.Direction) 58 | } 59 | -------------------------------------------------------------------------------- /packet/packet_writer.go: -------------------------------------------------------------------------------- 1 | package packet 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/google/gopacket/layers" 7 | "github.com/google/gopacket/pcapgo" 8 | ) 9 | 10 | // Writer writes packetc in PCAP fromat. 11 | type Writer struct { 12 | w *pcapgo.Writer 13 | f *os.File 14 | } 15 | 16 | // NewWriter creates a new Writer for dns packets. 17 | func NewWriter(file string) (*Writer, error) { 18 | stat, err := os.Stat(file) 19 | return newWriter(file, os.IsNotExist(err) || stat.Size() == 0) 20 | } 21 | 22 | func newWriter(file string, writeHeader bool) (*Writer, error) { 23 | f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | w := pcapgo.NewWriter(f) 29 | if writeHeader { 30 | // set file header only for new files 31 | if err = w.WriteFileHeader(65536, layers.LinkTypeEthernet); err != nil { 32 | return nil, err 33 | } 34 | } 35 | return &Writer{w, f}, nil 36 | } 37 | 38 | // Write writes slice of packets. 39 | func (w *Writer) Write(packet RawPacket) error { 40 | if w == nil { 41 | return nil 42 | } 43 | 44 | return w.w.WritePacket(packet.Raw().Metadata().CaptureInfo, packet.Raw().Data()) 45 | } 46 | 47 | // Close closes the file for saving packets. 48 | func (w *Writer) Close() error { 49 | return w.f.Close() 50 | } 51 | -------------------------------------------------------------------------------- /packet/packet_writer_test.go: -------------------------------------------------------------------------------- 1 | package packet 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "net" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | 11 | "github.com/google/gopacket" 12 | "github.com/google/gopacket/layers" 13 | "github.com/google/gopacket/pcapgo" 14 | "github.com/stretchr/testify/require" 15 | ) 16 | 17 | func TestWriteHeader(t *testing.T) { 18 | dir, err := ioutil.TempDir("", "packet_writer") 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | defer os.RemoveAll(dir) 23 | 24 | name := filepath.Join(dir, "pcap.out") 25 | 26 | // first time write header to file 27 | w, err := NewWriter(name) 28 | if err != nil { 29 | t.Fatal(err) 30 | } 31 | w.Close() 32 | 33 | // check if pcap reader accept file header 34 | closer, _ := newPcapReader(t, name) 35 | closer() 36 | 37 | // reopend file should have the same 38 | // content length and the same header 39 | b1, err := ioutil.ReadFile(name) 40 | if err != nil { 41 | t.Fatal(err) 42 | } 43 | 44 | w1, err := NewWriter(name) 45 | if err != nil { 46 | t.Fatal(err) 47 | } 48 | w1.Close() 49 | 50 | b2, err := ioutil.ReadFile(name) 51 | if err != nil { 52 | t.Fatal(err) 53 | } 54 | 55 | if len(b1) == 0 || !bytes.Equal(b1, b2) { 56 | t.Fatal("file headers different between writes", "\n", b1, "\n", b2) 57 | } 58 | } 59 | 60 | func TestWrite(t *testing.T) { 61 | dir, err := ioutil.TempDir("", "packet_writer") 62 | if err != nil { 63 | t.Fatal(err) 64 | } 65 | defer os.RemoveAll(dir) 66 | 67 | name := filepath.Join(dir, "out") 68 | w, err := NewWriter(name) 69 | if err != nil { 70 | t.Fatal(err) 71 | } 72 | defer w.Close() 73 | 74 | // write one packet to file 75 | rawPacket := gopacket.NewPacket(testPacketDNSQuery, layers.LinkTypeEthernet, gopacket.Default) 76 | // set proper packet length 77 | md := rawPacket.Metadata() 78 | md.CaptureLength, md.Length = testPacketDNSQueryLenght, testPacketDNSQueryLenght 79 | if err = w.Write(NewDNSPacket(rawPacket)); err != nil { 80 | t.Fatal(err) 81 | } 82 | 83 | // read packet and check if has proper dns fileds 84 | closer, r := newPcapReader(t, name) 85 | defer closer() 86 | 87 | data, _, err := r.ReadPacketData() 88 | if err != nil { 89 | t.Fatal(err) 90 | } 91 | 92 | rawPacket2 := gopacket.NewPacket(data, layers.LinkTypeEthernet, gopacket.Default) 93 | packet := NewDNSPacket(rawPacket2) 94 | 95 | require.NotNil(t, packet) 96 | require.Equal(t, "api.alphasoc.net", packet.FQDN, "invalid fqdn") 97 | require.Equal(t, "A", packet.RecordType, "invalid record type") 98 | require.True(t, net.IPv4(10, 0, 2, 15).Equal(packet.SrcIP), "invalid soruce ip") 99 | } 100 | 101 | func newPcapReader(t *testing.T, name string) (func() error, *pcapgo.Reader) { 102 | f, err := os.Open(name) 103 | if err != nil { 104 | t.Fatal(err) 105 | } 106 | 107 | r, err := pcapgo.NewReader(f) 108 | if err != nil { 109 | f.Close() 110 | t.Fatal(err) 111 | } 112 | return f.Close, r 113 | } 114 | -------------------------------------------------------------------------------- /scope.yml: -------------------------------------------------------------------------------- 1 | # 2 | # This is the Network Flight Recorder (NFR) monitoring scope file. Here you create 3 | # arbitrary groups of systems to monitor. For each group you should then define: 4 | # 5 | # - Source IP address ranges to monitor events from 6 | # - Source IP address ranges to exclude from scoring (if any) 7 | # - Trusted destination domains to whitelist (e.g. internal domains) 8 | # - Trusted destination IP ranges to whitelist (e.g. known good destinations) 9 | # 10 | # Only network events from the source addresses within scope to non-whitelisted 11 | # destinations are sent to the AlphaSOC Analytics Engine for scoring. 12 | # 13 | # Please contact support@alphasoc.com if you have any questions 14 | # 15 | 16 | groups: 17 | 18 | # Short arbitrary label for the group (e.g. default:, pci_zone:, guest_wifi:) 19 | default: 20 | 21 | # Long label describing the group, for use in alerts and UI elements 22 | label: "Default" 23 | 24 | # Source IP ranges of networks in scope, in CIDR slash-notation format 25 | # We default to internal private ranges here, but you can use 0.0.0.0/0 to 26 | # monitor all of the things. 27 | in_scope: 28 | - 10.0.0.0/8 29 | - 192.168.0.0/16 30 | - 172.16.0.0/12 31 | - fc00::/7 32 | 33 | # Source IP ranges of systems to exclude from monitoring in CIDR 34 | # slash-notation format (i.e. use "- 1.2.3.4/32" to suppress a single system), 35 | # written as a list in the same format as the in_scope section above. 36 | # This exclusion list is intended to suppress scoring of events from 37 | # recursive resolvers, mail servers, proxies, and other systems that 38 | # generating noise or repeating queries that are being captured elsewhere. 39 | out_scope: 40 | 41 | # Trusted destination domains which are whitelisted and ignored by the 42 | # DNS analytics module. Use this list to whitelist your internal domains, etc. 43 | trusted_domains: 44 | - "*.arpa" 45 | - "*.lan" 46 | - "*.local" 47 | - "*.internal" 48 | 49 | # Trusted destination IP ranges (in CIDR slash-notation format) which are 50 | # whitelisted and ignored by the IP analytics module. By default these are 51 | # private networks, so that we process Internet-bound traffic to flag anomalies. 52 | trusted_ips: 53 | - 10.0.0.0/8 54 | - 127.0.0.0/8 55 | - 169.254.0.0/16 56 | - 172.16.0.0/12 57 | - 192.168.0.0/16 58 | - 224.0.0.0/8 59 | - 255.255.255.255/32 60 | - fc00::/7 61 | - fe80::/10 62 | - ff00::/8 63 | -------------------------------------------------------------------------------- /scripts/postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | chmod +x /usr/bin/nfr 4 | systemctl daemon-reload 5 | 6 | exit 0 7 | -------------------------------------------------------------------------------- /scripts/postrm: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | systemctl daemon-reload 4 | 5 | exit 0 6 | -------------------------------------------------------------------------------- /scripts/prerm: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | systemctl stop nfr 4 | systemctl disable nfr 5 | 6 | exit 0 7 | -------------------------------------------------------------------------------- /sniffer/sniffer.go: -------------------------------------------------------------------------------- 1 | package sniffer 2 | 3 | import ( 4 | "github.com/google/gopacket" 5 | "github.com/google/gopacket/pcap" 6 | ) 7 | 8 | // Sniffer is an interface for iterate over captured packets. 9 | type Sniffer interface { 10 | Packets() chan gopacket.Packet // channel with captured packets 11 | } 12 | 13 | // PcapSniffer sniffs dns packets. 14 | type PcapSniffer struct { 15 | handle *pcap.Handle 16 | source *gopacket.PacketSource 17 | } 18 | 19 | // Config options for sniffer. 20 | type Config struct { 21 | BPFilter string 22 | } 23 | 24 | // NewLivePcapSniffer creates sniffer that capture packets from interface. 25 | func NewLivePcapSniffer(iface string, cfg *Config) (*PcapSniffer, error) { 26 | handle, err := pcap.OpenLive(iface, 1600, true, pcap.BlockForever) 27 | if err != nil { 28 | return nil, err 29 | } 30 | return newsniffer(handle, cfg) 31 | } 32 | 33 | // NewOfflinePcapSniffer creates sniffer that capture packets from pcap file. 34 | func NewOfflinePcapSniffer(file string, cfg *Config) (*PcapSniffer, error) { 35 | handle, err := pcap.OpenOffline(file) 36 | if err != nil { 37 | return nil, err 38 | } 39 | return newsniffer(handle, cfg) 40 | } 41 | 42 | // newsniffer creates new sniffer and sets pcap filter for it. 43 | func newsniffer(handle *pcap.Handle, cfg *Config) (*PcapSniffer, error) { 44 | if err := handle.SetBPFFilter(cfg.BPFilter); err != nil { 45 | handle.Close() 46 | return nil, err 47 | } 48 | 49 | return &PcapSniffer{ 50 | source: gopacket.NewPacketSource(handle, handle.LinkType()), 51 | handle: handle, 52 | }, nil 53 | } 54 | 55 | // Packets returns a channel of captured packets, allowing easy iterating over them. 56 | func (s *PcapSniffer) Packets() chan gopacket.Packet { 57 | return s.source.Packets() 58 | } 59 | 60 | // Close closes underlying handle and stops sniffer. 61 | func (s *PcapSniffer) Close() { 62 | s.handle.Close() 63 | } 64 | -------------------------------------------------------------------------------- /sniffer/sniffer_test.data: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alphasoc/nfr/03d7c4b8b628ba2c68a22c60a42b5a6329e09e44/sniffer/sniffer_test.data -------------------------------------------------------------------------------- /sniffer/sniffer_test.go: -------------------------------------------------------------------------------- 1 | package sniffer 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/gopacket/pcap" 7 | ) 8 | 9 | func TestPcapSnifferPackets(t *testing.T) { 10 | cfg := &Config{ 11 | BPFilter: "tcp or udp", 12 | } 13 | if _, err := NewOfflinePcapSniffer("no.data", cfg); err == nil { 14 | t.Fatal("sniffer create without error for non existing file") 15 | } 16 | 17 | s, err := NewOfflinePcapSniffer("sniffer_test.data", cfg) 18 | if err != nil { 19 | t.Fatal(err) 20 | } 21 | defer s.Close() 22 | 23 | i := 0 24 | for range s.Packets() { 25 | i++ 26 | } 27 | 28 | if i != 2 { 29 | t.Errorf("invalid packet count - got: %d, expected: 2", i) 30 | } 31 | } 32 | 33 | func TestNewSniffer(t *testing.T) { 34 | cfg := &Config{ 35 | BPFilter: "tcp or udp", 36 | } 37 | handle, err := pcap.OpenOffline("sniffer_test.data") 38 | if err != nil { 39 | t.Fatal(err) 40 | } 41 | if _, err := newsniffer(handle, cfg); err != nil { 42 | t.Fatal(err) 43 | } 44 | } 45 | 46 | func TestNewLivePcapSniffer(t *testing.T) { 47 | if _, err := NewLivePcapSniffer("__none", nil); err == nil { 48 | t.Fatal("sniffer create without error for non existing interface") 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /utils/net.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "errors" 5 | "net" 6 | ) 7 | 8 | // IsDomainName checks if a string is a presentation-format domain name 9 | // (currently restricted to hostname-compatible "preferred name" LDH labels and 10 | // SRV-like "underscore labels"; see golang.org/issue/12421). 11 | // source in net.isDomainName 12 | func IsDomainName(s string) bool { 13 | // See RFC 1035, RFC 3696. 14 | // Presentation format has dots before every label except the first, and the 15 | // terminal empty label is optional here because we assume fully-qualified 16 | // (absolute) input. We must therefore reserve space for the first and last 17 | // labels' length octets in wire format, where they are necessary and the 18 | // maximum total length is 255. 19 | // So our _effective_ maximum is 253, but 254 is not rejected if the last 20 | // character is a dot. 21 | l := len(s) 22 | if l == 0 || l > 254 || l == 254 && s[l-1] != '.' { 23 | return false 24 | } 25 | 26 | last := byte('.') 27 | ok := false // Ok once we've seen a letter. 28 | partlen := 0 29 | for i := 0; i < len(s); i++ { 30 | c := s[i] 31 | switch { 32 | default: 33 | return false 34 | case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '_': 35 | ok = true 36 | partlen++ 37 | case '0' <= c && c <= '9': 38 | // fine 39 | partlen++ 40 | case c == '-': 41 | // Byte before dash cannot be dot. 42 | if last == '.' { 43 | return false 44 | } 45 | partlen++ 46 | case c == '.': 47 | // Byte before dot cannot be dot, dash. 48 | if last == '.' || last == '-' { 49 | return false 50 | } 51 | if partlen > 63 || partlen == 0 { 52 | return false 53 | } 54 | partlen = 0 55 | } 56 | last = c 57 | } 58 | if last == '-' || partlen > 63 { 59 | return false 60 | } 61 | 62 | return ok 63 | } 64 | 65 | // InterfaceWithPublicIP finds first interface with public ip. 66 | // If interface is not found then error is returned. 67 | func InterfaceWithPublicIP() (*net.Interface, error) { 68 | ifaces, err := net.Interfaces() 69 | if err != nil { 70 | return nil, err 71 | } 72 | for _, iface := range ifaces { 73 | if iface.Flags&net.FlagUp == 0 || 74 | iface.Flags&net.FlagLoopback != 0 { 75 | continue 76 | } 77 | ip, err := publicIPFromInterface(&iface) 78 | if err != nil { 79 | return nil, err 80 | } 81 | if ip != nil { 82 | return &iface, nil 83 | } 84 | } 85 | return nil, errors.New("no interfaces with public ip") 86 | } 87 | 88 | func publicIPFromInterface(iface *net.Interface) (net.IP, error) { 89 | addrs, err := iface.Addrs() 90 | if err != nil { 91 | return nil, err 92 | } 93 | for _, addr := range addrs { 94 | var ip net.IP 95 | switch v := addr.(type) { 96 | case *net.IPAddr: 97 | ip = v.IP 98 | case *net.IPNet: 99 | ip = v.IP 100 | } 101 | if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { 102 | continue 103 | } 104 | return ip, nil 105 | } 106 | return nil, nil 107 | } 108 | -------------------------------------------------------------------------------- /utils/special_ips.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "net" 4 | 5 | // IsSpecialIP returns true if given ip belongs 6 | // to network address from RFC 3330 and RFC 5166. 7 | func IsSpecialIP(ip net.IP) bool { 8 | for _, net := range SpecialIPv4Addresses { 9 | if net.Contains(ip) { 10 | return true 11 | } 12 | } 13 | for _, net := range SpecialIPv6Addresses { 14 | if net.Contains(ip) { 15 | return true 16 | } 17 | } 18 | return false 19 | } 20 | 21 | // Special-Use IPv4 and IPv6 Addresses 22 | var ( 23 | // Special-Use IPv4 Addresses RFC 3330 (https://tools.ietf.org/html/rfc3330) 24 | SpecialIPv4Addresses = []*net.IPNet{ 25 | { 26 | // "This" Network 27 | IP: net.IPv4(0, 0, 0, 0), 28 | Mask: net.CIDRMask(8, 32), 29 | }, 30 | { 31 | // Private-Use Networks 32 | IP: net.IPv4(10, 0, 0, 0), 33 | Mask: net.CIDRMask(8, 32), 34 | }, 35 | { 36 | // Public-Data Networks 37 | IP: net.IPv4(14, 0, 0, 0), 38 | Mask: net.CIDRMask(8, 32), 39 | }, 40 | { 41 | // Cable Television Networks 42 | IP: net.IPv4(24, 0, 0, 0), 43 | Mask: net.CIDRMask(8, 32), 44 | }, 45 | { 46 | // Reserved but subject to allocation 47 | IP: net.IPv4(39, 0, 0, 0), 48 | Mask: net.CIDRMask(8, 32), 49 | }, 50 | { 51 | // Loopback 52 | IP: net.IPv4(127, 0, 0, 0), 53 | Mask: net.CIDRMask(8, 32), 54 | }, 55 | { 56 | // Reserved but subject to allocation 57 | IP: net.IPv4(128, 0, 0, 0), 58 | Mask: net.CIDRMask(16, 32), 59 | }, 60 | { 61 | // Link Local 62 | IP: net.IPv4(169, 254, 0, 0), 63 | Mask: net.CIDRMask(16, 32), 64 | }, 65 | { 66 | // Private-Use Networks 67 | IP: net.IPv4(172, 16, 0, 0), 68 | Mask: net.CIDRMask(12, 32), 69 | }, 70 | { 71 | // Reserved but subject to allocation 72 | IP: net.IPv4(191, 255, 0, 0), 73 | Mask: net.CIDRMask(16, 32), 74 | }, 75 | { 76 | // Reserved but subject to allocation 77 | IP: net.IPv4(192, 0, 0, 0), 78 | Mask: net.CIDRMask(24, 32), 79 | }, 80 | { 81 | // Test-Net 82 | IP: net.IPv4(192, 0, 2, 0), 83 | Mask: net.CIDRMask(24, 32), 84 | }, 85 | { 86 | // 6to4 Relay Anycast 87 | IP: net.IPv4(192, 88, 99, 0), 88 | Mask: net.CIDRMask(24, 32), 89 | }, 90 | { 91 | // Private-Use Networks 92 | IP: net.IPv4(192, 168, 0, 0), 93 | Mask: net.CIDRMask(16, 32), 94 | }, 95 | { 96 | // Network Interconnect Device Benchmark Testing 97 | IP: net.IPv4(198, 18, 0, 0), 98 | Mask: net.CIDRMask(15, 32), 99 | }, 100 | { 101 | // Reserved but subject to allocation 102 | IP: net.IPv4(223, 255, 255, 0), 103 | Mask: net.CIDRMask(24, 32), 104 | }, 105 | { 106 | // Multicast 107 | IP: net.IPv4(224, 0, 0, 0), 108 | Mask: net.CIDRMask(4, 32), 109 | }, 110 | { 111 | // Reserved for Future Use 112 | IP: net.IPv4(240, 0, 0, 0), 113 | Mask: net.CIDRMask(4, 32), 114 | }, 115 | } 116 | 117 | // Special-Use IPv6 Addresses RFC (https://tools.ietf.org/html/rfc5166) 118 | SpecialIPv6Addresses = []*net.IPNet{ 119 | { 120 | // loopback address 121 | IP: net.IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x01}, 122 | Mask: net.CIDRMask(128, 128), 123 | }, 124 | { 125 | // Link-Scoped Unicast addresses 126 | IP: net.IP{0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 127 | Mask: net.CIDRMask(10, 128), 128 | }, 129 | { 130 | // Unique-Local addresses 131 | IP: net.IP{0xfc, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 132 | Mask: net.CIDRMask(7, 128), 133 | }, 134 | { 135 | // Documentation addresses 136 | IP: net.IP{0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 137 | 138 | Mask: net.CIDRMask(32, 128), 139 | }, 140 | { 141 | // 6to4 addresses 142 | IP: net.IP{0x20, 0x02, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 143 | Mask: net.CIDRMask(16, 128), 144 | }, 145 | { 146 | // Teredo addresses 147 | IP: net.IP{0x20, 0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 148 | Mask: net.CIDRMask(32, 128), 149 | }, 150 | { 151 | // 6bone addresses 152 | IP: net.IP{0x5f, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 153 | Mask: net.CIDRMask(8, 128), 154 | }, 155 | { 156 | // 6bone addresses 157 | IP: net.IP{0x3f, 0xfe, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 158 | Mask: net.CIDRMask(16, 128), 159 | }, 160 | { 161 | // Overlay Routable Cryptographic Hash IDentifiers (ORCHID) addresses 162 | IP: net.IP{0x20, 0x01, 0, 0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 163 | Mask: net.CIDRMask(28, 128), 164 | }, 165 | { 166 | // Multicast addresses 167 | IP: net.IP{0xff, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 168 | Mask: net.CIDRMask(8, 128), 169 | }, 170 | } 171 | ) 172 | -------------------------------------------------------------------------------- /utils/special_ips_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | ) 7 | 8 | func BenchmarkIsSpecialIP(b *testing.B) { 9 | ip := net.IPv4(1, 2, 3, 4) 10 | for n := 0; n < b.N; n++ { 11 | IsSpecialIP(ip) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /utils/strings.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | // StringsContains checks if given val exists in s slice. 4 | func StringsContains(s []string, val string) bool { 5 | for i := range s { 6 | if s[i] == val { 7 | return true 8 | } 9 | } 10 | return false 11 | } 12 | -------------------------------------------------------------------------------- /utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "net/mail" 7 | "os" 8 | "strings" 9 | ) 10 | 11 | func emailValidator(email string) bool { 12 | _, err := mail.ParseAddress(email) 13 | return err == nil 14 | } 15 | 16 | // Details about user. 17 | type Details struct { 18 | Name string 19 | Email string 20 | } 21 | 22 | // GetAccountRegisterDetails prompts user for registration data 23 | // like name, email, organizatoin. 24 | func GetAccountRegisterDetails() (*Details, error) { 25 | name, err := getInfo("Full Name", nil) 26 | if err != nil { 27 | return nil, err 28 | } 29 | 30 | email, err := getInfo("Email", emailValidator) 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | address, err := mail.ParseAddress(email) 36 | if err != nil { 37 | return nil, err 38 | } 39 | email = address.Address 40 | 41 | return &Details{ 42 | Name: name, 43 | Email: email, 44 | }, nil 45 | } 46 | 47 | // maximum number of tries for user input before return error. 48 | const maxTries = 2 49 | 50 | func getInfo(prompt string, validator func(string) bool) (string, error) { 51 | scanner := bufio.NewScanner(os.Stdin) 52 | fmt.Printf("%s: ", prompt) 53 | for i := maxTries; scanner.Scan() && i > 0; i-- { 54 | text := scanner.Text() 55 | if text == "" { 56 | fmt.Printf("%s can't be blank, try again (%d tries left)\n", prompt, i) 57 | } else if validator != nil && !validator(text) { 58 | fmt.Printf("invalid format, try again (%d tries left)\n", i) 59 | } else { 60 | return text, nil 61 | } 62 | fmt.Printf("%s: ", prompt) 63 | } 64 | return "", fmt.Errorf("No input for %s", prompt) 65 | } 66 | 67 | // ShadowKey replaces middel of key with dots, so it could be safe 68 | // printed to the console. 69 | func ShadowKey(key string) string { 70 | l := len(key) 71 | switch { 72 | case l == 0: 73 | return "" 74 | case l < 3: 75 | return strings.Repeat(".", 5) 76 | case l < 10: 77 | return string(key[0]) + strings.Repeat(".", 5) + string(key[l-1]) 78 | default: 79 | return key[:3] + strings.Repeat(".", 5) + key[l-3:] 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | // Version nfr 4 | var Version = "0.0.0" 5 | --------------------------------------------------------------------------------