├── .circleci └── config.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── build.sh ├── build_tlds.sh ├── do_changelog.sh ├── go.mod ├── go.sum ├── main.go ├── snapcraft.yaml └── xioc ├── funcs.go ├── funcs_test.go ├── tests.json └── tlds.go /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Golang CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-go/ for more details 4 | version: 2 5 | jobs: 6 | build: 7 | docker: 8 | - image: circleci/golang:latest 9 | working_directory: /go/src/github.com/assafmo/xioc 10 | steps: 11 | - checkout 12 | - run: go version 13 | - run: go get -v -t -d ./... 14 | - run: go get github.com/mattn/goveralls 15 | - run: go test -v -cover -race -coverprofile=./coverage.out ./... 16 | - run: $GOPATH/bin/goveralls -coverprofile=./coverage.out -service=circle-ci -repotoken=$COVERALLS_TOKEN 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | debug 2 | debug.test 3 | .vscode 4 | *pdf 5 | release 6 | 7 | *.out 8 | *.pb.gz 9 | *.test 10 | *pprof* 11 | 12 | # snap 13 | stage 14 | snap 15 | prime 16 | parts 17 | *.snap 18 | *.xdelta3 19 | *.tar.bz2 -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v1.1.12 (Apr 19, 2020) 4 | - update tlds.go 5 | 6 | ## v1.1.11 (Jan 30, 2019) 7 | 8 | - fix crash if stdin line bigger than 5MiB (issue #7) 9 | 10 | ## v1.1.10 (Jan 23, 2019) 11 | 12 | - de-defang `{.}` => `.` 13 | - de-defang `{@}` => `@` 14 | - allow whitespace before and after special defangs like `{.}`, `[@]`, `(at)`, etc. 15 | 16 | ## v1.1.7 (Jan 21, 2019) 17 | 18 | - increase max read buffer from 64KiB to 5MiB 19 | 20 | ## v1.1.6 (Jan 21, 2019) 21 | 22 | - domain: support internationalized domain names (IDNs) 23 | 24 | ## v1.1.5 (Jan 21, 2019) 25 | 26 | - url: catch "hzzzp" and "hxxxp" 27 | - url: http is case insensitive 28 | 29 | ## v1.1.2 (Jan 21, 2019) 30 | 31 | - url: must contain a domain or an ip 32 | 33 | ## v1.1.1 (Jan 21, 2019) 34 | 35 | - url: catch "://(space)" 36 | 37 | ## v1.1.0 (Jan 20, 2019) 38 | 39 | - -o flag: extract only selected types 40 | 41 | ## v1.0.0 (Jan 20, 2019) 42 | 43 | - extract ip4, ip6, domain, url, email, md5, sha1, sha256 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Assaf Morami 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # xioc 2 | 3 | Extract indicators of compromise from text, including "escaped" ones like `hxxp://banana.com`, `1.1.1[.]1` and `phish at malicious dot com`. 4 | 5 | [![CircleCI](https://circleci.com/gh/assafmo/xioc.svg?style=shield&circle-token=53b168115c42a883184dd01267d549aed80c2f49)](https://circleci.com/gh/assafmo/xioc) 6 | [![Coverage Status](https://coveralls.io/repos/github/assafmo/xioc/badge.svg?branch=master)](https://coveralls.io/github/assafmo/xioc?branch=master) 7 | [![Go Report Card](https://goreportcard.com/badge/github.com/assafmo/xioc)](https://goreportcard.com/report/github.com/assafmo/xioc) 8 | [![GoDoc](https://godoc.org/github.com/assafmo/xioc/xioc?status.svg)](https://godoc.org/github.com/assafmo/xioc/xioc) 9 | 10 | ## Installation 11 | 12 | - Download a precompiled binary from https://github.com/assafmo/xioc/releases 13 | - Or... Use `go get`: 14 | 15 | ```bash 16 | go get -u github.com/assafmo/xioc 17 | ``` 18 | 19 | - Or... Use snap install (Ubuntu): 20 | 21 | ```bash 22 | snap install xioc 23 | ``` 24 | 25 | - Or use Ubuntu PPA: 26 | 27 | ```bash 28 | curl -SsL https://assafmo.github.io/ppa/ubuntu/KEY.gpg | sudo apt-key add - 29 | sudo curl -SsL -o /etc/apt/sources.list.d/assafmo.list https://assafmo.github.io/ppa/ubuntu/assafmo.list 30 | sudo apt update 31 | sudo apt install xioc 32 | ``` 33 | 34 | ## Features 35 | 36 | - Extract IOCs (indicators of compromise) from an input text: 37 | - IPv4 38 | - IPv6 39 | - Domain 40 | - URL 41 | - Email 42 | - MD5 43 | - SHA1 44 | - SHA256 45 | - Translate some kinds of "escaping"/"defanging" techniques: 46 | - `(dot)`, `[dot]`, `(.)`, `[.]`, `{.}` to `.`. 47 | - `(at)`, `[at]`, `(@)`, `[@]`, `{@}` to `@`. 48 | - `hxxp`, `hzzzp`, `hxxxp`, `hXXp`, `h__p`, `h**p` to `http`. 49 | - Command line interface 50 | - Go library 51 | 52 | ## Command line usage 53 | 54 | ```bash 55 | $ xioc -h 56 | Usage of xioc: 57 | -o string 58 | Extract only specified types. 59 | Types must be comma seperated. E.g: xioc -o "ip4,domain,url,md5" 60 | Available types: 61 | - ip4 62 | - ip6 63 | - domain 64 | - url 65 | - email 66 | - md5 67 | - sha1 68 | - sha256 69 | -v Print version and exit 70 | ``` 71 | 72 | ```bash 73 | $ REPORT="https://unit42.paloaltonetworks.com/digital-quartermaster-scenario-demonstrated-in-attacks-against-the-mongolian-government/" 74 | $ lynx -dump "$REPORT" | xioc 75 | sha256 5beb50d95c1e720143ca0004f5172cb8881d75f6c9f434ceaff59f34fa1fe378 76 | domain energy.gov.mn 77 | email altangadas@energy.gov.mn 78 | sha256 10090692ff40758a08bd66f806e0f2c831b4b9742bbf3d19c250e778de638f57 79 | # ... 80 | ``` 81 | 82 | ```bash 83 | $ REPORT="https://unit42.paloaltonetworks.com/digital-quartermaster-scenario-demonstrated-in-attacks-against-the-mongolian-government/" 84 | $ lynx -dump "$REPORT" | xioc -o email,sha256 85 | sha256 5beb50d95c1e720143ca0004f5172cb8881d75f6c9f434ceaff59f34fa1fe378 86 | email altangadas@energy.gov.mn 87 | sha256 10090692ff40758a08bd66f806e0f2c831b4b9742bbf3d19c250e778de638f57 88 | email ganbat_g@bpo.gov.mn 89 | # ... 90 | ``` 91 | 92 | ## Library usage 93 | 94 | Full API: 95 | [![GoDoc](https://godoc.org/github.com/assafmo/xioc/xioc?status.svg)](https://godoc.org/github.com/assafmo/xioc/xioc) 96 | 97 | ```golang 98 | package main 99 | 100 | import ( 101 | "fmt" 102 | 103 | "github.com/assafmo/xioc/xioc" 104 | ) 105 | 106 | func main() { 107 | input := `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 108 | banana.com 109 | hxxp://i.robot.com/robots.txt 110 | 1.2.3.4 111 | 1.1.1[.]1 112 | info at gmail dot com 113 | hxxps://m.twitter[dot]com/` 114 | 115 | fmt.Println(xioc.ExtractDomains(input)) // => [i.robot.com m.twitter.com gmail.com banana.com] 116 | fmt.Println(xioc.ExtractSHA256s(input)) // => [e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855] 117 | fmt.Println(xioc.ExtractMD5s(input)) // => [] 118 | fmt.Println(xioc.ExtractIPv4s(input)) // => [1.2.3.4 1.1.1.1] 119 | fmt.Println(xioc.ExtractURLs(input)) // => [http://i.robot.com/robots.txt https://m.twitter.com/] 120 | fmt.Println(xioc.ExtractEmails(input)) // => [info@gmail.com] 121 | } 122 | ``` 123 | 124 | ## Sources 125 | 126 | - Test email address: http://codefool.tumblr.com/post/15288874550/list-of-valid-and-invalid-email-addresses 127 | - Domains can start with a number: https://serverfault.com/a/638270 128 | - IPv6 Examples: http://www.gestioip.net/docu/ipv6_address_examples.html 129 | - Fang and defang IOCs: https://github.com/ioc-fang/ioc_fanger 130 | - Indicator of Compromise (De)Fanging Project: https://ioc-fang.hightower.space/ 131 | - InQuest/python-iocextract test data: https://github.com/InQuest/python-iocextract/tree/master/test_data 132 | - Email address can be treated as case-insensitive: https://stackoverflow.com/a/9808332 133 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # build into ./release/ 4 | 5 | set -e 6 | set -v 7 | 8 | go test -race -cover ./... 9 | 10 | rm -rf release 11 | mkdir -p release 12 | 13 | VERSION=$(git describe --tags $(git rev-list --tags --max-count=1)) 14 | 15 | # https://golang.org/doc/install/source#environment 16 | GOOS=linux GOARCH=amd64 go build -ldflags '-s -w' -o "release/xioc-linux64-${VERSION}" 17 | GOOS=windows GOARCH=amd64 go build -ldflags '-s -w' -o "release/xioc-win64-${VERSION}.exe" 18 | GOOS=darwin GOARCH=amd64 go build -ldflags '-s -w' -o "release/xioc-macos64-${VERSION}" 19 | 20 | ( 21 | # zip 22 | cd release 23 | find -type f | 24 | parallel --bar 'zip "$(echo "{}" | sed "s/.exe//").zip" "{}" && rm -f "{}"' 25 | 26 | # deb 27 | mkdir -p ./deb/bin 28 | unzip -o -d ./deb/bin xioc-linux64-* 29 | mv -f ./deb/bin/xioc-linux64-* ./deb/bin/xioc 30 | 31 | mkdir -p ./deb/DEBIAN 32 | cat > ./deb/DEBIAN/control < 38 | Homepage: https://github.com/assafmo/xioc 39 | Installed-Size: $(ls -l --block-size=KB ./deb/bin/xioc | awk '{print $5}' | tr -d 'kB') 40 | Description: Extract indicators of compromise from text, including "escaped" ones. 41 | EOF 42 | 43 | dpkg-deb --build ./deb/ . 44 | rm -rf ./deb/ 45 | ) 46 | 47 | # publish ubuntu snap 48 | 49 | rm -rf snap *.snap* *_source.tar.bz2 50 | snapcraft 51 | snapcraft push *.snap 52 | REV=$(snapcraft list-revisions xioc | head -2 | tail -1 | awk '{print $1}') 53 | snapcraft release xioc "$REV" stable 54 | snapcraft clean 55 | rm -rf snap *.snap* *_source.tar.bz2 -------------------------------------------------------------------------------- /build_tlds.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo -e "package xioc\n" > xioc/tlds.go 4 | echo "// KnownTLDs is a set of all known TLDs" >> xioc/tlds.go 5 | echo "var KnownTLDs = map[string]bool{" >> xioc/tlds.go 6 | 7 | curl -s --compressed https://data.iana.org/TLD/tlds-alpha-by-domain.txt | 8 | grep -vF '#' | 9 | awk '{print "\""tolower($0)"\": true,"}' >> xioc/tlds.go 10 | 11 | echo '"onion": true,' >> xioc/tlds.go 12 | 13 | echo "}" >> xioc/tlds.go 14 | -------------------------------------------------------------------------------- /do_changelog.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 4 | 5 | echo '# Changelog' > "${ROOT_DIR}/CHANGELOG.md" 6 | 7 | git log --decorate | 8 | sed -r 's/.*?tag: ([0-9.v]+).*/## \1/' | 9 | awk '/##/{printf $0} /Date:/{print " ("$3,$4",",$6")"} /^ /{print "-"$0}' | 10 | sed 's/ / /' | 11 | grep -vP '^ \(' | 12 | grep -vP '^[ -]+v' | 13 | sed -r 's/##/\n##/' >> "${ROOT_DIR}/CHANGELOG.md" -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/assafmo/xioc 2 | 3 | go 1.14 4 | 5 | require golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 2 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= 3 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 4 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 5 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 6 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 7 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 8 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "os" 10 | "strings" 11 | 12 | "github.com/assafmo/xioc/xioc" 13 | ) 14 | 15 | type extractFunction func(text string) []string 16 | 17 | var availableFunctions = map[string]extractFunction{ 18 | "domain": xioc.ExtractDomains, 19 | "email": xioc.ExtractEmails, 20 | "ip4": xioc.ExtractIPv4s, 21 | "ip6": xioc.ExtractIPv6s, 22 | "url": xioc.ExtractURLs, 23 | "md5": xioc.ExtractMD5s, 24 | "sha1": xioc.ExtractSHA1s, 25 | "sha256": xioc.ExtractSHA256s, 26 | } 27 | 28 | const version = "1.1.11" 29 | 30 | var versionFlag bool 31 | var onlyFlag string 32 | 33 | func init() { 34 | flag.BoolVar(&versionFlag, "v", false, "Print version and exit") 35 | flag.StringVar(&onlyFlag, "o", "", `Extract only specified types. 36 | Types must be comma seperated. E.g: xioc -o "ip4,domain,url,md5" 37 | Available types: 38 | - ip4 39 | - ip6 40 | - domain 41 | - url 42 | - email 43 | - md5 44 | - sha1 45 | - sha256`) 46 | 47 | flag.Parse() 48 | } 49 | 50 | func main() { 51 | // if -v flag, print version and exit 52 | if versionFlag { 53 | fmt.Printf("xioc v%s\n\n", version) 54 | fmt.Println("Extract domains, ips, urls, emails, md5, sha1 and sha256 from text.") 55 | fmt.Println("For more info visit https://github.com/assafmo/xioc") 56 | return 57 | } 58 | 59 | functions := availableFunctions 60 | if onlyFlag != "" { 61 | functions = map[string]extractFunction{} 62 | 63 | types := strings.Split(onlyFlag, ",") 64 | for _, t := range types { 65 | if f, ok := availableFunctions[t]; ok { 66 | functions[t] = f 67 | } else { 68 | fmt.Printf(`Unknown extraction type "%s"`+"\n", t) 69 | os.Exit(1) 70 | } 71 | } 72 | } 73 | 74 | fi, _ := os.Stdin.Stat() 75 | 76 | if (fi.Mode() & os.ModeCharDevice) != 0 { 77 | fmt.Println("Cannot read data from stdin.") 78 | os.Exit(1) 79 | } 80 | 81 | scanner := bufio.NewScanner(os.Stdin) 82 | buf := make([]byte, 64*1024) // 64KiB initial size 83 | scanner.Buffer(buf, maxLineSize+1) // 5MiB max size 84 | scanner.Split(scanLinesMax5MiB) 85 | 86 | for scanner.Scan() { 87 | text := scanner.Text() 88 | for iocType, f := range functions { 89 | results := f(text) 90 | for _, ioc := range results { 91 | fmt.Printf("%s\t%s\n", iocType, ioc) 92 | } 93 | } 94 | 95 | } 96 | 97 | if err := scanner.Err(); err != nil { 98 | log.Fatal(err) 99 | } 100 | } 101 | 102 | // dropCR drops a terminal \r from the data. 103 | // taken from https://golang.org/src/bufio/scan.go 104 | func dropCR(data []byte) []byte { 105 | if len(data) > 0 && data[len(data)-1] == '\r' { 106 | return data[0 : len(data)-1] 107 | } 108 | return data 109 | } 110 | 111 | var maxLineSize = 5 * 1024 * 1024 112 | 113 | // modified scanLines from https://golang.org/src/bufio/scan.go 114 | func scanLinesMax5MiB(data []byte, atEOF bool) (advance int, token []byte, err error) { 115 | if atEOF && len(data) == 0 { 116 | return 0, nil, nil 117 | } 118 | if i := bytes.IndexByte(data, '\n'); i >= 0 { 119 | // We have a full newline-terminated line. 120 | return i + 1, dropCR(data[0:i]), nil 121 | } 122 | if len(data) > maxLineSize { 123 | return len(data), dropCR(data), nil 124 | } 125 | // If we're at EOF, we have a final, non-terminated line. Return it. 126 | if atEOF { 127 | return len(data), dropCR(data), nil 128 | } 129 | // Request more data. 130 | return 0, nil, nil 131 | } 132 | -------------------------------------------------------------------------------- /snapcraft.yaml: -------------------------------------------------------------------------------- 1 | name: xioc 2 | version: git 3 | version-script: | 4 | echo $(git describe --tags $(git rev-list --tags --max-count=1))-$(git log --oneline | head -1 | awk '{print $1}') 5 | summary: Extract indicators of compromise from text, including "escaped" ones. 6 | description: | 7 | Extract indicators of compromise from text, including "escaped" ones. 8 | grade: stable 9 | confinement: strict 10 | 11 | parts: 12 | xioc: 13 | source: . 14 | plugin: go 15 | go-importpath: github.com/assafmo/xioc 16 | 17 | apps: 18 | xioc: 19 | command: xioc 20 | -------------------------------------------------------------------------------- /xioc/funcs.go: -------------------------------------------------------------------------------- 1 | package xioc 2 | 3 | import ( 4 | "log" 5 | "net" 6 | "net/mail" 7 | "net/url" 8 | "regexp" 9 | "strings" 10 | 11 | "golang.org/x/net/idna" 12 | ) 13 | 14 | func init() { 15 | for tld := range KnownTLDs { 16 | if strings.HasPrefix(tld, "xn--") { 17 | u, err := idna.ToUnicode(tld) 18 | if err != nil { 19 | log.Fatalf(`cannot encode IDNA TLD "%s" to unicode: %v`, tld, err) 20 | } 21 | KnownTLDs[u] = true 22 | } 23 | } 24 | } 25 | 26 | var dot = `(\.|\p{Z}dot\p{Z}|\p{Z}?(\(dot\)|\[dot\]|\(\.\)|\[\.\]|\{\.\})\p{Z}?)` 27 | var dotRegex = regexp.MustCompile(`(?i)` + dot) 28 | 29 | func replaceDot(s string) string { 30 | return dotRegex.ReplaceAllString(s, ".") 31 | } 32 | 33 | var at = `(@|\p{Z}at\p{Z}|\p{Z}?(\(at\)|\[at\]|\(@\)|\[@\]|\{@\})\p{Z}?)` 34 | var atRegex = regexp.MustCompile(`(?i)` + at) 35 | 36 | func replaceAt(s string) string { 37 | return atRegex.ReplaceAllString(s, "@") 38 | } 39 | 40 | func filterOnlyValidIPs(ips []string) []string { 41 | resultSet := map[string]bool{} 42 | result := []string{} 43 | for _, ip := range ips { 44 | ip = strings.ToLower(replaceDot(ip)) 45 | 46 | if resultSet[ip] { 47 | continue 48 | } 49 | 50 | if net.ParseIP(ip) == nil { 51 | continue 52 | } 53 | 54 | if !resultSet[ip] { 55 | resultSet[ip] = true 56 | result = append(result, ip) 57 | } 58 | } 59 | return result 60 | } 61 | 62 | var ip4Regex = regexp.MustCompile(`(?i)([0-9]|` + dot + `)+`) 63 | 64 | // ExtractIPv4s extracts IPv4 addresses from an input string 65 | func ExtractIPv4s(text string) []string { 66 | ips := ip4Regex.FindAllString(text, -1) 67 | return filterOnlyValidIPs(ips) 68 | } 69 | 70 | var ip6Regex = regexp.MustCompile(`(?i)[a-f0-9:]+`) 71 | 72 | // ExtractIPv6s extracts IPv6 addresses from an input string 73 | func ExtractIPv6s(text string) []string { 74 | ips := ip6Regex.FindAllString(text, -1) 75 | return filterOnlyValidIPs(ips) 76 | } 77 | 78 | func hasKnownTLD(input string) bool { 79 | domainParts := strings.Split(input, ".") 80 | return KnownTLDs[domainParts[len(domainParts)-1]] 81 | } 82 | 83 | var emailRegex = regexp.MustCompile(`(?i)\b\S+?` + at + `\S+?` + dot + `\S+\b`) 84 | 85 | // ExtractEmails extracts email addresses from an input string 86 | func ExtractEmails(text string) []string { 87 | emails := emailRegex.FindAllString(text, -1) 88 | 89 | resultSet := map[string]bool{} 90 | result := []string{} 91 | for _, email := range emails { 92 | email = strings.ToLower(email) 93 | email = replaceAt(email) 94 | email = replaceDot(email) 95 | 96 | e, err := mail.ParseAddress(email) 97 | if err != nil { 98 | continue 99 | } 100 | 101 | email = e.Address 102 | 103 | if resultSet[email] { 104 | continue 105 | } 106 | 107 | domain := strings.Split(email, "@")[1] 108 | isIP := len(filterOnlyValidIPs([]string{domain})) == 1 109 | 110 | if !hasKnownTLD(email) && !isIP { 111 | continue 112 | } 113 | 114 | if !resultSet[email] { 115 | resultSet[email] = true 116 | result = append(result, email) 117 | } 118 | } 119 | 120 | return result 121 | } 122 | 123 | var urlRegex = regexp.MustCompile(`(?i)(h...?ps?|ftp)\[?:\]?//\s?\S+`) 124 | var urlHTTPDefangRegex = regexp.MustCompile(`(?i)(hxxp|h__p|h\*\*p|hzzzp|hxxxp)`) 125 | 126 | // ExtractURLs extracts ftp and http addresses from an input string 127 | func ExtractURLs(text string) []string { 128 | urls := urlRegex.FindAllString(text, -1) 129 | 130 | resultSet := map[string]bool{} 131 | result := []string{} 132 | for _, url := range urls { 133 | url = replaceDot(url) 134 | url = urlHTTPDefangRegex.ReplaceAllString(url, "http") 135 | url = strings.Replace(url, "[:]//", "://", -1) 136 | url = strings.Replace(url, "[:]//", "://", -1) 137 | url = strings.Replace(url, ":// ", "://", -1) 138 | url = strings.Replace(url, " ://", "://", -1) 139 | url = strings.Replace(url, "[com]", "com", -1) 140 | 141 | if !strings.HasPrefix(url, "http") && !strings.HasPrefix(url, "ftp") { 142 | continue 143 | } 144 | 145 | if !strings.Contains(url, ".") && // check for domain without mutual calls 146 | len(ExtractIPv4s(url)) == 0 && 147 | len(ExtractIPv6s(url)) == 0 { 148 | continue 149 | } 150 | 151 | if !resultSet[url] { 152 | resultSet[url] = true 153 | result = append(result, url) 154 | } 155 | } 156 | 157 | return result 158 | 159 | } 160 | 161 | var domainRegex = regexp.MustCompile(`(?i)([\p{L}\p{N}][\p{L}\p{N}\-]*` + dot + `)+\p{L}{2,}`) 162 | 163 | // ExtractDomains extracts domain names from an input string 164 | func ExtractDomains(text string) []string { 165 | domains := []string{} 166 | 167 | urls := ExtractURLs(text) 168 | for _, u := range urls { 169 | parsedURL, err := url.Parse(u) 170 | if err != nil { 171 | continue 172 | } 173 | domains = append(domains, parsedURL.Hostname()) 174 | } 175 | 176 | emails := ExtractEmails(text) 177 | for _, email := range emails { 178 | domain := strings.Split(email, "@")[1] 179 | domains = append(domains, domain) 180 | } 181 | 182 | domains = append(domains, domainRegex.FindAllString(text, -1)...) 183 | 184 | resultSet := map[string]bool{} 185 | result := []string{} 186 | for _, domain := range domains { 187 | domain = strings.ToLower(domain) 188 | domain = replaceDot(domain) 189 | 190 | if resultSet[domain] { 191 | continue 192 | } 193 | 194 | if strings.ContainsAny(domain, `!#$%^&*()+=,@:/'\"[]`+"`") || 195 | strings.Contains(domain, "..") || 196 | strings.Contains(domain, ".-") { 197 | continue 198 | } 199 | 200 | if net.ParseIP(domain) != nil { 201 | continue 202 | } 203 | 204 | if !hasKnownTLD(domain) { 205 | continue 206 | } 207 | 208 | if !resultSet[domain] { 209 | resultSet[domain] = true 210 | result = append(result, domain) 211 | } 212 | } 213 | 214 | return result 215 | } 216 | 217 | var md5Regex = regexp.MustCompile(`(?i)\b[a-f0-9]{32}\b`) 218 | var sha1Regex = regexp.MustCompile(`(?i)\b[a-f0-9]{40}\b`) 219 | var sha256Regex = regexp.MustCompile(`(?i)\b[a-f0-9]{64}\b`) 220 | 221 | // ExtractMD5s extracts md5 hex strings from an input string 222 | func ExtractMD5s(text string) []string { 223 | return md5Regex.FindAllString(text, -1) 224 | } 225 | 226 | // ExtractSHA1s extracts sha1 hex strings from an input string 227 | func ExtractSHA1s(text string) []string { 228 | return sha1Regex.FindAllString(text, -1) 229 | } 230 | 231 | // ExtractSHA256s extracts sha256 hex strings from an input string 232 | func ExtractSHA256s(text string) []string { 233 | return sha256Regex.FindAllString(text, -1) 234 | } 235 | -------------------------------------------------------------------------------- /xioc/funcs_test.go: -------------------------------------------------------------------------------- 1 | package xioc 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func contains(arr []string, s string) bool { 10 | for _, v := range arr { 11 | if v == s { 12 | return true 13 | } 14 | } 15 | return false 16 | } 17 | 18 | func testHelper(t *testing.T, testName string, extracted []string, expected []string) { 19 | t.Run(testName, func(t *testing.T) { 20 | if len(extracted) == 0 && len(expected) == 0 { 21 | return 22 | } 23 | 24 | for _, answer := range expected { 25 | if !contains(extracted, answer) { 26 | t.Fatalf(`"%s" should be in extracted: %v`, answer, extracted) 27 | } 28 | } 29 | 30 | for _, e := range extracted { 31 | if !contains(expected, e) { 32 | t.Fatalf(`"%s" extracted but not in expected: %v`, e, expected) 33 | } 34 | } 35 | }) 36 | } 37 | 38 | func TestExtractAddress(t *testing.T) { 39 | var tests map[string]map[string][]string 40 | 41 | f, err := os.Open("tests.json") 42 | if err != nil { 43 | t.Fatal(err) 44 | } 45 | defer f.Close() 46 | json.NewDecoder(f).Decode(&tests) 47 | 48 | testTypes := []string{"domains", "ip4s", "ip6s", "urls", "emails"} 49 | for input, expectedOutputs := range tests { 50 | for _, testType := range testTypes { 51 | var extracted []string 52 | if testType == "domains" { 53 | extracted = ExtractDomains(input) 54 | } else if testType == "ip4s" { 55 | extracted = ExtractIPv4s(input) 56 | } else if testType == "ip6s" { 57 | extracted = ExtractIPv6s(input) 58 | } else if testType == "urls" { 59 | extracted = ExtractURLs(input) 60 | } else if testType == "emails" { 61 | extracted = ExtractEmails(input) 62 | } else { 63 | t.Fatal("wat") 64 | } 65 | 66 | expected, ok := expectedOutputs[testType] 67 | if !ok { 68 | expected = []string{} 69 | } 70 | 71 | testHelper(t, testType+"=>"+input, extracted, expected) 72 | } 73 | } 74 | } 75 | 76 | func TestExtractHashes(t *testing.T) { 77 | tests := map[string]map[string][]string{ 78 | "d41d8cd98f00b204e9800998ecf8427x": { 79 | "md5s": {}, 80 | "sha1s": {}, 81 | "sha256s": {}, 82 | }, 83 | "d41d8cd98f00b204e9800998ecf8427e": { 84 | "md5s": {"d41d8cd98f00b204e9800998ecf8427e"}, 85 | "sha1s": {}, 86 | "sha256s": {}, 87 | }, 88 | "da39a3ee5e6b4b0d3255bfef95601890afd80709": { 89 | "md5s": {}, 90 | "sha1s": {"da39a3ee5e6b4b0d3255bfef95601890afd80709"}, 91 | "sha256s": {}, 92 | }, 93 | "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855": { 94 | "md5s": {}, 95 | "sha1s": {}, 96 | "sha256s": {"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, 97 | }, 98 | } 99 | 100 | testTypes := []string{"md5s", "sha1s", "sha256s"} 101 | for input, expectedOutputs := range tests { 102 | for _, testType := range testTypes { 103 | var extracted []string 104 | if testType == "md5s" { 105 | extracted = ExtractMD5s(input) 106 | } else if testType == "sha1s" { 107 | extracted = ExtractSHA1s(input) 108 | } else if testType == "sha256s" { 109 | extracted = ExtractSHA256s(input) 110 | } else { 111 | t.Fatal("wat") 112 | } 113 | 114 | expected, ok := expectedOutputs[testType] 115 | if !ok { 116 | expected = []string{} 117 | } 118 | 119 | testHelper(t, testType+"=>"+input, extracted, expected) 120 | } 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /xioc/tests.json: -------------------------------------------------------------------------------- 1 | { 2 | "banana.com": { 3 | "domains": ["banana.com"], 4 | "urls": [], 5 | "emails": [], 6 | "ip4s": [], 7 | "ip6s": [] 8 | }, 9 | "www.banana.com": { 10 | "domains": ["www.banana.com"], 11 | "urls": [], 12 | "emails": [], 13 | "ip4s": [], 14 | "ip6s": [] 15 | }, 16 | "banana[.]com": { 17 | "domains": ["banana.com"], 18 | "urls": [], 19 | "emails": [], 20 | "ip4s": [], 21 | "ip6s": [] 22 | }, 23 | "www[.]banana[.]com": { 24 | "domains": ["www.banana.com"], 25 | "urls": [], 26 | "emails": [], 27 | "ip4s": [], 28 | "ip6s": [] 29 | }, 30 | "banana[dot]com": { 31 | "domains": ["banana.com"], 32 | "urls": [], 33 | "emails": [], 34 | "ip4s": [], 35 | "ip6s": [] 36 | }, 37 | "banana-papaya.com": { 38 | "domains": ["banana-papaya.com"], 39 | "urls": [], 40 | "emails": [], 41 | "ip4s": [], 42 | "ip6s": [] 43 | }, 44 | "http://banana.papaya.com": { 45 | "domains": ["banana.papaya.com"], 46 | "urls": ["http://banana.papaya.com"], 47 | "emails": [], 48 | "ip4s": [], 49 | "ip6s": [] 50 | }, 51 | "www[.]banana-papaya[dot]com": { 52 | "domains": ["www.banana-papaya.com"], 53 | "urls": [], 54 | "emails": [], 55 | "ip4s": [], 56 | "ip6s": [] 57 | }, 58 | "-banana.com": { 59 | "domains": ["banana.com"], 60 | "urls": [], 61 | "emails": [], 62 | "ip4s": [], 63 | "ip6s": [] 64 | }, 65 | "www.-banana.com": { 66 | "domains": ["banana.com"], 67 | "urls": [], 68 | "emails": [], 69 | "ip4s": [], 70 | "ip6s": [] 71 | }, 72 | "banana..com": { 73 | "domains": [], 74 | "urls": [], 75 | "emails": [], 76 | "ip4s": [], 77 | "ip6s": [] 78 | }, 79 | "banana.doc": { 80 | "domains": [], 81 | "urls": [], 82 | "emails": [], 83 | "ip4s": [], 84 | "ip6s": [] 85 | }, 86 | "1.1.1.1": { 87 | "domains": [], 88 | "urls": [], 89 | "emails": [], 90 | "ip4s": ["1.1.1.1"] 91 | }, 92 | "1(.)1[.]1 dot 1": { 93 | "domains": [], 94 | "urls": [], 95 | "emails": [], 96 | "ip4s": ["1.1.1.1"] 97 | }, 98 | "stix_import.com": { 99 | "domains": ["import.com"], 100 | "urls": [], 101 | "emails": [], 102 | "ip4s": [], 103 | "ip6s": [] 104 | }, 105 | "stix_import(.)com": { 106 | "domains": ["import.com"], 107 | "urls": [], 108 | "emails": [], 109 | "ip4s": [], 110 | "ip6s": [] 111 | }, 112 | "stix_import(dot)com": { 113 | "domains": ["import.com"], 114 | "urls": [], 115 | "emails": [], 116 | "ip4s": [], 117 | "ip6s": [] 118 | }, 119 | "stix_import dot com": { 120 | "domains": ["import.com"], 121 | "urls": [], 122 | "emails": [], 123 | "ip4s": [], 124 | "ip6s": [] 125 | }, 126 | "dll-host-update.com;dll-host-check.com;dll-host.com": { 127 | "domains": ["dll-host-update.com", "dll-host-check.com", "dll-host.com"], 128 | "urls": [], 129 | "emails": [], 130 | "ip4s": [], 131 | "ip6s": [] 132 | }, 133 | "{PHONE_NUMBER}_npki.zip": { 134 | "domains": ["npki.zip"], 135 | "urls": [], 136 | "emails": [], 137 | "ip4s": [], 138 | "ip6s": [] 139 | }, 140 | "hxxp://m.facebook.com----------------validate---step1.rickytaylk[dot]com/sign_in.html": { 141 | "domains": [ 142 | "m.facebook.com----------------validate---step1.rickytaylk.com" 143 | ], 144 | "urls": [ 145 | "http://m.facebook.com----------------validate---step1.rickytaylk.com/sign_in.html" 146 | ], 147 | "emails": [], 148 | "ip4s": [], 149 | "ip6s": [] 150 | }, 151 | "m.facebook.com------- ---------validate---step1.rickytaylk[dot]com": { 152 | "domains": ["m.facebook.com", "validate---step1.rickytaylk.com"], 153 | "urls": [], 154 | "emails": [], 155 | "ip4s": [], 156 | "ip6s": [] 157 | }, 158 | "www.--banana.com": { 159 | "domains": ["banana.com"], 160 | "urls": [], 161 | "emails": [], 162 | "ip4s": [], 163 | "ip6s": [] 164 | }, 165 | "bne_DNSTRWIN-LJLV2NKIOKPR1009969912_779542217.shalaghlagh.tk": { 166 | "domains": ["779542217.shalaghlagh.tk"], 167 | "urls": [], 168 | "emails": [], 169 | "ip4s": [], 170 | "ip6s": [] 171 | }, 172 | "ct...pcd...m4v...m...fxg...flac..eps...dxb...drw...dot...db3...cpi...cls...cdr...arw...ai.": { 173 | "domains": [], 174 | "urls": [], 175 | "emails": [], 176 | "ip4s": [], 177 | "ip6s": [] 178 | }, 179 | "keycodes777[dot]ru/x1/home[dot]php": { 180 | "domains": ["keycodes777.ru"], 181 | "urls": [], 182 | "emails": [], 183 | "ip4s": [], 184 | "ip6s": [] 185 | }, 186 | "York……..my": { 187 | "domains": [], 188 | "urls": [], 189 | "emails": [], 190 | "ip4s": [], 191 | "ip6s": [] 192 | }, 193 | "you....so": { 194 | "domains": [], 195 | "urls": [], 196 | "emails": [], 197 | "ip4s": [], 198 | "ip6s": [] 199 | }, 200 | "whois ________.com": { 201 | "domains": [], 202 | "urls": [], 203 | "emails": [], 204 | "ip4s": [], 205 | "ip6s": [] 206 | }, 207 | "Stratford-tc(dot)gov(dot)uk": { 208 | "domains": ["stratford-tc.gov.uk"], 209 | "urls": [], 210 | "emails": [], 211 | "ip4s": [], 212 | "ip6s": [] 213 | }, 214 | "1.1.1.1000": { 215 | "domains": [], 216 | "urls": [], 217 | "emails": [], 218 | "ip4s": [], 219 | "ip6s": [] 220 | }, 221 | "192[DOT]95[DOT]42[DOT]205": { 222 | "domains": [], 223 | "urls": [], 224 | "emails": [], 225 | "ip4s": ["192.95.42.205"] 226 | }, 227 | "a@1.1.1.1": { 228 | "domains": [], 229 | "urls": [], 230 | "emails": ["a@1.1.1.1"], 231 | "ip4s": ["1.1.1.1"] 232 | }, 233 | "a@com": { 234 | "domains": [], 235 | "urls": [], 236 | "emails": [], 237 | "ip4s": [], 238 | "ip6s": [] 239 | }, 240 | "a..a@a.com": { 241 | "domains": ["a.com"], 242 | "urls": [], 243 | "emails": [], 244 | "ip4s": [], 245 | "ip6s": [] 246 | }, 247 | "a+a@a.com": { 248 | "domains": ["a.com"], 249 | "urls": [], 250 | "emails": ["a+a@a.com"], 251 | "ip4s": [], 252 | "ip6s": [] 253 | }, 254 | "t$……j@h.0": { 255 | "domains": [], 256 | "urls": [], 257 | "emails": [], 258 | "ip4s": [], 259 | "ip6s": [] 260 | }, 261 | "beta[at]zimperium[dot]com": { 262 | "domains": ["zimperium.com"], 263 | "urls": [], 264 | "emails": ["beta@zimperium.com"], 265 | "ip4s": [], 266 | "ip6s": [] 267 | }, 268 | "epiclesis[at]protonmail.ch": { 269 | "domains": ["protonmail.ch"], 270 | "urls": [], 271 | "emails": ["epiclesis@protonmail.ch"], 272 | "ip4s": [], 273 | "ip6s": [] 274 | }, 275 | "certbot failed at 0day[dot]rocks?": { 276 | "domains": ["0day.rocks"], 277 | "urls": [], 278 | "emails": ["failed@0day.rocks"], 279 | "ip4s": [], 280 | "ip6s": [] 281 | }, 282 | "tool!medium.com/@vysec.private…": { 283 | "domains": ["medium.com"], 284 | "urls": [], 285 | "emails": [], 286 | "ip4s": [], 287 | "ip6s": [] 288 | }, 289 | "ilya-soro*****12@mail.ru, v*****re@gmail.com,": { 290 | "domains": ["mail.ru", "gmail.com"], 291 | "urls": [], 292 | "emails": ["ilya-soro*****12@mail.ru", "v*****re@gmail.com"], 293 | "ip4s": [], 294 | "ip6s": [] 295 | }, 296 | "x0rz+stickers@0day[.]rocks)": { 297 | "domains": ["0day.rocks"], 298 | "urls": [], 299 | "emails": ["x0rz+stickers@0day.rocks"], 300 | "ip4s": [], 301 | "ip6s": [] 302 | }, 303 | "http://www.eset.eu/encyclopaedia/win32dursg-a-p2p-worm-agent-aak-w32-sillyp2p-trojan-c?lng=en": { 304 | "domains": ["www.eset.eu"], 305 | "urls": [ 306 | "http://www.eset.eu/encyclopaedia/win32dursg-a-p2p-worm-agent-aak-w32-sillyp2p-trojan-c?lng=en" 307 | ], 308 | "emails": [], 309 | "ip4s": [], 310 | "ip6s": [] 311 | }, 312 | "(http://blogs.pcmag.com/securitywatch/2010/02/sw_tests_show_prob": { 313 | "domains": ["blogs.pcmag.com"], 314 | "urls": ["http://blogs.pcmag.com/securitywatch/2010/02/sw_tests_show_prob"], 315 | "emails": [], 316 | "ip4s": [], 317 | "ip6s": [] 318 | }, 319 | "https://twitter.com/Nrg8000/status/957318498102865920": { 320 | "domains": ["twitter.com"], 321 | "urls": ["https://twitter.com/Nrg8000/status/957318498102865920"], 322 | "emails": [], 323 | "ip4s": [], 324 | "ip6s": [] 325 | }, 326 | "http://en.wikipedia.org/wiki/Lynx_%28web_browser%29": { 327 | "domains": ["en.wikipedia.org"], 328 | "urls": ["http://en.wikipedia.org/wiki/Lynx_%28web_browser%29"], 329 | "emails": [], 330 | "ip4s": [], 331 | "ip6s": [] 332 | }, 333 | "ftp://alpha[.]gnu.org/gnu/emacs/pretest/emacs-25[dot]0.90.tar.xz": { 334 | "domains": ["alpha.gnu.org"], 335 | "urls": ["ftp://alpha.gnu.org/gnu/emacs/pretest/emacs-25.0.90.tar.xz"], 336 | "emails": [], 337 | "ip4s": [], 338 | "ip6s": [] 339 | }, 340 | "hxxp://www.easy-home-sales[.]co.za//db_template.php": { 341 | "domains": ["www.easy-home-sales.co.za"], 342 | "urls": ["http://www.easy-home-sales.co.za//db_template.php"], 343 | "emails": [], 344 | "ip4s": [], 345 | "ip6s": [] 346 | }, 347 | "Hxxp://www.easy-home-sales[.]co.za//db_template.php": { 348 | "domains": ["www.easy-home-sales.co.za"], 349 | "urls": ["http://www.easy-home-sales.co.za//db_template.php"], 350 | "emails": [], 351 | "ip4s": [], 352 | "ip6s": [] 353 | }, 354 | "send email to sales AT zimperium dot com": { 355 | "domains": ["zimperium.com"], 356 | "urls": [], 357 | "emails": ["sales@zimperium.com"], 358 | "ip4s": [], 359 | "ip6s": [] 360 | }, 361 | "unicode fi => verifiedaccounts(dot)byethost9(dot)com/go(dot)html": { 362 | "domains": ["verifiedaccounts.byethost9.com"], 363 | "urls": [], 364 | "emails": [], 365 | "ip4s": [], 366 | "ip6s": [] 367 | }, 368 | "URL: big-product[DOT]info -> bigproducts[DOT]info/campaigns/rockyiphone/rockyiphone[DOT]php": { 369 | "domains": ["big-product.info", "bigproducts.info"], 370 | "urls": [], 371 | "emails": [], 372 | "ip4s": [], 373 | "ip6s": [] 374 | }, 375 | "https://www.youtube(dot)com/watch?v=r56baR7zSpU": { 376 | "domains": ["www.youtube.com"], 377 | "urls": ["https://www.youtube.com/watch?v=r56baR7zSpU"], 378 | "emails": [], 379 | "ip4s": [], 380 | "ip6s": [] 381 | }, 382 | "contact(at)malwaretech(dot)com ": { 383 | "domains": ["malwaretech.com"], 384 | "urls": [], 385 | "emails": ["contact@malwaretech.com"], 386 | "ip4s": [], 387 | "ip6s": [] 388 | }, 389 | "https://a:b@c.com/index.html": { 390 | "domains": ["c.com"], 391 | "urls": ["https://a:b@c.com/index.html"], 392 | "emails": [], 393 | "ip4s": [], 394 | "ip6s": [] 395 | }, 396 | " root@192.168(.)11.1 ": { 397 | "domains": [], 398 | "urls": [], 399 | "emails": ["root@192.168.11.1"], 400 | "ip4s": ["192.168.11.1"] 401 | }, 402 | "h__p://m103.pay2earn[.]net/devices": { 403 | "domains": ["m103.pay2earn.net"], 404 | "urls": ["http://m103.pay2earn.net/devices"], 405 | "emails": [], 406 | "ip4s": [], 407 | "ip6s": [] 408 | }, 409 | "http[:]//118.184.48[.]95:8000/info.vbs": { 410 | "domains": [], 411 | "urls": ["http://118.184.48.95:8000/info.vbs"], 412 | "emails": [], 413 | "ip4s": ["118.184.48.95"] 414 | }, 415 | "http://fpw.[com].my/zy": { 416 | "domains": ["fpw.com.my"], 417 | "urls": ["http://fpw.com.my/zy"], 418 | "emails": [], 419 | "ip4s": [], 420 | "ip6s": [] 421 | }, 422 | "hXXp://playback.savefrom.biz/video/video1.cab": { 423 | "domains": ["playback.savefrom.biz", "video1.cab"], 424 | "urls": ["http://playback.savefrom.biz/video/video1.cab"], 425 | "emails": [], 426 | "ip4s": [], 427 | "ip6s": [] 428 | }, 429 | "email@example.com": { 430 | "emails": ["email@example.com"], 431 | "domains": ["example.com"], 432 | "urls": [], 433 | "ip4s": [], 434 | "ip6s": [] 435 | }, 436 | "firstname.lastname@example.com": { 437 | "emails": ["firstname.lastname@example.com"], 438 | "domains": ["example.com"], 439 | "urls": [], 440 | "ip4s": [], 441 | "ip6s": [] 442 | }, 443 | "email@subdomain.example.com": { 444 | "emails": ["email@subdomain.example.com"], 445 | "domains": ["subdomain.example.com"], 446 | "urls": [], 447 | "ip4s": [], 448 | "ip6s": [] 449 | }, 450 | "firstname+lastname@example.com": { 451 | "emails": ["firstname+lastname@example.com"], 452 | "domains": ["example.com"], 453 | "urls": [], 454 | "ip4s": [], 455 | "ip6s": [] 456 | }, 457 | "email@123.123.123.123": { 458 | "emails": ["email@123.123.123.123"], 459 | "domains": [], 460 | "urls": [], 461 | "ip4s": ["123.123.123.123"] 462 | }, 463 | "1234567890@example.com": { 464 | "emails": ["1234567890@example.com"], 465 | "domains": ["example.com"], 466 | "urls": [], 467 | "ip4s": [], 468 | "ip6s": [] 469 | }, 470 | "email@example-one.com": { 471 | "emails": ["email@example-one.com"], 472 | "domains": ["example-one.com"], 473 | "urls": [], 474 | "ip4s": [], 475 | "ip6s": [] 476 | }, 477 | "_______@example.com": { 478 | "emails": ["_______@example.com"], 479 | "domains": ["example.com"], 480 | "urls": [], 481 | "ip4s": [], 482 | "ip6s": [] 483 | }, 484 | "email@example.name": { 485 | "emails": ["email@example.name"], 486 | "domains": ["example.name"], 487 | "urls": [], 488 | "ip4s": [], 489 | "ip6s": [] 490 | }, 491 | "email@example.museum": { 492 | "emails": ["email@example.museum"], 493 | "domains": ["example.museum"], 494 | "urls": [], 495 | "ip4s": [], 496 | "ip6s": [] 497 | }, 498 | "user@example.co.jp ssh user@example.co.jp": { 499 | "emails": ["user@example.co.jp"], 500 | "domains": ["example.co.jp"], 501 | "urls": [], 502 | "ip4s": [], 503 | "ip6s": [] 504 | }, 505 | "firstname-lastname@example.com": { 506 | "emails": ["firstname-lastname@example.com"], 507 | "domains": ["example.com"], 508 | "urls": [], 509 | "ip4s": [], 510 | "ip6s": [] 511 | }, 512 | "firstname-lastname..example.com": { 513 | "domains": ["example.com"], 514 | "urls": [], 515 | "ip4s": [], 516 | "ip6s": [] 517 | }, 518 | "2001:0db8:0a0b:12f0:0000:0000:0000:0001": { 519 | "ip6s": ["2001:0db8:0a0b:12f0:0000:0000:0000:0001"] 520 | }, 521 | "2001:db8:a0b:12f0::1": { 522 | "ip6s": ["2001:db8:a0b:12f0::1"] 523 | }, 524 | "2001:0db8::0001": { 525 | "ip6s": ["2001:0db8::0001"] 526 | }, 527 | "2001:db8::1 ": { 528 | "ip6s": ["2001:db8::1"] 529 | }, 530 | "2001:db8:0:0:0:0:2:1": { 531 | "ip6s": ["2001:db8:0:0:0:0:2:1"] 532 | }, 533 | "2001:db8::2:1": { 534 | "ip6s": ["2001:db8::2:1"] 535 | }, 536 | "2001:db8:0:1:1:1:1:1": { 537 | "ip6s": ["2001:db8:0:1:1:1:1:1"] 538 | }, 539 | "http://[2001:db8:a0b:12f0::1]/index.html http://[2001:db8:a0b:12f0::1]:80/index.html": { 540 | "ip6s": ["2001:db8:a0b:12f0::1"], 541 | "urls": [ 542 | "http://[2001:db8:a0b:12f0::1]/index.html", 543 | "http://[2001:db8:a0b:12f0::1]:80/index.html" 544 | ] 545 | }, 546 | "[2001:db8:a0b:12f0::1]:21": { 547 | "ip6s": ["2001:db8:a0b:12f0::1"] 548 | }, 549 | "2001:db8:a0b:12f0::1/64": { 550 | "ip6s": ["2001:db8:a0b:12f0::1"] 551 | }, 552 | "2001:db8:a0b:12f0::1%eth0": { 553 | "ip6s": ["2001:db8:a0b:12f0::1"] 554 | }, 555 | "[2001:db8:a0b:12f0::1%25eth0]:21": { 556 | "ip6s": ["2001:db8:a0b:12f0::1"] 557 | }, 558 | "[2001:db8:a0b:12f0::1%eth0]:21": { 559 | "ip6s": ["2001:db8:a0b:12f0::1"] 560 | }, 561 | "hxxp:// feeds.rapidfeeds[.]com/88604/": { 562 | "urls": ["http://feeds.rapidfeeds.com/88604/"], 563 | "domains": ["feeds.rapidfeeds.com"] 564 | }, 565 | "ed to enter http:// or https:// or www.). Clic": {}, 566 | "http://:": {}, 567 | "hzzzp://www15.youtube.com.silssl.com/watch.php?v=o8h2mD8b&c=SG&feature=youtu": { 568 | "urls": [ 569 | "http://www15.youtube.com.silssl.com/watch.php?v=o8h2mD8b&c=SG&feature=youtu" 570 | ], 571 | "domains": ["www15.youtube.com.silssl.com"] 572 | }, 573 | "HZZZP://www15.youtube.com.silssl.com/watch.php?v=o8h2mD8b&c=SG&feature=youtu": { 574 | "urls": [ 575 | "http://www15.youtube.com.silssl.com/watch.php?v=o8h2mD8b&c=SG&feature=youtu" 576 | ], 577 | "domains": ["www15.youtube.com.silssl.com"] 578 | }, 579 | "habcp://www15.youtube.com.silssl.com/watch.php?v=o8h2mD8b&c=SG&feature=youtu": { 580 | "domains": ["www15.youtube.com.silssl.com"] 581 | }, 582 | "banana.קום": { 583 | "domains": ["banana.קום"] 584 | }, 585 | "hxxp://papaya.рф/sitemap.xml": { 586 | "urls": ["http://papaya.рф/sitemap.xml"], 587 | "domains": ["papaya.рф"] 588 | }, 589 | "hxxps://revcode{.}eu/": { 590 | "urls": ["https://revcode.eu/"], 591 | "domains": ["revcode.eu"] 592 | }, 593 | "research {@} fireeye DOT COM": { 594 | "emails": ["research@fireeye.com"], 595 | "domains": ["fireeye.com"] 596 | }, 597 | "aminjalali_58 (at) yahoo.com intelreports (at) kaspersky [dot] com": { 598 | "emails": ["aminjalali_58@yahoo.com", "intelreports@kaspersky.com"], 599 | "domains": ["yahoo.com", "kaspersky.com"] 600 | }, 601 | "rEsearch@fireeye.COM": { 602 | "emails": ["research@fireeye.com"], 603 | "domains": ["fireeye.com"] 604 | } 605 | } 606 | -------------------------------------------------------------------------------- /xioc/tlds.go: -------------------------------------------------------------------------------- 1 | package xioc 2 | 3 | // KnownTLDs is a set of all known TLDs 4 | var KnownTLDs = map[string]bool{ 5 | "aaa": true, 6 | "aarp": true, 7 | "abarth": true, 8 | "abb": true, 9 | "abbott": true, 10 | "abbvie": true, 11 | "abc": true, 12 | "able": true, 13 | "abogado": true, 14 | "abudhabi": true, 15 | "ac": true, 16 | "academy": true, 17 | "accenture": true, 18 | "accountant": true, 19 | "accountants": true, 20 | "aco": true, 21 | "actor": true, 22 | "ad": true, 23 | "adac": true, 24 | "ads": true, 25 | "adult": true, 26 | "ae": true, 27 | "aeg": true, 28 | "aero": true, 29 | "aetna": true, 30 | "af": true, 31 | "afamilycompany": true, 32 | "afl": true, 33 | "africa": true, 34 | "ag": true, 35 | "agakhan": true, 36 | "agency": true, 37 | "ai": true, 38 | "aig": true, 39 | "aigo": true, 40 | "airbus": true, 41 | "airforce": true, 42 | "airtel": true, 43 | "akdn": true, 44 | "al": true, 45 | "alfaromeo": true, 46 | "alibaba": true, 47 | "alipay": true, 48 | "allfinanz": true, 49 | "allstate": true, 50 | "ally": true, 51 | "alsace": true, 52 | "alstom": true, 53 | "am": true, 54 | "americanexpress": true, 55 | "americanfamily": true, 56 | "amex": true, 57 | "amfam": true, 58 | "amica": true, 59 | "amsterdam": true, 60 | "analytics": true, 61 | "android": true, 62 | "anquan": true, 63 | "anz": true, 64 | "ao": true, 65 | "aol": true, 66 | "apartments": true, 67 | "app": true, 68 | "apple": true, 69 | "aq": true, 70 | "aquarelle": true, 71 | "ar": true, 72 | "arab": true, 73 | "aramco": true, 74 | "archi": true, 75 | "army": true, 76 | "arpa": true, 77 | "art": true, 78 | "arte": true, 79 | "as": true, 80 | "asda": true, 81 | "asia": true, 82 | "associates": true, 83 | "at": true, 84 | "athleta": true, 85 | "attorney": true, 86 | "au": true, 87 | "auction": true, 88 | "audi": true, 89 | "audible": true, 90 | "audio": true, 91 | "auspost": true, 92 | "author": true, 93 | "auto": true, 94 | "autos": true, 95 | "avianca": true, 96 | "aw": true, 97 | "aws": true, 98 | "ax": true, 99 | "axa": true, 100 | "az": true, 101 | "azure": true, 102 | "ba": true, 103 | "baby": true, 104 | "baidu": true, 105 | "banamex": true, 106 | "bananarepublic": true, 107 | "band": true, 108 | "bank": true, 109 | "bar": true, 110 | "barcelona": true, 111 | "barclaycard": true, 112 | "barclays": true, 113 | "barefoot": true, 114 | "bargains": true, 115 | "baseball": true, 116 | "basketball": true, 117 | "bauhaus": true, 118 | "bayern": true, 119 | "bb": true, 120 | "bbc": true, 121 | "bbt": true, 122 | "bbva": true, 123 | "bcg": true, 124 | "bcn": true, 125 | "bd": true, 126 | "be": true, 127 | "beats": true, 128 | "beauty": true, 129 | "beer": true, 130 | "bentley": true, 131 | "berlin": true, 132 | "best": true, 133 | "bestbuy": true, 134 | "bet": true, 135 | "bf": true, 136 | "bg": true, 137 | "bh": true, 138 | "bharti": true, 139 | "bi": true, 140 | "bible": true, 141 | "bid": true, 142 | "bike": true, 143 | "bing": true, 144 | "bingo": true, 145 | "bio": true, 146 | "biz": true, 147 | "bj": true, 148 | "black": true, 149 | "blackfriday": true, 150 | "blockbuster": true, 151 | "blog": true, 152 | "bloomberg": true, 153 | "blue": true, 154 | "bm": true, 155 | "bms": true, 156 | "bmw": true, 157 | "bn": true, 158 | "bnpparibas": true, 159 | "bo": true, 160 | "boats": true, 161 | "boehringer": true, 162 | "bofa": true, 163 | "bom": true, 164 | "bond": true, 165 | "boo": true, 166 | "book": true, 167 | "booking": true, 168 | "bosch": true, 169 | "bostik": true, 170 | "boston": true, 171 | "bot": true, 172 | "boutique": true, 173 | "box": true, 174 | "br": true, 175 | "bradesco": true, 176 | "bridgestone": true, 177 | "broadway": true, 178 | "broker": true, 179 | "brother": true, 180 | "brussels": true, 181 | "bs": true, 182 | "bt": true, 183 | "budapest": true, 184 | "bugatti": true, 185 | "build": true, 186 | "builders": true, 187 | "business": true, 188 | "buy": true, 189 | "buzz": true, 190 | "bv": true, 191 | "bw": true, 192 | "by": true, 193 | "bz": true, 194 | "bzh": true, 195 | "ca": true, 196 | "cab": true, 197 | "cafe": true, 198 | "cal": true, 199 | "call": true, 200 | "calvinklein": true, 201 | "cam": true, 202 | "camera": true, 203 | "camp": true, 204 | "cancerresearch": true, 205 | "canon": true, 206 | "capetown": true, 207 | "capital": true, 208 | "capitalone": true, 209 | "car": true, 210 | "caravan": true, 211 | "cards": true, 212 | "care": true, 213 | "career": true, 214 | "careers": true, 215 | "cars": true, 216 | "casa": true, 217 | "case": true, 218 | "caseih": true, 219 | "cash": true, 220 | "casino": true, 221 | "cat": true, 222 | "catering": true, 223 | "catholic": true, 224 | "cba": true, 225 | "cbn": true, 226 | "cbre": true, 227 | "cbs": true, 228 | "cc": true, 229 | "cd": true, 230 | "ceb": true, 231 | "center": true, 232 | "ceo": true, 233 | "cern": true, 234 | "cf": true, 235 | "cfa": true, 236 | "cfd": true, 237 | "cg": true, 238 | "ch": true, 239 | "chanel": true, 240 | "channel": true, 241 | "charity": true, 242 | "chase": true, 243 | "chat": true, 244 | "cheap": true, 245 | "chintai": true, 246 | "christmas": true, 247 | "chrome": true, 248 | "church": true, 249 | "ci": true, 250 | "cipriani": true, 251 | "circle": true, 252 | "cisco": true, 253 | "citadel": true, 254 | "citi": true, 255 | "citic": true, 256 | "city": true, 257 | "cityeats": true, 258 | "ck": true, 259 | "cl": true, 260 | "claims": true, 261 | "cleaning": true, 262 | "click": true, 263 | "clinic": true, 264 | "clinique": true, 265 | "clothing": true, 266 | "cloud": true, 267 | "club": true, 268 | "clubmed": true, 269 | "cm": true, 270 | "cn": true, 271 | "co": true, 272 | "coach": true, 273 | "codes": true, 274 | "coffee": true, 275 | "college": true, 276 | "cologne": true, 277 | "com": true, 278 | "comcast": true, 279 | "commbank": true, 280 | "community": true, 281 | "company": true, 282 | "compare": true, 283 | "computer": true, 284 | "comsec": true, 285 | "condos": true, 286 | "construction": true, 287 | "consulting": true, 288 | "contact": true, 289 | "contractors": true, 290 | "cooking": true, 291 | "cookingchannel": true, 292 | "cool": true, 293 | "coop": true, 294 | "corsica": true, 295 | "country": true, 296 | "coupon": true, 297 | "coupons": true, 298 | "courses": true, 299 | "cpa": true, 300 | "cr": true, 301 | "credit": true, 302 | "creditcard": true, 303 | "creditunion": true, 304 | "cricket": true, 305 | "crown": true, 306 | "crs": true, 307 | "cruise": true, 308 | "cruises": true, 309 | "csc": true, 310 | "cu": true, 311 | "cuisinella": true, 312 | "cv": true, 313 | "cw": true, 314 | "cx": true, 315 | "cy": true, 316 | "cymru": true, 317 | "cyou": true, 318 | "cz": true, 319 | "dabur": true, 320 | "dad": true, 321 | "dance": true, 322 | "data": true, 323 | "date": true, 324 | "dating": true, 325 | "datsun": true, 326 | "day": true, 327 | "dclk": true, 328 | "dds": true, 329 | "de": true, 330 | "deal": true, 331 | "dealer": true, 332 | "deals": true, 333 | "degree": true, 334 | "delivery": true, 335 | "dell": true, 336 | "deloitte": true, 337 | "delta": true, 338 | "democrat": true, 339 | "dental": true, 340 | "dentist": true, 341 | "desi": true, 342 | "design": true, 343 | "dev": true, 344 | "dhl": true, 345 | "diamonds": true, 346 | "diet": true, 347 | "digital": true, 348 | "direct": true, 349 | "directory": true, 350 | "discount": true, 351 | "discover": true, 352 | "dish": true, 353 | "diy": true, 354 | "dj": true, 355 | "dk": true, 356 | "dm": true, 357 | "dnp": true, 358 | "do": true, 359 | "docs": true, 360 | "doctor": true, 361 | "dog": true, 362 | "domains": true, 363 | "dot": true, 364 | "download": true, 365 | "drive": true, 366 | "dtv": true, 367 | "dubai": true, 368 | "duck": true, 369 | "dunlop": true, 370 | "dupont": true, 371 | "durban": true, 372 | "dvag": true, 373 | "dvr": true, 374 | "dz": true, 375 | "earth": true, 376 | "eat": true, 377 | "ec": true, 378 | "eco": true, 379 | "edeka": true, 380 | "edu": true, 381 | "education": true, 382 | "ee": true, 383 | "eg": true, 384 | "email": true, 385 | "emerck": true, 386 | "energy": true, 387 | "engineer": true, 388 | "engineering": true, 389 | "enterprises": true, 390 | "epson": true, 391 | "equipment": true, 392 | "er": true, 393 | "ericsson": true, 394 | "erni": true, 395 | "es": true, 396 | "esq": true, 397 | "estate": true, 398 | "esurance": true, 399 | "et": true, 400 | "etisalat": true, 401 | "eu": true, 402 | "eurovision": true, 403 | "eus": true, 404 | "events": true, 405 | "exchange": true, 406 | "expert": true, 407 | "exposed": true, 408 | "express": true, 409 | "extraspace": true, 410 | "fage": true, 411 | "fail": true, 412 | "fairwinds": true, 413 | "faith": true, 414 | "family": true, 415 | "fan": true, 416 | "fans": true, 417 | "farm": true, 418 | "farmers": true, 419 | "fashion": true, 420 | "fast": true, 421 | "fedex": true, 422 | "feedback": true, 423 | "ferrari": true, 424 | "ferrero": true, 425 | "fi": true, 426 | "fiat": true, 427 | "fidelity": true, 428 | "fido": true, 429 | "film": true, 430 | "final": true, 431 | "finance": true, 432 | "financial": true, 433 | "fire": true, 434 | "firestone": true, 435 | "firmdale": true, 436 | "fish": true, 437 | "fishing": true, 438 | "fit": true, 439 | "fitness": true, 440 | "fj": true, 441 | "fk": true, 442 | "flickr": true, 443 | "flights": true, 444 | "flir": true, 445 | "florist": true, 446 | "flowers": true, 447 | "fly": true, 448 | "fm": true, 449 | "fo": true, 450 | "foo": true, 451 | "food": true, 452 | "foodnetwork": true, 453 | "football": true, 454 | "ford": true, 455 | "forex": true, 456 | "forsale": true, 457 | "forum": true, 458 | "foundation": true, 459 | "fox": true, 460 | "fr": true, 461 | "free": true, 462 | "fresenius": true, 463 | "frl": true, 464 | "frogans": true, 465 | "frontdoor": true, 466 | "frontier": true, 467 | "ftr": true, 468 | "fujitsu": true, 469 | "fujixerox": true, 470 | "fun": true, 471 | "fund": true, 472 | "furniture": true, 473 | "futbol": true, 474 | "fyi": true, 475 | "ga": true, 476 | "gal": true, 477 | "gallery": true, 478 | "gallo": true, 479 | "gallup": true, 480 | "game": true, 481 | "games": true, 482 | "gap": true, 483 | "garden": true, 484 | "gay": true, 485 | "gb": true, 486 | "gbiz": true, 487 | "gd": true, 488 | "gdn": true, 489 | "ge": true, 490 | "gea": true, 491 | "gent": true, 492 | "genting": true, 493 | "george": true, 494 | "gf": true, 495 | "gg": true, 496 | "ggee": true, 497 | "gh": true, 498 | "gi": true, 499 | "gift": true, 500 | "gifts": true, 501 | "gives": true, 502 | "giving": true, 503 | "gl": true, 504 | "glade": true, 505 | "glass": true, 506 | "gle": true, 507 | "global": true, 508 | "globo": true, 509 | "gm": true, 510 | "gmail": true, 511 | "gmbh": true, 512 | "gmo": true, 513 | "gmx": true, 514 | "gn": true, 515 | "godaddy": true, 516 | "gold": true, 517 | "goldpoint": true, 518 | "golf": true, 519 | "goo": true, 520 | "goodyear": true, 521 | "goog": true, 522 | "google": true, 523 | "gop": true, 524 | "got": true, 525 | "gov": true, 526 | "gp": true, 527 | "gq": true, 528 | "gr": true, 529 | "grainger": true, 530 | "graphics": true, 531 | "gratis": true, 532 | "green": true, 533 | "gripe": true, 534 | "grocery": true, 535 | "group": true, 536 | "gs": true, 537 | "gt": true, 538 | "gu": true, 539 | "guardian": true, 540 | "gucci": true, 541 | "guge": true, 542 | "guide": true, 543 | "guitars": true, 544 | "guru": true, 545 | "gw": true, 546 | "gy": true, 547 | "hair": true, 548 | "hamburg": true, 549 | "hangout": true, 550 | "haus": true, 551 | "hbo": true, 552 | "hdfc": true, 553 | "hdfcbank": true, 554 | "health": true, 555 | "healthcare": true, 556 | "help": true, 557 | "helsinki": true, 558 | "here": true, 559 | "hermes": true, 560 | "hgtv": true, 561 | "hiphop": true, 562 | "hisamitsu": true, 563 | "hitachi": true, 564 | "hiv": true, 565 | "hk": true, 566 | "hkt": true, 567 | "hm": true, 568 | "hn": true, 569 | "hockey": true, 570 | "holdings": true, 571 | "holiday": true, 572 | "homedepot": true, 573 | "homegoods": true, 574 | "homes": true, 575 | "homesense": true, 576 | "honda": true, 577 | "horse": true, 578 | "hospital": true, 579 | "host": true, 580 | "hosting": true, 581 | "hot": true, 582 | "hoteles": true, 583 | "hotels": true, 584 | "hotmail": true, 585 | "house": true, 586 | "how": true, 587 | "hr": true, 588 | "hsbc": true, 589 | "ht": true, 590 | "hu": true, 591 | "hughes": true, 592 | "hyatt": true, 593 | "hyundai": true, 594 | "ibm": true, 595 | "icbc": true, 596 | "ice": true, 597 | "icu": true, 598 | "id": true, 599 | "ie": true, 600 | "ieee": true, 601 | "ifm": true, 602 | "ikano": true, 603 | "il": true, 604 | "im": true, 605 | "imamat": true, 606 | "imdb": true, 607 | "immo": true, 608 | "immobilien": true, 609 | "in": true, 610 | "inc": true, 611 | "industries": true, 612 | "infiniti": true, 613 | "info": true, 614 | "ing": true, 615 | "ink": true, 616 | "institute": true, 617 | "insurance": true, 618 | "insure": true, 619 | "int": true, 620 | "intel": true, 621 | "international": true, 622 | "intuit": true, 623 | "investments": true, 624 | "io": true, 625 | "ipiranga": true, 626 | "iq": true, 627 | "ir": true, 628 | "irish": true, 629 | "is": true, 630 | "ismaili": true, 631 | "ist": true, 632 | "istanbul": true, 633 | "it": true, 634 | "itau": true, 635 | "itv": true, 636 | "iveco": true, 637 | "jaguar": true, 638 | "java": true, 639 | "jcb": true, 640 | "jcp": true, 641 | "je": true, 642 | "jeep": true, 643 | "jetzt": true, 644 | "jewelry": true, 645 | "jio": true, 646 | "jll": true, 647 | "jm": true, 648 | "jmp": true, 649 | "jnj": true, 650 | "jo": true, 651 | "jobs": true, 652 | "joburg": true, 653 | "jot": true, 654 | "joy": true, 655 | "jp": true, 656 | "jpmorgan": true, 657 | "jprs": true, 658 | "juegos": true, 659 | "juniper": true, 660 | "kaufen": true, 661 | "kddi": true, 662 | "ke": true, 663 | "kerryhotels": true, 664 | "kerrylogistics": true, 665 | "kerryproperties": true, 666 | "kfh": true, 667 | "kg": true, 668 | "kh": true, 669 | "ki": true, 670 | "kia": true, 671 | "kim": true, 672 | "kinder": true, 673 | "kindle": true, 674 | "kitchen": true, 675 | "kiwi": true, 676 | "km": true, 677 | "kn": true, 678 | "koeln": true, 679 | "komatsu": true, 680 | "kosher": true, 681 | "kp": true, 682 | "kpmg": true, 683 | "kpn": true, 684 | "kr": true, 685 | "krd": true, 686 | "kred": true, 687 | "kuokgroup": true, 688 | "kw": true, 689 | "ky": true, 690 | "kyoto": true, 691 | "kz": true, 692 | "la": true, 693 | "lacaixa": true, 694 | "lamborghini": true, 695 | "lamer": true, 696 | "lancaster": true, 697 | "lancia": true, 698 | "land": true, 699 | "landrover": true, 700 | "lanxess": true, 701 | "lasalle": true, 702 | "lat": true, 703 | "latino": true, 704 | "latrobe": true, 705 | "law": true, 706 | "lawyer": true, 707 | "lb": true, 708 | "lc": true, 709 | "lds": true, 710 | "lease": true, 711 | "leclerc": true, 712 | "lefrak": true, 713 | "legal": true, 714 | "lego": true, 715 | "lexus": true, 716 | "lgbt": true, 717 | "li": true, 718 | "lidl": true, 719 | "life": true, 720 | "lifeinsurance": true, 721 | "lifestyle": true, 722 | "lighting": true, 723 | "like": true, 724 | "lilly": true, 725 | "limited": true, 726 | "limo": true, 727 | "lincoln": true, 728 | "linde": true, 729 | "link": true, 730 | "lipsy": true, 731 | "live": true, 732 | "living": true, 733 | "lixil": true, 734 | "lk": true, 735 | "llc": true, 736 | "llp": true, 737 | "loan": true, 738 | "loans": true, 739 | "locker": true, 740 | "locus": true, 741 | "loft": true, 742 | "lol": true, 743 | "london": true, 744 | "lotte": true, 745 | "lotto": true, 746 | "love": true, 747 | "lpl": true, 748 | "lplfinancial": true, 749 | "lr": true, 750 | "ls": true, 751 | "lt": true, 752 | "ltd": true, 753 | "ltda": true, 754 | "lu": true, 755 | "lundbeck": true, 756 | "lupin": true, 757 | "luxe": true, 758 | "luxury": true, 759 | "lv": true, 760 | "ly": true, 761 | "ma": true, 762 | "macys": true, 763 | "madrid": true, 764 | "maif": true, 765 | "maison": true, 766 | "makeup": true, 767 | "man": true, 768 | "management": true, 769 | "mango": true, 770 | "map": true, 771 | "market": true, 772 | "marketing": true, 773 | "markets": true, 774 | "marriott": true, 775 | "marshalls": true, 776 | "maserati": true, 777 | "mattel": true, 778 | "mba": true, 779 | "mc": true, 780 | "mckinsey": true, 781 | "md": true, 782 | "me": true, 783 | "med": true, 784 | "media": true, 785 | "meet": true, 786 | "melbourne": true, 787 | "meme": true, 788 | "memorial": true, 789 | "men": true, 790 | "menu": true, 791 | "merckmsd": true, 792 | "metlife": true, 793 | "mg": true, 794 | "mh": true, 795 | "miami": true, 796 | "microsoft": true, 797 | "mil": true, 798 | "mini": true, 799 | "mint": true, 800 | "mit": true, 801 | "mitsubishi": true, 802 | "mk": true, 803 | "ml": true, 804 | "mlb": true, 805 | "mls": true, 806 | "mm": true, 807 | "mma": true, 808 | "mn": true, 809 | "mo": true, 810 | "mobi": true, 811 | "mobile": true, 812 | "moda": true, 813 | "moe": true, 814 | "moi": true, 815 | "mom": true, 816 | "monash": true, 817 | "money": true, 818 | "monster": true, 819 | "mormon": true, 820 | "mortgage": true, 821 | "moscow": true, 822 | "moto": true, 823 | "motorcycles": true, 824 | "mov": true, 825 | "movie": true, 826 | "mp": true, 827 | "mq": true, 828 | "mr": true, 829 | "ms": true, 830 | "msd": true, 831 | "mt": true, 832 | "mtn": true, 833 | "mtr": true, 834 | "mu": true, 835 | "museum": true, 836 | "mutual": true, 837 | "mv": true, 838 | "mw": true, 839 | "mx": true, 840 | "my": true, 841 | "mz": true, 842 | "na": true, 843 | "nab": true, 844 | "nagoya": true, 845 | "name": true, 846 | "nationwide": true, 847 | "natura": true, 848 | "navy": true, 849 | "nba": true, 850 | "nc": true, 851 | "ne": true, 852 | "nec": true, 853 | "net": true, 854 | "netbank": true, 855 | "netflix": true, 856 | "network": true, 857 | "neustar": true, 858 | "new": true, 859 | "newholland": true, 860 | "news": true, 861 | "next": true, 862 | "nextdirect": true, 863 | "nexus": true, 864 | "nf": true, 865 | "nfl": true, 866 | "ng": true, 867 | "ngo": true, 868 | "nhk": true, 869 | "ni": true, 870 | "nico": true, 871 | "nike": true, 872 | "nikon": true, 873 | "ninja": true, 874 | "nissan": true, 875 | "nissay": true, 876 | "nl": true, 877 | "no": true, 878 | "nokia": true, 879 | "northwesternmutual": true, 880 | "norton": true, 881 | "now": true, 882 | "nowruz": true, 883 | "nowtv": true, 884 | "np": true, 885 | "nr": true, 886 | "nra": true, 887 | "nrw": true, 888 | "ntt": true, 889 | "nu": true, 890 | "nyc": true, 891 | "nz": true, 892 | "obi": true, 893 | "observer": true, 894 | "off": true, 895 | "office": true, 896 | "okinawa": true, 897 | "olayan": true, 898 | "olayangroup": true, 899 | "oldnavy": true, 900 | "ollo": true, 901 | "om": true, 902 | "omega": true, 903 | "one": true, 904 | "ong": true, 905 | "onl": true, 906 | "online": true, 907 | "onyourside": true, 908 | "ooo": true, 909 | "open": true, 910 | "oracle": true, 911 | "orange": true, 912 | "org": true, 913 | "organic": true, 914 | "origins": true, 915 | "osaka": true, 916 | "otsuka": true, 917 | "ott": true, 918 | "ovh": true, 919 | "pa": true, 920 | "page": true, 921 | "panasonic": true, 922 | "paris": true, 923 | "pars": true, 924 | "partners": true, 925 | "parts": true, 926 | "party": true, 927 | "passagens": true, 928 | "pay": true, 929 | "pccw": true, 930 | "pe": true, 931 | "pet": true, 932 | "pf": true, 933 | "pfizer": true, 934 | "pg": true, 935 | "ph": true, 936 | "pharmacy": true, 937 | "phd": true, 938 | "philips": true, 939 | "phone": true, 940 | "photo": true, 941 | "photography": true, 942 | "photos": true, 943 | "physio": true, 944 | "pics": true, 945 | "pictet": true, 946 | "pictures": true, 947 | "pid": true, 948 | "pin": true, 949 | "ping": true, 950 | "pink": true, 951 | "pioneer": true, 952 | "pizza": true, 953 | "pk": true, 954 | "pl": true, 955 | "place": true, 956 | "play": true, 957 | "playstation": true, 958 | "plumbing": true, 959 | "plus": true, 960 | "pm": true, 961 | "pn": true, 962 | "pnc": true, 963 | "pohl": true, 964 | "poker": true, 965 | "politie": true, 966 | "porn": true, 967 | "post": true, 968 | "pr": true, 969 | "pramerica": true, 970 | "praxi": true, 971 | "press": true, 972 | "prime": true, 973 | "pro": true, 974 | "prod": true, 975 | "productions": true, 976 | "prof": true, 977 | "progressive": true, 978 | "promo": true, 979 | "properties": true, 980 | "property": true, 981 | "protection": true, 982 | "pru": true, 983 | "prudential": true, 984 | "ps": true, 985 | "pt": true, 986 | "pub": true, 987 | "pw": true, 988 | "pwc": true, 989 | "py": true, 990 | "qa": true, 991 | "qpon": true, 992 | "quebec": true, 993 | "quest": true, 994 | "qvc": true, 995 | "racing": true, 996 | "radio": true, 997 | "raid": true, 998 | "re": true, 999 | "read": true, 1000 | "realestate": true, 1001 | "realtor": true, 1002 | "realty": true, 1003 | "recipes": true, 1004 | "red": true, 1005 | "redstone": true, 1006 | "redumbrella": true, 1007 | "rehab": true, 1008 | "reise": true, 1009 | "reisen": true, 1010 | "reit": true, 1011 | "reliance": true, 1012 | "ren": true, 1013 | "rent": true, 1014 | "rentals": true, 1015 | "repair": true, 1016 | "report": true, 1017 | "republican": true, 1018 | "rest": true, 1019 | "restaurant": true, 1020 | "review": true, 1021 | "reviews": true, 1022 | "rexroth": true, 1023 | "rich": true, 1024 | "richardli": true, 1025 | "ricoh": true, 1026 | "rightathome": true, 1027 | "ril": true, 1028 | "rio": true, 1029 | "rip": true, 1030 | "rmit": true, 1031 | "ro": true, 1032 | "rocher": true, 1033 | "rocks": true, 1034 | "rodeo": true, 1035 | "rogers": true, 1036 | "room": true, 1037 | "rs": true, 1038 | "rsvp": true, 1039 | "ru": true, 1040 | "rugby": true, 1041 | "ruhr": true, 1042 | "run": true, 1043 | "rw": true, 1044 | "rwe": true, 1045 | "ryukyu": true, 1046 | "sa": true, 1047 | "saarland": true, 1048 | "safe": true, 1049 | "safety": true, 1050 | "sakura": true, 1051 | "sale": true, 1052 | "salon": true, 1053 | "samsclub": true, 1054 | "samsung": true, 1055 | "sandvik": true, 1056 | "sandvikcoromant": true, 1057 | "sanofi": true, 1058 | "sap": true, 1059 | "sarl": true, 1060 | "sas": true, 1061 | "save": true, 1062 | "saxo": true, 1063 | "sb": true, 1064 | "sbi": true, 1065 | "sbs": true, 1066 | "sc": true, 1067 | "sca": true, 1068 | "scb": true, 1069 | "schaeffler": true, 1070 | "schmidt": true, 1071 | "scholarships": true, 1072 | "school": true, 1073 | "schule": true, 1074 | "schwarz": true, 1075 | "science": true, 1076 | "scjohnson": true, 1077 | "scor": true, 1078 | "scot": true, 1079 | "sd": true, 1080 | "se": true, 1081 | "search": true, 1082 | "seat": true, 1083 | "secure": true, 1084 | "security": true, 1085 | "seek": true, 1086 | "select": true, 1087 | "sener": true, 1088 | "services": true, 1089 | "ses": true, 1090 | "seven": true, 1091 | "sew": true, 1092 | "sex": true, 1093 | "sexy": true, 1094 | "sfr": true, 1095 | "sg": true, 1096 | "sh": true, 1097 | "shangrila": true, 1098 | "sharp": true, 1099 | "shaw": true, 1100 | "shell": true, 1101 | "shia": true, 1102 | "shiksha": true, 1103 | "shoes": true, 1104 | "shop": true, 1105 | "shopping": true, 1106 | "shouji": true, 1107 | "show": true, 1108 | "showtime": true, 1109 | "shriram": true, 1110 | "si": true, 1111 | "silk": true, 1112 | "sina": true, 1113 | "singles": true, 1114 | "site": true, 1115 | "sj": true, 1116 | "sk": true, 1117 | "ski": true, 1118 | "skin": true, 1119 | "sky": true, 1120 | "skype": true, 1121 | "sl": true, 1122 | "sling": true, 1123 | "sm": true, 1124 | "smart": true, 1125 | "smile": true, 1126 | "sn": true, 1127 | "sncf": true, 1128 | "so": true, 1129 | "soccer": true, 1130 | "social": true, 1131 | "softbank": true, 1132 | "software": true, 1133 | "sohu": true, 1134 | "solar": true, 1135 | "solutions": true, 1136 | "song": true, 1137 | "sony": true, 1138 | "soy": true, 1139 | "space": true, 1140 | "sport": true, 1141 | "spot": true, 1142 | "spreadbetting": true, 1143 | "sr": true, 1144 | "srl": true, 1145 | "ss": true, 1146 | "st": true, 1147 | "stada": true, 1148 | "staples": true, 1149 | "star": true, 1150 | "statebank": true, 1151 | "statefarm": true, 1152 | "stc": true, 1153 | "stcgroup": true, 1154 | "stockholm": true, 1155 | "storage": true, 1156 | "store": true, 1157 | "stream": true, 1158 | "studio": true, 1159 | "study": true, 1160 | "style": true, 1161 | "su": true, 1162 | "sucks": true, 1163 | "supplies": true, 1164 | "supply": true, 1165 | "support": true, 1166 | "surf": true, 1167 | "surgery": true, 1168 | "suzuki": true, 1169 | "sv": true, 1170 | "swatch": true, 1171 | "swiftcover": true, 1172 | "swiss": true, 1173 | "sx": true, 1174 | "sy": true, 1175 | "sydney": true, 1176 | "symantec": true, 1177 | "systems": true, 1178 | "sz": true, 1179 | "tab": true, 1180 | "taipei": true, 1181 | "talk": true, 1182 | "taobao": true, 1183 | "target": true, 1184 | "tatamotors": true, 1185 | "tatar": true, 1186 | "tattoo": true, 1187 | "tax": true, 1188 | "taxi": true, 1189 | "tc": true, 1190 | "tci": true, 1191 | "td": true, 1192 | "tdk": true, 1193 | "team": true, 1194 | "tech": true, 1195 | "technology": true, 1196 | "tel": true, 1197 | "temasek": true, 1198 | "tennis": true, 1199 | "teva": true, 1200 | "tf": true, 1201 | "tg": true, 1202 | "th": true, 1203 | "thd": true, 1204 | "theater": true, 1205 | "theatre": true, 1206 | "tiaa": true, 1207 | "tickets": true, 1208 | "tienda": true, 1209 | "tiffany": true, 1210 | "tips": true, 1211 | "tires": true, 1212 | "tirol": true, 1213 | "tj": true, 1214 | "tjmaxx": true, 1215 | "tjx": true, 1216 | "tk": true, 1217 | "tkmaxx": true, 1218 | "tl": true, 1219 | "tm": true, 1220 | "tmall": true, 1221 | "tn": true, 1222 | "to": true, 1223 | "today": true, 1224 | "tokyo": true, 1225 | "tools": true, 1226 | "top": true, 1227 | "toray": true, 1228 | "toshiba": true, 1229 | "total": true, 1230 | "tours": true, 1231 | "town": true, 1232 | "toyota": true, 1233 | "toys": true, 1234 | "tr": true, 1235 | "trade": true, 1236 | "trading": true, 1237 | "training": true, 1238 | "travel": true, 1239 | "travelchannel": true, 1240 | "travelers": true, 1241 | "travelersinsurance": true, 1242 | "trust": true, 1243 | "trv": true, 1244 | "tt": true, 1245 | "tube": true, 1246 | "tui": true, 1247 | "tunes": true, 1248 | "tushu": true, 1249 | "tv": true, 1250 | "tvs": true, 1251 | "tw": true, 1252 | "tz": true, 1253 | "ua": true, 1254 | "ubank": true, 1255 | "ubs": true, 1256 | "ug": true, 1257 | "uk": true, 1258 | "unicom": true, 1259 | "university": true, 1260 | "uno": true, 1261 | "uol": true, 1262 | "ups": true, 1263 | "us": true, 1264 | "uy": true, 1265 | "uz": true, 1266 | "va": true, 1267 | "vacations": true, 1268 | "vana": true, 1269 | "vanguard": true, 1270 | "vc": true, 1271 | "ve": true, 1272 | "vegas": true, 1273 | "ventures": true, 1274 | "verisign": true, 1275 | "versicherung": true, 1276 | "vet": true, 1277 | "vg": true, 1278 | "vi": true, 1279 | "viajes": true, 1280 | "video": true, 1281 | "vig": true, 1282 | "viking": true, 1283 | "villas": true, 1284 | "vin": true, 1285 | "vip": true, 1286 | "virgin": true, 1287 | "visa": true, 1288 | "vision": true, 1289 | "viva": true, 1290 | "vivo": true, 1291 | "vlaanderen": true, 1292 | "vn": true, 1293 | "vodka": true, 1294 | "volkswagen": true, 1295 | "volvo": true, 1296 | "vote": true, 1297 | "voting": true, 1298 | "voto": true, 1299 | "voyage": true, 1300 | "vu": true, 1301 | "vuelos": true, 1302 | "wales": true, 1303 | "walmart": true, 1304 | "walter": true, 1305 | "wang": true, 1306 | "wanggou": true, 1307 | "watch": true, 1308 | "watches": true, 1309 | "weather": true, 1310 | "weatherchannel": true, 1311 | "webcam": true, 1312 | "weber": true, 1313 | "website": true, 1314 | "wed": true, 1315 | "wedding": true, 1316 | "weibo": true, 1317 | "weir": true, 1318 | "wf": true, 1319 | "whoswho": true, 1320 | "wien": true, 1321 | "wiki": true, 1322 | "williamhill": true, 1323 | "win": true, 1324 | "windows": true, 1325 | "wine": true, 1326 | "winners": true, 1327 | "wme": true, 1328 | "wolterskluwer": true, 1329 | "woodside": true, 1330 | "work": true, 1331 | "works": true, 1332 | "world": true, 1333 | "wow": true, 1334 | "ws": true, 1335 | "wtc": true, 1336 | "wtf": true, 1337 | "xbox": true, 1338 | "xerox": true, 1339 | "xfinity": true, 1340 | "xihuan": true, 1341 | "xin": true, 1342 | "xn--11b4c3d": true, 1343 | "xn--1ck2e1b": true, 1344 | "xn--1qqw23a": true, 1345 | "xn--2scrj9c": true, 1346 | "xn--30rr7y": true, 1347 | "xn--3bst00m": true, 1348 | "xn--3ds443g": true, 1349 | "xn--3e0b707e": true, 1350 | "xn--3hcrj9c": true, 1351 | "xn--3oq18vl8pn36a": true, 1352 | "xn--3pxu8k": true, 1353 | "xn--42c2d9a": true, 1354 | "xn--45br5cyl": true, 1355 | "xn--45brj9c": true, 1356 | "xn--45q11c": true, 1357 | "xn--4gbrim": true, 1358 | "xn--54b7fta0cc": true, 1359 | "xn--55qw42g": true, 1360 | "xn--55qx5d": true, 1361 | "xn--5su34j936bgsg": true, 1362 | "xn--5tzm5g": true, 1363 | "xn--6frz82g": true, 1364 | "xn--6qq986b3xl": true, 1365 | "xn--80adxhks": true, 1366 | "xn--80ao21a": true, 1367 | "xn--80aqecdr1a": true, 1368 | "xn--80asehdb": true, 1369 | "xn--80aswg": true, 1370 | "xn--8y0a063a": true, 1371 | "xn--90a3ac": true, 1372 | "xn--90ae": true, 1373 | "xn--90ais": true, 1374 | "xn--9dbq2a": true, 1375 | "xn--9et52u": true, 1376 | "xn--9krt00a": true, 1377 | "xn--b4w605ferd": true, 1378 | "xn--bck1b9a5dre4c": true, 1379 | "xn--c1avg": true, 1380 | "xn--c2br7g": true, 1381 | "xn--cck2b3b": true, 1382 | "xn--cg4bki": true, 1383 | "xn--clchc0ea0b2g2a9gcd": true, 1384 | "xn--czr694b": true, 1385 | "xn--czrs0t": true, 1386 | "xn--czru2d": true, 1387 | "xn--d1acj3b": true, 1388 | "xn--d1alf": true, 1389 | "xn--e1a4c": true, 1390 | "xn--eckvdtc9d": true, 1391 | "xn--efvy88h": true, 1392 | "xn--fct429k": true, 1393 | "xn--fhbei": true, 1394 | "xn--fiq228c5hs": true, 1395 | "xn--fiq64b": true, 1396 | "xn--fiqs8s": true, 1397 | "xn--fiqz9s": true, 1398 | "xn--fjq720a": true, 1399 | "xn--flw351e": true, 1400 | "xn--fpcrj9c3d": true, 1401 | "xn--fzc2c9e2c": true, 1402 | "xn--fzys8d69uvgm": true, 1403 | "xn--g2xx48c": true, 1404 | "xn--gckr3f0f": true, 1405 | "xn--gecrj9c": true, 1406 | "xn--gk3at1e": true, 1407 | "xn--h2breg3eve": true, 1408 | "xn--h2brj9c": true, 1409 | "xn--h2brj9c8c": true, 1410 | "xn--hxt814e": true, 1411 | "xn--i1b6b1a6a2e": true, 1412 | "xn--imr513n": true, 1413 | "xn--io0a7i": true, 1414 | "xn--j1aef": true, 1415 | "xn--j1amh": true, 1416 | "xn--j6w193g": true, 1417 | "xn--jlq61u9w7b": true, 1418 | "xn--jvr189m": true, 1419 | "xn--kcrx77d1x4a": true, 1420 | "xn--kprw13d": true, 1421 | "xn--kpry57d": true, 1422 | "xn--kpu716f": true, 1423 | "xn--kput3i": true, 1424 | "xn--l1acc": true, 1425 | "xn--lgbbat1ad8j": true, 1426 | "xn--mgb9awbf": true, 1427 | "xn--mgba3a3ejt": true, 1428 | "xn--mgba3a4f16a": true, 1429 | "xn--mgba7c0bbn0a": true, 1430 | "xn--mgbaakc7dvf": true, 1431 | "xn--mgbaam7a8h": true, 1432 | "xn--mgbab2bd": true, 1433 | "xn--mgbah1a3hjkrd": true, 1434 | "xn--mgbai9azgqp6j": true, 1435 | "xn--mgbayh7gpa": true, 1436 | "xn--mgbbh1a": true, 1437 | "xn--mgbbh1a71e": true, 1438 | "xn--mgbc0a9azcg": true, 1439 | "xn--mgbca7dzdo": true, 1440 | "xn--mgbcpq6gpa1a": true, 1441 | "xn--mgberp4a5d4ar": true, 1442 | "xn--mgbgu82a": true, 1443 | "xn--mgbi4ecexp": true, 1444 | "xn--mgbpl2fh": true, 1445 | "xn--mgbt3dhd": true, 1446 | "xn--mgbtx2b": true, 1447 | "xn--mgbx4cd0ab": true, 1448 | "xn--mix891f": true, 1449 | "xn--mk1bu44c": true, 1450 | "xn--mxtq1m": true, 1451 | "xn--ngbc5azd": true, 1452 | "xn--ngbe9e0a": true, 1453 | "xn--ngbrx": true, 1454 | "xn--node": true, 1455 | "xn--nqv7f": true, 1456 | "xn--nqv7fs00ema": true, 1457 | "xn--nyqy26a": true, 1458 | "xn--o3cw4h": true, 1459 | "xn--ogbpf8fl": true, 1460 | "xn--otu796d": true, 1461 | "xn--p1acf": true, 1462 | "xn--p1ai": true, 1463 | "xn--pbt977c": true, 1464 | "xn--pgbs0dh": true, 1465 | "xn--pssy2u": true, 1466 | "xn--q7ce6a": true, 1467 | "xn--q9jyb4c": true, 1468 | "xn--qcka1pmc": true, 1469 | "xn--qxa6a": true, 1470 | "xn--qxam": true, 1471 | "xn--rhqv96g": true, 1472 | "xn--rovu88b": true, 1473 | "xn--rvc1e0am3e": true, 1474 | "xn--s9brj9c": true, 1475 | "xn--ses554g": true, 1476 | "xn--t60b56a": true, 1477 | "xn--tckwe": true, 1478 | "xn--tiq49xqyj": true, 1479 | "xn--unup4y": true, 1480 | "xn--vermgensberater-ctb": true, 1481 | "xn--vermgensberatung-pwb": true, 1482 | "xn--vhquv": true, 1483 | "xn--vuq861b": true, 1484 | "xn--w4r85el8fhu5dnra": true, 1485 | "xn--w4rs40l": true, 1486 | "xn--wgbh1c": true, 1487 | "xn--wgbl6a": true, 1488 | "xn--xhq521b": true, 1489 | "xn--xkc2al3hye2a": true, 1490 | "xn--xkc2dl3a5ee0h": true, 1491 | "xn--y9a3aq": true, 1492 | "xn--yfro4i67o": true, 1493 | "xn--ygbi2ammx": true, 1494 | "xn--zfr164b": true, 1495 | "xxx": true, 1496 | "xyz": true, 1497 | "yachts": true, 1498 | "yahoo": true, 1499 | "yamaxun": true, 1500 | "yandex": true, 1501 | "ye": true, 1502 | "yodobashi": true, 1503 | "yoga": true, 1504 | "yokohama": true, 1505 | "you": true, 1506 | "youtube": true, 1507 | "yt": true, 1508 | "yun": true, 1509 | "za": true, 1510 | "zappos": true, 1511 | "zara": true, 1512 | "zero": true, 1513 | "zip": true, 1514 | "zm": true, 1515 | "zone": true, 1516 | "zuerich": true, 1517 | "zw": true, 1518 | "onion": true, 1519 | } 1520 | --------------------------------------------------------------------------------