├── .github └── workflows │ ├── codeql-analysis.yml │ ├── main.yml │ └── release.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── api ├── controllers │ ├── email.go │ ├── error.go │ ├── ip.go │ └── request.go └── transport │ ├── api.go │ └── middlewares │ └── ratelimiter.go ├── browser ├── user_agent.go └── user_agent_test.go ├── cmd └── threatbite │ └── main.go ├── config.env ├── config ├── config.go └── config_test.go ├── email ├── datasource │ ├── datasource.go │ ├── datasource_empty.go │ ├── datasource_list.go │ ├── datasource_test.go │ ├── datasource_url.go │ └── domain.go ├── disposal.go ├── disposal_test.go ├── email.go ├── email_test.go ├── free.go ├── free_test.go ├── net.go └── username.go ├── go.mod ├── go.sum ├── ip ├── datasource │ ├── datasource.go │ ├── datasource_directory.go │ ├── datasource_empty.go │ ├── datasource_list.go │ ├── datasource_test.go │ ├── datasource_url.go │ ├── ipnet.go │ └── ipnet_test.go ├── dc.go ├── dc_test.go ├── geoip.go ├── ip.go ├── ip_test.go ├── maxmind.go ├── net.go ├── proxy.go ├── search_engine.go ├── spam.go ├── tor.go ├── vpn.go └── vpn_test.go └── resources └── static ├── index.html ├── swagger.yml └── threatbite_diagram.png /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | name: "CodeQL" 7 | 8 | on: 9 | push: 10 | branches: [master] 11 | pull_request: 12 | # The branches below must be a subset of the branches above 13 | branches: [master] 14 | schedule: 15 | - cron: '0 12 * * 0' 16 | 17 | jobs: 18 | analyze: 19 | name: Analyze 20 | runs-on: ubuntu-latest 21 | 22 | strategy: 23 | fail-fast: false 24 | matrix: 25 | # Override automatic language detection by changing the below list 26 | # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] 27 | language: ['go'] 28 | # Learn more... 29 | # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection 30 | 31 | steps: 32 | - name: Checkout repository 33 | uses: actions/checkout@v2 34 | with: 35 | # We must fetch at least the immediate parents so that if this is 36 | # a pull request then we can checkout the head. 37 | fetch-depth: 2 38 | 39 | # If this run was triggered by a pull request event, then checkout 40 | # the head of the pull request instead of the merge commit. 41 | - run: git checkout HEAD^2 42 | if: ${{ github.event_name == 'pull_request' }} 43 | 44 | # Initializes the CodeQL tools for scanning. 45 | - name: Initialize CodeQL 46 | uses: github/codeql-action/init@v1 47 | with: 48 | languages: ${{ matrix.language }} 49 | # If you wish to specify custom queries, you can do so here or in a config file. 50 | # By default, queries listed here will override any specified in a config file. 51 | # Prefix the list here with "+" to use these queries and those in the config file. 52 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 53 | 54 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 55 | # If this step fails, then you should remove it and run the build manually (see below) 56 | - name: Autobuild 57 | uses: github/codeql-action/autobuild@v1 58 | 59 | # ℹ️ Command-line programs to run using the OS shell. 60 | # 📚 https://git.io/JvXDl 61 | 62 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 63 | # and modify them (or add more) to build your code if your project 64 | # uses a compiled language 65 | 66 | #- run: | 67 | # make bootstrap 68 | # make release 69 | 70 | - name: Perform CodeQL Analysis 71 | uses: github/codeql-action/analyze@v1 72 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Check & test & build 2 | on: 3 | push: 4 | branches: 5 | - master 6 | pull_request: 7 | 8 | jobs: 9 | check: 10 | name: Quality & security checks 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: Set up Go 14 | uses: actions/setup-go@v1 15 | with: 16 | go-version: 1.13 17 | 18 | - name: Check out code 19 | uses: actions/checkout@v1 20 | 21 | - name: Lint Go Code 22 | run: | 23 | export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14 24 | make check 25 | 26 | test: 27 | name: Test & coverage 28 | runs-on: ubuntu-latest 29 | steps: 30 | - name: Set up Go 31 | uses: actions/setup-go@v1 32 | with: 33 | go-version: 1.13 34 | 35 | - name: Check out code 36 | uses: actions/checkout@v1 37 | 38 | - name: Run unit tests with 39 | run: make test 40 | 41 | build: 42 | name: Build 43 | runs-on: ubuntu-latest 44 | needs: [check, test] 45 | steps: 46 | - name: Set up Go 47 | uses: actions/setup-go@v1 48 | with: 49 | go-version: 1.13 50 | 51 | - name: Check out code 52 | uses: actions/checkout@v1 53 | 54 | - name: Build 55 | run: make build 56 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | push: 4 | tags: 5 | - '*' 6 | 7 | jobs: 8 | release: 9 | name: Publish to github releases page 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Set up Go 13 | uses: actions/setup-go@v1 14 | with: 15 | go-version: 1.13 16 | 17 | - name: Check out code 18 | uses: actions/checkout@v1 19 | 20 | - name: Release 21 | env: 22 | GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} 23 | run: | 24 | export PATH=$PATH:$(go env GOPATH)/bin # temporary fix. See https://github.com/actions/setup-go/issues/14 25 | 26 | VERSION=$(git describe --abbrev=0 --tags) 27 | UNAME_SYS=$(uname -s) 28 | UNAME_HW=$(uname -m) 29 | TAR_THREATBITE=threatbite_${UNAME_SYS}_${UNAME_HW}.tar.gz 30 | CHANGELOG=$(git log --oneline $(git describe --tags --abbrev=0 @^)..@) 31 | 32 | make build 33 | tar -cvzf ./bin/${TAR_THREATBITE} -C ./bin ./threatbite 34 | 35 | go get github.com/tcnksm/ghr 36 | ghr -t ${GITHUB_TOKEN} -b "${CHANGELOG}" -delete ${VERSION} ./bin/${TAR_THREATBITE} 37 | 38 | registry: 39 | name: Publish to docker hub 40 | runs-on: ubuntu-latest 41 | steps: 42 | - uses: actions/checkout@master 43 | 44 | - name: Version tag 45 | uses: elgohr/Publish-Docker-Github-Action@master 46 | with: 47 | name: optimatiq/threatbite 48 | username: ${{ secrets.DOCKERHUB_USER }} 49 | password: ${{ secrets.DOCKERHUB_TOKEN }} 50 | 51 | - name: Latest tag 52 | uses: elgohr/Publish-Docker-Github-Action@master 53 | with: 54 | name: optimatiq/threatbite 55 | username: ${{ secrets.DOCKERHUB_USER }} 56 | password: ${{ secrets.DOCKERHUB_TOKEN }} 57 | tag_names: true -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | bin/ 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | *.out 9 | 10 | # Test binary, build with `go test -c` 11 | *.test 12 | 13 | # GoLand 14 | .idea/ 15 | 16 | # vs-code 17 | .vscode/ 18 | 19 | # vendor 20 | vendor/ 21 | 22 | # local config file 23 | config_local.env 24 | 25 | # generated dynamically 26 | resources/maxmind/ 27 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.13 AS builder 2 | 3 | WORKDIR /app 4 | 5 | COPY go.mod . 6 | COPY go.sum . 7 | RUN go mod download 8 | 9 | COPY . . 10 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make build 11 | 12 | FROM alpine:latest 13 | 14 | RUN apk --no-cache add ca-certificates 15 | WORKDIR /root/ 16 | COPY --from=builder /app/bin . 17 | COPY --from=builder /app/resources ./resources/ 18 | 19 | ENV PORT 8080 20 | 21 | CMD ["./threatbite"] 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Business Source License 1.1 2 | 3 | Parameters 4 | 5 | Licensor: Optimatiq Sp. z o.o. 6 | Licensed Work: Threatbite 7 | The Licensed Work is (c) 2020 Optimatiq Sp. z o.o. 8 | Additional Use Grant: You may make use of the Licensed Work, provided that you do 9 | not use the Licensed Work for a "Reputation Service". 10 | 11 | A "Reputation Service" is a commercial offering 12 | that allows third parties (other than your employees and 13 | contractors) to access the functionality of the Licensed 14 | Work so that such third parties directly benefit from the 15 | checking the reputation of a given target using the 16 | Licensed Work. 17 | 18 | Change Date: 2023-01-01 (v1.0). For other versions change date is four years from release date. 19 | 20 | Change License: Apache License, Version 2.0 21 | 22 | For information about alternative licensing arrangements for the Software, 23 | please visit: https://optimatiq.com/threatbite/ 24 | 25 | Notice 26 | 27 | The Business Source License (this document, or the "License") is not an Open 28 | Source license. However, the Licensed Work will eventually be made available 29 | under an Open Source License, as stated in this License. 30 | 31 | License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. 32 | "Business Source License" is a trademark of MariaDB Corporation Ab. 33 | 34 | ----------------------------------------------------------------------------- 35 | 36 | Business Source License 1.1 37 | 38 | Terms 39 | 40 | The Licensor hereby grants you the right to copy, modify, create derivative 41 | works, redistribute, and make non-production use of the Licensed Work. The 42 | Licensor may make an Additional Use Grant, above, permitting limited 43 | production use. 44 | 45 | Effective on the Change Date, or the fourth anniversary of the first publicly 46 | available distribution of a specific version of the Licensed Work under this 47 | License, whichever comes first, the Licensor hereby grants you rights under 48 | the terms of the Change License, and the rights granted in the paragraph 49 | above terminate. 50 | 51 | If your use of the Licensed Work does not comply with the requirements 52 | currently in effect as described in this License, you must purchase a 53 | commercial license from the Licensor, its affiliated entities, or authorized 54 | resellers, or you must refrain from using the Licensed Work. 55 | 56 | All copies of the original and modified Licensed Work, and derivative works 57 | of the Licensed Work, are subject to this License. This License applies 58 | separately for each version of the Licensed Work and the Change Date may vary 59 | for each version of the Licensed Work released by Licensor. 60 | 61 | You must conspicuously display this License on each original or modified copy 62 | of the Licensed Work. If you receive the Licensed Work in original or 63 | modified form from a third party, the terms and conditions set forth in this 64 | License apply to your use of that work. 65 | 66 | Any use of the Licensed Work in violation of this License will automatically 67 | terminate your rights under this License for the current and all other 68 | versions of the Licensed Work. 69 | 70 | This License does not grant you any right in any trademark or logo of 71 | Licensor or its affiliates (provided that you may use a trademark or logo of 72 | Licensor as expressly required by this License). 73 | 74 | TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON 75 | AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, 76 | EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF 77 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND 78 | TITLE. 79 | 80 | MariaDB hereby grants you permission to use this License’s text to license 81 | your works, and to refer to it using the trademark "Business Source License", 82 | as long as you comply with the Covenants of Licensor below. 83 | 84 | Covenants of Licensor 85 | 86 | In consideration of the right to use this License’s text and the "Business 87 | Source License" name and trademark, Licensor covenants to MariaDB, and to all 88 | other recipients of the licensed work to be provided by Licensor: 89 | 90 | 1. To specify as the Change License the GPL Version 2.0 or any later version, 91 | or a license that is compatible with GPL Version 2.0 or a later version, 92 | where "compatible" means that software provided under the Change License can 93 | be included in a program with software provided under GPL Version 2.0 or a 94 | later version. Licensor may specify additional Change Licenses without 95 | limitation. 96 | 97 | 2. To either: (a) specify an additional grant of rights to use that does not 98 | impose any additional restriction on the right granted in this License, as 99 | the Additional Use Grant; or (b) insert the text "None". 100 | 101 | 3. To specify a Change Date. 102 | 103 | 4. Not to modify this License in any other way. 104 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | export GO111MODULE=on 3 | export GOPROXY=https://proxy.golang.org 4 | 5 | .DEFAULT_GOAL: all 6 | 7 | GIT_TAG := `git describe --abbrev=0 --tags` 8 | GIT_COMMIT := `git rev-parse HEAD` 9 | 10 | LDFLAGS=-ldflags "-s -w -X=main.date=$(shell date +%FT%T%z) -X=main.tag=$(GIT_TAG) -X=main.commit=$(GIT_COMMIT) " 11 | 12 | .PHONY: build check clean format format-check git-tag-major git-tag-minor git-tag-patch help test tidy 13 | 14 | all: check test build ## Default target: check, test, build, 15 | 16 | build: ## Build all excecutables, located under ./bin/ 17 | @echo "[threatbite] Building..." 18 | @go build -trimpath -o ./bin/threatbite $(LDFLAGS) cmd/threatbite/main.go 19 | 20 | clean: ## Remove all artifacts from ./bin/ and ./resources 21 | @rm -rf ./bin/* 22 | 23 | format: ## Format go code with goimports 24 | @go get golang.org/x/tools/cmd/goimports 25 | @goimports -l -w . 26 | 27 | format-check: ## Check if the code is formatted 28 | @go get golang.org/x/tools/cmd/goimports 29 | @for i in $$(goimports -l .); do echo "[ERROR] Code is not formated run 'make format'" && exit 1; done 30 | 31 | test: ## Run tests 32 | @go test -race ./... 33 | 34 | tidy: ## Run go mod tidy 35 | @go mod tidy 36 | 37 | check: format-check ## Linting and static analysis 38 | @if grep -r --include='*.go' -E "fmt.Print|spew.Dump" *; then \ 39 | echo "code contains fmt.Print* or spew.Dump function"; \ 40 | exit 1; \ 41 | fi 42 | 43 | @if test ! -e ./bin/golangci-lint; then \ 44 | curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh; \ 45 | fi 46 | @./bin/golangci-lint run --timeout 180s -E gosec -E stylecheck -E golint -E goimports -E whitespace 47 | 48 | git-tag-patch: ## Push new tag to repository with patch number incremented 49 | $(eval NEW_VERSION=$(shell git describe --tags --abbrev=0 | awk -F'[a-z.]' '{$$4++;print "v" $$2 "." $$3 "." $$4}')) 50 | @echo Version: $(NEW_VERSION) 51 | @git tag -a $(NEW_VERSION) -m "new patch release" 52 | @git push origin $(NEW_VERSION) 53 | 54 | git-tag-minor: ## Push new tag to repository with minor number incremented 55 | $(eval NEW_VERSION=$(shell git describe --tags --abbrev=0 | awk -F'[a-z.]' '{$$3++;print "v" $$2 "." $$3 "." 0}')) 56 | @echo Version: $(NEW_VERSION) 57 | @git tag -a $(NEW_VERSION) -m "new minor release" 58 | @git push origin $(NEW_VERSION) 59 | 60 | git-tag-major: ## Push new tag to repository with major number incremented 61 | $(eval NEW_VERSION=$(shell git describe --tags --abbrev=0 | awk -F'[a-z.]' '{$$2++;print "v" $$2 "." 0 "." 0}')) 62 | @echo Version: $(NEW_VERSION) 63 | @git tag -a $(NEW_VERSION) -m "new major release" 64 | @git push origin $(NEW_VERSION) 65 | 66 | help: ## Show help 67 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 68 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ThreatBite - reputation checking tool 2 | 3 | ![GithubActions](https://github.com/optimatiq/threatbite/workflows/Check%20&%20test%20&%20build/badge.svg) 4 | [![Go Report Card](https://goreportcard.com/badge/github.com/optimatiq/threatbite)](https://goreportcard.com/report/github.com/optimatiq/threatbite) 5 | ![Version](https://img.shields.io/github/v/tag/optimatiq/threatbite) 6 | 7 | # About 8 | ThreatBite is a global reputation system used internally by [Optimatiq](https://optimatiq.com). It analyses incoming internet users and sends back a standardized score representing their trustworthiness. The program uses independent databases that collect up-to-date information about bad internet actors. Because it is an API service, it works in real-time, allowing you to act fast and block or keep an eye on suspicious user behavior. 9 | 10 | # Architecture 11 | ![Architecture](resources/static/threatbite_diagram.png) 12 | 13 | # Features 14 | 15 | ### Identifying the source of threat 16 | ThreatBite identifies potential sources of fraud by comparing user identification data to over 500 databases of bad internet actors. 17 | 18 | ### Account creation protection 19 | ThreatBite protects against automatic account creation and user account hijacking. 20 | 21 | ### Spam detection 22 | ThreatBite Identifies potential sources of spammers. 23 | 24 | ### Tor users detection 25 | ThreatBite recognizes addresses belonging to the Tor network. 26 | 27 | ### Proxy/VPN 28 | ThreatBite detects addresses that are used as proxys or VPNs. 29 | 30 | ## Download 31 | - Grab the latest binary from the [releases](https://github.com/optimatiq/threatbite/releases) page and run it: 32 | 33 | ```shell 34 | ./threatbite 35 | ``` 36 | - Or use the official Docker image: 37 | 38 | ```shell 39 | docker run -d -p 8080:8080 optimatiq/threatbite 40 | ``` 41 | 42 | - Or get the sources: 43 | 44 | ```shell 45 | git clone https://github.com/optimatiq/threatbite 46 | cd ./threatbite 47 | make build && ./bin/threatbite 48 | ``` 49 | 50 | ## Quickstart 51 | 52 | ### Scoring for email 53 | `curl localhost:8080/v1/score/email/noreply@o2.pl` 54 | 55 | ### Scoring for IP address 56 | `curl localhost:8080/v1/score/ip/1.1.1.1` 57 | 58 | ### Scoring for HTTP request 59 | 60 | ``` 61 | curl \ 62 | -X POST \ 63 | localhost:8080/v1/score/request \ 64 | -H 'Content-Type: application/json' \ 65 | -d '{"ip":"1.2.3.4", "host":"host.pl", "uri":"/", "method":"GET", "user_agent":"curl", "headers": {"x-header": 1}}' 66 | ``` 67 | 68 | or 69 | 70 | ``` 71 | curl \ 72 | -X POST \ 73 | localhost:8080/v1/score/request \ 74 | -d 'ip=1.2.3.4' \ 75 | -d 'host=host.pl' \ 76 | -d 'uri=/' \ 77 | -d 'method=POST' \ 78 | -d 'user_agent=curl' 79 | ``` 80 | ### API documentation 81 | `chrome localhost:8080` 82 | 83 | ### Rate limits 84 | 10 requests per seconds are allowed, after reaching limit 429 HTTP status code is returned 85 | 86 | ### Configuration 87 | Configuration is done via env variables or config.env file. All parameters are optional: 88 | * `PORT` - API listening port default 8080 89 | * `DEBUG` - values: false, true, 1, 0 or empty 90 | * `AUTO_TLS` - values: false, true, 1, 0 or empty, automatic access to certificates from Let's Encrypt 91 | 92 | License keys for these external services will improve the quality of the results. It is highly recommended to set them. 93 | * `PWNED_KEY` - obtained from https://haveibeenpwned.com/ 94 | * `MAXMIND_KEY` - obtained from https://www.maxmind.com/en/accounts/current/license-key 95 | 96 | Correct communication with the MTA server requires the following settings. Otherwise, the server may close the connection with the error: 97 | * `SMTP_HELLO` - the domain name or IP address of the SMTP client that will be provided as an argument to the HELO command 98 | * `SMTP_FROM` - MAIL FROM value passed to the SMTP server 99 | 100 | IP/CIDR lists contain information about addresses used as proxy/VPN or other malicious activity. 101 | You can provide one or many sources separated by whitespace. 102 | The format of the data is straightforward, and each line contains one IP or CIDR addresses. 103 | Threadbite open-source version provides public sources that are limited in scope and might be outdated with no SLA. 104 | If you interested in curated and more accurate lists with SLA, please contact us at threatbite@optimatiq.com 105 | 106 | * `PROXY_LIST` - URL or set of URLs separated by space, default: https://get.threatbite.com/public/proxy.txt 107 | * `SPAM_LIST` - URL or set of URLs separated by space, default: https://get.threatbite.com/public/spam.txt 108 | * `VPN_LIST` - URL or set of URLs separated by space, default: https://get.threatbite.com/public/vpn.txt 109 | * `DC_LIST` - URL or set of URLs separated by space, default: https://get.threatbite.com/public/dc-names.txt 110 | 111 | Email lists contain information about domains used as disposal emails or free solutions which are often used in spam or phishing campaigns. 112 | You can provide one or many sources separated by whitespace. 113 | The format of the data is straightforward, and each line contains one domain 114 | Threadbite open-source version provides public sources that are limited in scope and might be outdated with no SLA. 115 | If you interested in curated and more accurate lists with SLA, please contact us at threatbite@optimatiq.com 116 | 117 | * `EMAIL_DISPOSAL_LIST` - URL or set of URLs separated by space, default: https://get.threatbite.com/public/disposal.txt 118 | * `EMAIL_FREE_LIST ` - URL or set of URLs separated by space, default: https://get.threatbite.com/public/free.txt 119 | 120 | ### config.env file 121 | You can store your custom configuration in config.env. The format is defined as below: 122 | 123 | ``` 124 | DEBUG=true 125 | PORT=443 126 | AUTO_TLS=true 127 | PROXY_LIST=https://provider1.com https://provider2.com 128 | ``` 129 | 130 | By default threatbite binary is looking for config.env file in the same directory, 131 | but you can use `-config` flag to change this and point to any file in a filesystem. 132 | 133 | `./bin/threatbite -confg=/etc/threatbite/config.env` 134 | 135 | ## Development 136 | 137 | ### Go 138 | At least version 1.13 is required 139 | 140 | ### Building & running 141 | `make bulid && ./bin/threatbite` 142 | 143 | ### Run tests: 144 | `make test` 145 | 146 | ### Quality & linteners: 147 | `make check` 148 | 149 | ### Other targets 150 | `make help` 151 | 152 | ### Internal endpoints 153 | `/internal/*` endpoints should not be public, they contains sensitive data. 154 | 155 | ### Health check 156 | `/internal/health` 157 | 158 | ### Monitoring 159 | Prometheus endpoint is available at: `/internal/metrics` 160 | 161 | ### Profiling 162 | `go tool pprof localhost:8080/internal/debug/pprof/profile?seconds=20` 163 | 164 | `go tool pprof localhost:8080/internal/debug/pprof/heap` 165 | -------------------------------------------------------------------------------- /api/controllers/email.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | 7 | lru "github.com/hashicorp/golang-lru" 8 | "github.com/optimatiq/threatbite/email" 9 | ) 10 | 11 | // EmailResult response object, which contains detailed information returned from Check method. 12 | type EmailResult struct { 13 | Scoring uint8 `json:"scoring"` 14 | AccountExists bool `json:"exists"` 15 | CatchAll bool `json:"catchall"` 16 | DefaultUser bool `json:"default"` 17 | Disposal bool `json:"disposal"` 18 | Free bool `json:"free"` 19 | Leaked bool `json:"leaked"` 20 | Valid bool `json:"valid"` 21 | } 22 | 23 | // Email is a controller container with the cache. 24 | type Email struct { 25 | cache *lru.Cache 26 | emailInfo *email.Email 27 | } 28 | 29 | // NewEmail creates an Email scoring controller. 30 | func NewEmail(emailInfo *email.Email) (*Email, error) { 31 | cache, err := lru.New(4096) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | return &Email{ 37 | cache: cache, 38 | emailInfo: emailInfo, 39 | }, nil 40 | } 41 | 42 | var reEmail = regexp.MustCompile("(?i)^[a-z0-9!#$%&'*+/i=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$") 43 | 44 | // Validate returns nil if provided email is valid, 45 | // otherwise, returns error, which can be presented to the user. 46 | func (e *Email) Validate(email string) error { 47 | if !reEmail.MatchString(email) { 48 | return ErrInvalidEmail 49 | } 50 | return nil 51 | } 52 | 53 | // Check is the main module functions, which is used to perform all checks for given argument. 54 | func (e *Email) Check(address string) (*EmailResult, error) { 55 | if len(strings.Split(address, "@")) != 2 { 56 | return nil, ErrInvalidEmail 57 | } 58 | 59 | if v, ok := e.cache.Get(address); ok { 60 | return v.(*EmailResult), nil 61 | } 62 | 63 | info := e.emailInfo.GetInfo(address) 64 | 65 | result := &EmailResult{ 66 | Scoring: info.EmailScoring, 67 | AccountExists: info.IsExistingAccount, 68 | CatchAll: info.IsCatchAll, 69 | DefaultUser: info.IsDefaultUser, 70 | Disposal: info.IsDisposal, 71 | Free: info.IsFree, 72 | Leaked: info.IsLeaked, 73 | Valid: info.IsValid, 74 | } 75 | 76 | if !e.cache.Contains(address) { 77 | e.cache.Add(address, result) 78 | } 79 | 80 | return result, nil 81 | } 82 | -------------------------------------------------------------------------------- /api/controllers/error.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | // ErrInvalidIP indicates that IP address is invalid 8 | var ErrInvalidIP = errors.New("invalid IP address") 9 | 10 | // ErrInvalidEmail indicates that email is invalid 11 | var ErrInvalidEmail = errors.New("invalid email") 12 | -------------------------------------------------------------------------------- /api/controllers/ip.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "net" 5 | 6 | lru "github.com/hashicorp/golang-lru" 7 | "github.com/optimatiq/threatbite/ip" 8 | ) 9 | 10 | // IPResult response object, which contains detailed information returned from Check method. 11 | type IPResult struct { 12 | Scoring uint8 `json:"scoring"` 13 | Action string `json:"action"` 14 | Company string `json:"company"` 15 | Country string `json:"country"` 16 | BadReputation bool `json:"bad"` 17 | Bot bool `json:"bot"` 18 | Datacenter bool `json:"dc"` 19 | Private bool `json:"private"` 20 | Proxy bool `json:"proxy"` 21 | SearchEngine bool `json:"se"` 22 | Spam bool `json:"spam"` 23 | Tor bool `json:"tor"` 24 | Vpn bool `json:"vpn"` 25 | } 26 | 27 | // IP a container for IP controller. 28 | type IP struct { 29 | ipinfo *ip.IP 30 | cache *lru.Cache 31 | } 32 | 33 | // NewIP creates new IP scoring controller. 34 | func NewIP(ipinfo *ip.IP) (*IP, error) { 35 | cache, err := lru.New(4096) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | ip := &IP{ 41 | cache: cache, 42 | ipinfo: ipinfo, 43 | } 44 | 45 | return ip, nil 46 | } 47 | 48 | // Validate returns nil if provided IP address is valid, 49 | // otherwise, returns error, which can be presented to the user. 50 | func (i *IP) Validate(addr string) error { 51 | if net.ParseIP(addr) == nil { 52 | return ErrInvalidIP 53 | } 54 | return nil 55 | } 56 | 57 | // Check is the main module functions, which is used to perform all checks for given argument. 58 | func (i *IP) Check(addr string) (*IPResult, error) { 59 | ip := net.ParseIP(addr) 60 | if ip == nil { 61 | return nil, ErrInvalidIP 62 | } 63 | 64 | if v, ok := i.cache.Get(addr); ok { 65 | return v.(*IPResult), nil 66 | } 67 | 68 | info, err := i.ipinfo.GetInfo(ip) 69 | if err != nil { 70 | return nil, err 71 | } 72 | 73 | result := &IPResult{ 74 | Scoring: info.IPScoring, 75 | Country: info.Country, 76 | Company: info.Company, 77 | Tor: info.IsTor, 78 | Proxy: info.IsProxy, 79 | SearchEngine: info.IsSearchEngine, 80 | Private: info.IsPrivate, 81 | Spam: info.IsSpam, 82 | Datacenter: info.IsDatacenter, 83 | Vpn: info.IsVpn, 84 | } 85 | 86 | if !i.cache.Contains(addr) { 87 | i.cache.Add(addr, result) 88 | } 89 | 90 | return result, nil 91 | } 92 | -------------------------------------------------------------------------------- /api/controllers/request.go: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import ( 4 | "crypto/md5" // #nosec 5 | "encoding/hex" 6 | "encoding/json" 7 | "net" 8 | 9 | "github.com/go-playground/validator" 10 | lru "github.com/hashicorp/golang-lru" 11 | "github.com/optimatiq/threatbite/browser" 12 | "github.com/optimatiq/threatbite/ip" 13 | ) 14 | 15 | // RequestResult response object, which contains detailed information returned from Check method. 16 | type RequestResult struct { 17 | IPResult 18 | browser.UserAgent 19 | Bot bool 20 | Mobile bool 21 | Script bool 22 | } 23 | 24 | // RequestQuery struct, which is used to calculate scoring for given request (based on HTTP values). 25 | // Some fields are required (IP, Host, URI, Method, UserAgent) other are options. 26 | type RequestQuery struct { 27 | // Required fields 28 | IP string `json:"ip" form:"ip" validate:"required,ip"` 29 | Host string `json:"host" form:"host" validate:"required,hostname"` 30 | URI string `json:"uri" form:"uri" validate:"required,uri"` 31 | Method string `json:"method" form:"method" validate:"required,oneof=GET HEAD POST PUT DELETE TRACE OPTIONS PATCH"` 32 | UserAgent string `json:"user_agent" form:"user_agent" validate:"required"` 33 | 34 | // Optional fields 35 | Protocol string `json:"protocol" form:"protocol"` 36 | Scheme string `json:"scheme" form:"scheme" validate:"omitempty,oneof=http https"` 37 | ContentType string `json:"content_type" form:"content_type"` 38 | Headers map[string]string `json:"headers" form:"headers"` 39 | } 40 | 41 | func (r RequestQuery) hash() (string, error) { 42 | key, err := json.Marshal(r) 43 | if err != nil { 44 | return "", err 45 | } 46 | 47 | h := md5.New() // #nosec 48 | _, err = h.Write(key) 49 | if err != nil { 50 | return "", err 51 | } 52 | return hex.EncodeToString(h.Sum(nil)), nil 53 | } 54 | 55 | // Request is a container for HTTP request controller. 56 | type Request struct { 57 | ipinfo *ip.IP 58 | cache *lru.Cache 59 | validator *validator.Validate 60 | } 61 | 62 | // NewRequest creates new HTTP request scoring module. 63 | func NewRequest(ipinfo *ip.IP) (*Request, error) { 64 | cache, err := lru.New(4096) 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | request := &Request{ 70 | validator: validator.New(), 71 | cache: cache, 72 | ipinfo: ipinfo, 73 | } 74 | 75 | return request, nil 76 | } 77 | 78 | // Validate returns nil if provided data is valid, 79 | // otherwise, returns error, which can be presented to the user. 80 | // Validation rules are defined as a struct tags. 81 | func (r *Request) Validate(request RequestQuery) error { 82 | // TODO translate errors, now they are connected with RequestQuery struct fields as internal representation 83 | // - example: 'RequestQuery.IP' Error:Field validation for 'IP' failed on the 'required' tag 84 | return r.validator.Struct(request) 85 | } 86 | 87 | // Check is the main module functions which is used to perform all checks for given argument. 88 | func (r *Request) Check(request RequestQuery) (*RequestResult, error) { 89 | // TODO add business logic 90 | key, err := request.hash() 91 | if err != nil { 92 | return nil, err 93 | } 94 | 95 | if v, ok := r.cache.Get(key); ok { 96 | return v.(*RequestResult), nil 97 | } 98 | 99 | ip := net.ParseIP(request.IP) 100 | if ip == nil { 101 | return nil, ErrInvalidIP 102 | } 103 | 104 | info, err := r.ipinfo.GetInfo(ip) 105 | if err != nil { 106 | return nil, err 107 | } 108 | 109 | result := &RequestResult{ 110 | IPResult: IPResult{ 111 | Country: info.Country, 112 | Tor: info.IsTor, 113 | Proxy: info.IsProxy, 114 | SearchEngine: info.IsSearchEngine, 115 | Private: info.IsPrivate, 116 | Spam: info.IsSpam, 117 | Datacenter: info.IsDatacenter, 118 | }, 119 | UserAgent: *browser.GetUserAgent(request.UserAgent), 120 | Bot: browser.IsBotUserAgent(request.UserAgent), 121 | Mobile: browser.IsMobileUserAgent(request.UserAgent), 122 | Script: browser.IsScriptUserAgent(request.UserAgent), 123 | } 124 | if !r.cache.Contains(key) { 125 | r.cache.Add(key, result) 126 | } 127 | return result, nil 128 | } 129 | -------------------------------------------------------------------------------- /api/transport/api.go: -------------------------------------------------------------------------------- 1 | package transport 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "net/http/pprof" 7 | "net/url" 8 | "time" 9 | 10 | "github.com/labstack/echo-contrib/prometheus" 11 | "github.com/labstack/echo/v4" 12 | "github.com/labstack/echo/v4/middleware" 13 | "github.com/labstack/gommon/log" 14 | "github.com/optimatiq/threatbite/api/controllers" 15 | "github.com/optimatiq/threatbite/api/transport/middlewares" 16 | "github.com/optimatiq/threatbite/config" 17 | "github.com/optimatiq/threatbite/email" 18 | emailDatasource "github.com/optimatiq/threatbite/email/datasource" 19 | "github.com/optimatiq/threatbite/ip" 20 | ipDatasource "github.com/optimatiq/threatbite/ip/datasource" 21 | "golang.org/x/crypto/acme/autocert" 22 | ) 23 | 24 | // API state container 25 | type API struct { 26 | config *config.Config 27 | echo *echo.Echo 28 | controllerEmail *controllers.Email 29 | controllerIP *controllers.IP 30 | controllerRequest *controllers.Request 31 | } 32 | 33 | // NewAPI returns new HTTP server, which is listening on given port 34 | func NewAPI(config *config.Config) (*API, error) { 35 | emailData := email.NewEmail( 36 | config.PwnedKey, 37 | config.SMTPHello, 38 | config.SMTPFrom, 39 | emailDatasource.NewURLDataSource(config.EmailDisposalList), 40 | emailDatasource.NewURLDataSource(config.EmailFreeList), 41 | ) 42 | emailData.RunUpdates() 43 | 44 | emailController, err := controllers.NewEmail(emailData) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | ipdata := ip.NewIP( 50 | config.MaxmindKey, 51 | ipDatasource.NewURLDataSource(config.ProxyList), 52 | ipDatasource.NewURLDataSource(config.SpamList), 53 | ipDatasource.NewURLDataSource(config.VPNList), 54 | ipDatasource.NewURLDataSource(config.DCList), 55 | ) 56 | ipdata.RunUpdates() 57 | 58 | ipController, err := controllers.NewIP(ipdata) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | requestController, err := controllers.NewRequest(ipdata) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | return &API{ 69 | config: config, 70 | echo: echo.New(), 71 | controllerEmail: emailController, 72 | controllerIP: ipController, 73 | controllerRequest: requestController, 74 | }, nil 75 | } 76 | 77 | // Run starts HTTP server and exposes endpoints 78 | func (a *API) Run() { 79 | a.echo.HideBanner = true 80 | a.echo.Server = &http.Server{ 81 | ReadTimeout: 10 * time.Second, 82 | WriteTimeout: 60 * time.Second, 83 | IdleTimeout: 60 * time.Second, 84 | } 85 | 86 | a.echo.Use(middleware.Logger()) 87 | a.echo.Use(middleware.Recover()) 88 | a.echo.Use(middleware.BodyLimit("2M")) 89 | a.echo.Use(middleware.RequestID()) 90 | a.echo.Use(middlewares.NewRatelimiter()) 91 | 92 | // Internal endpoints should be protected from public access 93 | internal := a.echo.Group("/internal") 94 | internal.GET("/health", a.handleHealth) 95 | internal.GET("/debug/pprof/profile", echo.WrapHandler(http.HandlerFunc(pprof.Profile))) 96 | internal.GET("/debug/pprof/heap", echo.WrapHandler(pprof.Handler("heap"))) 97 | internal.GET("/routes", a.handleRoutes) 98 | p := prometheus.NewPrometheus("threatbite", nil) 99 | p.MetricsPath = "/internal/metrics" 100 | p.Use(a.echo) 101 | 102 | // Public pages 103 | a.echo.File("/", "resources/static/index.html") 104 | a.echo.File("/swagger.yml", "resources/static/swagger.yml") 105 | 106 | // Public API endpoints authorization token required 107 | endpoints := a.echo.Group("/v1/score") 108 | endpoints.GET("/ip/:ip", a.handleIP) 109 | endpoints.POST("/request", a.handleRequest) 110 | endpoints.GET("/email/:email", a.handleEmail) 111 | 112 | if a.config.AutoTLS { 113 | a.echo.AutoTLSManager.Cache = autocert.DirCache("./resources/tls_cache") 114 | a.echo.Logger.Fatal(a.echo.StartAutoTLS(fmt.Sprintf(":%d", a.config.Port))) 115 | } 116 | a.echo.Logger.Fatal(a.echo.Start(fmt.Sprintf(":%d", a.config.Port))) 117 | } 118 | 119 | func (a *API) handleHealth(c echo.Context) error { 120 | return c.String(http.StatusOK, "OK") 121 | } 122 | 123 | func (a *API) handleRoutes(c echo.Context) error { 124 | return c.JSONPretty(http.StatusOK, a.echo.Routes(), " ") 125 | } 126 | 127 | func (a *API) handleIP(c echo.Context) error { 128 | // echo params are not urledecoded automatically 129 | ip, err := url.QueryUnescape(c.Param("ip")) 130 | if err != nil { 131 | return echo.NewHTTPError(http.StatusBadRequest, "invalid IP address") 132 | } 133 | if err := a.controllerIP.Validate(ip); err != nil { 134 | return echo.NewHTTPError(http.StatusBadRequest, err.Error()) 135 | } 136 | 137 | result, err := a.controllerIP.Check(ip) 138 | if err != nil { 139 | log.Errorf("ip: %s, error: %s", ip, err) 140 | return echo.ErrInternalServerError 141 | } 142 | 143 | return c.JSONPretty(http.StatusOK, result, " ") 144 | } 145 | 146 | func (a *API) handleEmail(c echo.Context) error { 147 | // echo params are not urledecoded automatically, so query like this lame%40o2.pl will not be valid email. 148 | email, err := url.QueryUnescape(c.Param("email")) 149 | if err != nil { 150 | return echo.NewHTTPError(http.StatusBadRequest, "invalid email") 151 | } 152 | if err := a.controllerEmail.Validate(email); err != nil { 153 | return echo.NewHTTPError(http.StatusBadRequest, err.Error()) 154 | } 155 | 156 | result, err := a.controllerEmail.Check(email) 157 | if err != nil { 158 | log.Errorf("err: %s, email: %s", err, email) 159 | return echo.ErrInternalServerError 160 | } 161 | 162 | return c.JSONPretty(http.StatusOK, result, " ") 163 | } 164 | 165 | func (a *API) handleRequest(c echo.Context) error { 166 | request := controllers.RequestQuery{} 167 | if err := c.Bind(&request); err != nil { 168 | return echo.NewHTTPError(http.StatusBadRequest, err.Error()) 169 | } 170 | 171 | if err := a.controllerRequest.Validate(request); err != nil { 172 | return echo.NewHTTPError(http.StatusBadRequest, err.Error()) 173 | } 174 | 175 | result, err := a.controllerRequest.Check(request) 176 | if err != nil { 177 | log.Errorf("err: %s, email: %s", err, request) 178 | return echo.ErrInternalServerError 179 | } 180 | 181 | return c.JSONPretty(http.StatusOK, result, " ") 182 | } 183 | -------------------------------------------------------------------------------- /api/transport/middlewares/ratelimiter.go: -------------------------------------------------------------------------------- 1 | package middlewares 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/coinpaprika/ratelimiter" 7 | "github.com/labstack/echo/v4" 8 | "github.com/prometheus/common/log" 9 | ) 10 | 11 | // NewRatelimiter returns rate limiter. 12 | func NewRatelimiter() echo.MiddlewareFunc { 13 | var maxLimit int64 = 600 14 | windowSize := 60 * time.Second 15 | dataStore := ratelimiter.NewMapLimitStore(2*windowSize, windowSize) 16 | rateLimiter := ratelimiter.New(dataStore, maxLimit, windowSize) 17 | 18 | return func(next echo.HandlerFunc) echo.HandlerFunc { 19 | return func(c echo.Context) error { 20 | key := c.RealIP() 21 | limitStatus, err := rateLimiter.Check(key) 22 | if err != nil { 23 | log.Error(err) 24 | return next(c) 25 | } 26 | 27 | if limitStatus.IsLimited { 28 | return echo.ErrTooManyRequests 29 | } 30 | 31 | if err := rateLimiter.Inc(key); err != nil { 32 | log.Error(err) 33 | } 34 | 35 | return next(c) 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /browser/user_agent.go: -------------------------------------------------------------------------------- 1 | package browser 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | 7 | aho "github.com/BobuSumisu/aho-corasick" 8 | "github.com/avct/uasurfer" 9 | "github.com/labstack/gommon/log" 10 | ) 11 | 12 | // BrowserNames for comparison 13 | const ( 14 | BrowserUnknown = "Unknown" 15 | BrowserChrome = "Chrome" 16 | BrowserIE = "IE" 17 | BrowserSafari = "Safari" 18 | BrowserFirefox = "Firefox" 19 | BrowserAndroid = "Android" 20 | BrowserOpera = "Opera" 21 | 22 | DevicePhone = "Phone" 23 | 24 | OSAndroid = "Android" 25 | 26 | MinChromeMajorVersion = 50 27 | MinIEMajorVersion = 16 28 | MinFirefoxMajorVersion = 60 29 | MinAndroidMajorVersion = 5 30 | MinSafariMajorVersion = 10 31 | MinOperaMajorVersion = 10 32 | ) 33 | 34 | // Browser information about browser name and version taken from user agent header. 35 | // IsOld indicates that browser is considered as very old. 36 | type Browser struct { 37 | Name string 38 | Version struct { 39 | Major int 40 | Minor int 41 | Patch int 42 | } 43 | IsOld bool 44 | } 45 | 46 | // OS information about operating system taken from user agent header. 47 | type OS struct { 48 | Name string 49 | Version struct { 50 | Major int 51 | Minor int 52 | Patch int 53 | } 54 | Platform string 55 | } 56 | 57 | // UserAgent hold specific data for current uer agent. 58 | // github.com/avct/uasurfer is used to parse data. 59 | type UserAgent struct { 60 | Lowercase string 61 | Browser Browser 62 | OS OS 63 | Device string 64 | } 65 | 66 | // GetUserAgent returns information about the browser/operating system and device based on user agent header value. 67 | func GetUserAgent(agent string) *UserAgent { 68 | ua := uasurfer.Parse(agent) 69 | 70 | userAgent := &UserAgent{} 71 | userAgent.Lowercase = strings.ToLower(agent) 72 | userAgent.OS.Name = ua.OS.Name.StringTrimPrefix() 73 | userAgent.OS.Platform = ua.OS.Platform.StringTrimPrefix() 74 | userAgent.OS.Version = ua.OS.Version 75 | userAgent.Device = ua.DeviceType.StringTrimPrefix() 76 | userAgent.Browser.Name = ua.Browser.Name.StringTrimPrefix() 77 | userAgent.Browser.Version = ua.Browser.Version 78 | userAgent.Browser.IsOld = userAgent.isOldBrowser() 79 | 80 | return userAgent 81 | } 82 | 83 | // IsOldBrowser return bool value indicating if browser is outdated 84 | func (ua *UserAgent) isOldBrowser() bool { 85 | switch { 86 | case ua.Browser.Name == BrowserIE: 87 | if ua.Browser.Version.Major < MinIEMajorVersion { 88 | return true 89 | } 90 | case ua.Browser.Name == BrowserChrome: 91 | if ua.Device == DevicePhone && ua.OS.Name == OSAndroid && ua.OS.Version.Major >= 8 && ua.Browser.Version.Major < 60 { 92 | return true 93 | } else if ua.Browser.Version.Major < MinChromeMajorVersion { 94 | return true 95 | } 96 | case ua.Browser.Name == BrowserFirefox: 97 | if ua.Browser.Version.Major < MinFirefoxMajorVersion { 98 | return true 99 | } 100 | case ua.Browser.Name == BrowserAndroid: 101 | if ua.Browser.Version.Major < MinAndroidMajorVersion { 102 | return true 103 | } 104 | case ua.Browser.Name == BrowserSafari: 105 | if ua.Browser.Version.Major < MinSafariMajorVersion { 106 | return true 107 | } 108 | case ua.Browser.Name == BrowserOpera: 109 | if ua.Browser.Version.Major < MinOperaMajorVersion { 110 | return true 111 | } 112 | } 113 | 114 | return false 115 | } 116 | 117 | var trie = aho.NewTrieBuilder(). 118 | AddStrings([]string{ 119 | "12soso", "192.comagent", "1noonbot", "1on1searchbot", 120 | "3d_search", "3de_search2", "3g bot", "3gse", 121 | "50.nu", "a1 sitemap generator", "a1 website download", "a6-indexer", 122 | "aasp", "abachobot", "abonti", "abotemailsearch", 123 | "aboundex", "aboutusbot", "accmonitor compliance server", "accoon", 124 | "achulkov.net page walker", "acme.spider", "acoonbot", "acquia-crawler", 125 | "activetouristbot", "ad muncher", "adamm bot", "adbeat_bot", 126 | "adminshop.com", "advanced email extractor", "aesop_com_spiderman", "aespider", 127 | "af knowledge now verity spider", "aggregator:vocus", "ah-ha.com crawler", "ahrefsbot", 128 | "aibot", "aidu", "aihitbot", "aipbot", 129 | "aisiid", "aitcsrobot/1.1", "ajsitemap", "akamai-sitesnapshot", 130 | "alexawebsearchplatform", "alexfdownload", "alexibot", "alkalinebot", 131 | "all acronyms bot", "alpha search agent", "amerla search bot", "amfibibot", 132 | "ampmppc.com", "amznkassocbot", "anemone", "anonymous", 133 | "anotherbot", "answerbot", "answerbus", "answerchase prove", 134 | "antbot", "antibot", "antisantyworm", "antro.net", 135 | "aonde-spider", "aport", "appengine-google", "appid: s~stremor-crawler-", 136 | "aqua_products", "arabot", "arachmo", "arachnophilia", 137 | "aria equalizer", "arianna.libero.it", "arikus_spider", "art-online.com", 138 | "artavisbot", "artera", "asaha search engine turkey", "ask", 139 | "aspider", "aspseek", "asterias", "astrofind", 140 | "athenusbot", "atlocalbot", "atomic_email_hunter", "attach", 141 | "attrakt", "attributor", "augurfind", "auresys", 142 | "autobaron crawler", "autoemailspider", "autowebdir", "avsearch-", 143 | "axfeedsbot", "axonize-bot", "ayna", "b2w", 144 | "backdoorbot", "backrub", "backstreet browser", "backweb", 145 | "baidu", "bandit", "batchftp", "baypup", 146 | "bdfetch", "becomebot", "becomejpbot", "beetlebot", 147 | "bender", "besserscheitern-crawl", "betabot", "big brother", 148 | "big data", "bigado.com", "bigcliquebot", "bigfoot", 149 | "biglotron", "bilbo", "bilgibetabot", "bilgibot", 150 | "bintellibot", "bitlybot", "bitvouseragent", "bizbot003", 151 | "bizbot04", "bizworks retriever", "black hole", "black.hole", 152 | "blackbird", "blackmask.net search engine", "blackwidow", "bladder fusion", 153 | "blaiz-bee", "blexbot", "blinkx", "blitzbot", 154 | "blog conversation project", "blogmyway", "blogpulselive", "blogrefsbot", 155 | "blogscope", "blogslive", "bloobybot", "blowfish", 156 | "blt", "bnf.fr_bot", "boaconstrictor", "boardreader", 157 | "boi_crawl_00", "boia-scan-agent", "boia.org", "boitho", 158 | "bookmark buddy bookmark checker", "bookmark search tool", "bosug", "bot apoena", 159 | "botalot", "botrighthere", "botswana", "bottybot", 160 | "bpbot", "braintime_search", "brokenlinkcheck.com", "browseremulator", 161 | "browsermob", "bruinbot", "bsearchr&d", "bspider", 162 | "btbot", "btsearch", "bubing", "buddy", 163 | "buibui", "buildcms crawler", "builtbottough", "bullseye", 164 | "bumblebee", "bunnyslippers", "buscadorclarin", "buscaplus robi", 165 | "butterfly", "buyhawaiibot", "buzzbot", "byindia", 166 | "byspider", "byteserver", "bzbot", "c r a w l 3 r", 167 | "cacheblaster", "caddbot", "cafi", "camcrawler", 168 | "camelstampede", "canon-webrecord", "careerbot", "cataguru", 169 | "catchbot", "cazoodle", "ccbot", "ccgcrawl", 170 | "ccubee", "cd-preload", "ce-preload", "cegbfeieh", 171 | "cerberian drtrs", "cert figleafbot", "cfetch", "cfnetwork", 172 | "chameleon", "charlotte", "check&get", "checkbot", 173 | "checklinks", "cheesebot", "chemiede-nodebot", "cherrypicker", 174 | "chilkat", "chinaclaw", "cipinetbot", "cis455crawler", 175 | "citeseerxbot", "cizilla", "clariabot", "climate ark", 176 | "climateark spider", "cliqzbot", "clshttp", "clushbot", 177 | "coast scan engine", "coast webmaster pro", "coccoc", "collapsarweb", 178 | "collector", "colocrossing", "combine", "connectsearch", 179 | "conpilot", "contentsmartz", "contextad bot", "contype", 180 | "cookienet", "coolbot", "coolcheck", "copernic", 181 | "copier", "copyrightcheck", "core-project", "cosmos", 182 | "covario-ids", "cowbot-", "cowdog bot", "crabbybot", 183 | "craftbot@yahoo.com", "crawl_application", "crawler.kpricorn.org", "crawler43.ejupiter.com", 184 | "crawler4j", "crawler@", "crawler_for_infomine", "crawly", 185 | "creativecommons", "crescent", "cs-crawler", "cse html validator", 186 | "cshttpclient", "cuasarbot", "culsearch", "curl", 187 | "custo", "cvaulev", "cyberdog", "cybernavi_webget", 188 | "cyberpatrol sitecat webbot", "cyberspyder", "cydralspider", "d1garabicengine", 189 | "datacha0s", "datafountains", "dataparksearch", "dataprovider.com", 190 | "datascape robot", "dataspearspiderbot", "dataspider", "dattatec.com", 191 | "daumoa", "dblbot", "dcpbot", "declumbot", 192 | "deepindex", "deepnet crawler", "deeptrawl", "dejan", 193 | "del.icio.us-thumbnails", "deltascan", "delvubot", "der gro§e bildersauger", 194 | "der große bildersauger", "deusu", "dfs-fetch", "diagem", 195 | "diamond", "dibot", "didaxusbot", "digext", 196 | "digger", "digi-rssbot", "digitalarchivesbot", "digout4u", 197 | "diibot", "dillo", "dir_snatch.exe", "disco", 198 | "distilled-reputation-monitor", "djangotraineebot", "dkimrepbot", "dmoz downloader", 199 | "docomo", "dof-verify", "domaincrawler", "domainscan", 200 | "domainwatcher bot", "dotbot", "dotspotsbot", "dow jones searchbot", 201 | "download", "doy", "dragonfly", "drip", 202 | "drone", "dtaagent", "dtsearchspider", "dumbot", 203 | "dwaar", "dxseeker", "e-societyrobot", "eah", 204 | "earth platform indexer", "earth science educator robot", "easydl", "ebingbong", 205 | "ec2linkfinder", "ecairn-grabber", "ecatch", "echoosebot", 206 | "edisterbot", "edugovsearch", "egothor", "eidetica.com", 207 | "eirgrabber", "elblindo the blind bot", "elisabot", "ellerdalebot", 208 | "email exractor", "emailcollector", "emailleach", "emailsiphon", 209 | "emailwolf", "emeraldshield", "empas_robot", "enabot", 210 | "endeca", "enigmabot", "enswer neuro bot", "enter user-agent", 211 | "entitycubebot", "erocrawler", "estylesearch", "esyndicat bot", 212 | "eurosoft-bot", "evaal", "eventware", "everest-vulcan inc.", 213 | "exabot", "exactsearch", "exactseek", "exooba", 214 | "exploder", "express webpictures", "extractor", "eyenetie", 215 | "ez-robot", "ezooms", "f-bot test pilot", "factbot", 216 | "fairad client", "falcon", "fast data search document retriever", "fast esp", 217 | "fast-search-engine", "fastbot crawler", "fastbot.de crawler", "fatbot", 218 | "favcollector", "faviconizer", "favorites sweeper", "fdm", 219 | "fdse robot", "fedcontractorbot", "fembot", "fetch api request", 220 | "fetch_ici", "fgcrawler", "filangy", "filehound", 221 | "findanisp.com_isp_finder", "findlinks", "findweb", "firebat", 222 | "firstgov.gov search", "flaming attackbot", "flamingo_searchengine", "flashcapture", 223 | "flashget", "flickysearchbot", "fluffy the spider", "flunky", 224 | "focused_crawler", "followsite", "foobot", "fooooo_web_video_crawl", 225 | "fopper", "formulafinderbot", "forschungsportal", "fr_crawler", 226 | "francis", "freewebmonitoring sitechecker", "freshcrawler", "freshdownload", 227 | "freshlinks.exe", "friendfeedbot", "frodo.at", "froggle", 228 | "frontpage", "froola bot", "fu-nbi", "full_breadth_crawler", 229 | "funnelback", "furlbot", "g10-bot", "gaisbot", 230 | "galaxybot", "gazz", "gbplugin", "generate_infomine_category_classifiers", 231 | "genevabot", "geniebot", "genieo", "geomaxenginebot", 232 | "geometabot", "geonabot", "geovisu", "germcrawler", 233 | "gethtmlcontents", "getleft", "getright", "getsmart", 234 | "geturl.rexx", "getweb!", "giant", "gigablastopensource", 235 | "gigabot", "girafabot", "gleamebot", "gnome-vfs", 236 | "go!zilla", "go-ahead-got-it", "go-http-client", "goforit.com", 237 | "goforitbot", "gold crawler", "goldfire server", "golem", 238 | "goodjelly", "gordon-college-google-mini", "goroam", "goseebot", 239 | "gotit", "govbot", "gpu p2p crawler", "grabber", 240 | "grabnet", "grafula", "grapefx", "grapeshot", 241 | "grbot", "greenyogi", "gromit", "grub", 242 | "gsa", "gslfbot", "gulliver", "gulperbot", 243 | "gurujibot", "gvc business crawler", "gvc crawler", "gvc search bot", 244 | "gvc web crawler", "gvc weblink crawler", "gvc world links", "gvcbot.com", 245 | "happyfunbot", "harvest", "hatena antenna", "hawler", 246 | "hcat", "hclsreport-crawler", "hd nutch agent", "header_test_client", 247 | "healia", "helix", "here will be link to crawler site", "heritrix", 248 | "hiscan", "hisoftware accmonitor server", "hisoftware accverify", "hitcrawler", 249 | "hivabot", "hloader", "hmsebot", "hmview", 250 | "hoge", "holmes", "homepagesearch", "hooblybot-image", 251 | "hoowwwer", "hostcrawler", "hsft - link scanner", "hsft - lvu scanner", 252 | "hslide", "ht://check", "htdig", "html link validator", 253 | "htmlparser", "httplib", "httrack", "huaweisymantecspider", 254 | "hul-wax", "humanlinks", "hyperestraier", "hyperix", 255 | "ia_archiver", "iaarchiver-", "ibuena", "icab", 256 | "icds-ingestion", "ichiro", "icopyright conductor", "ieautodiscovery", 257 | "iecheck", "ihwebchecker", "iiitbot", "iim_405", 258 | "ilsebot", "iltrovatore", "image stripper", "image sucker", 259 | "image-fetcher", "imagebot", "imagefortress", "imageshereimagesthereimageseverywhere", 260 | "imagevisu", "imds_monitor", "imo-google-robot-intelink", "inagist.com url crawler", 261 | "indexer", "industry cortex webcrawler", "indy library", "indylabs_marius", 262 | "inelabot", "inet32 ctrl", "inetbot", "info seeker", 263 | "infolink", "infomine", "infonavirobot", "informant", 264 | "infoseek sidewinder", "infotekies", "infousabot", "ingrid", 265 | "inktomi", "insightscollector", "insightsworksbot", "inspirebot", 266 | "insumascout", "intelix", "intelliseek", "interget", 267 | "internet ninja", "internet radio crawler", "internetlinkagent", "interseek", 268 | "ioi", "ip-web-crawler.com", "ipadd bot", "ips-agent", 269 | "ipselonbot", "iria", "irlbot", "iron33", 270 | "isara", "isearch", "isilox", "istellabot", 271 | "its-learning crawler", "iu_csci_b659_class_crawler", "ivia", "jadynave", 272 | "java", "jbot", "jemmathetourist", "jennybot", 273 | "jetbot", "jetbrains omea pro", "jetcar", "jim", 274 | "jobo", "jobspider_ba", "joc", "joedog", 275 | "joyscapebot", "jspyda", "junut bot", "justview", 276 | "jyxobot", "k.s.bot", "kakclebot", "kalooga", 277 | "katatudo-spider", "kbeta1", "keepni web site monitor", "kenjin.spider", 278 | "keybot translation-search-machine", "keywenbot", "keyword density", "keyword.density", 279 | "kinjabot", "kitenga-crawler-bot", "kiwistatus", "kmbot-", 280 | "kmccrew bot search", "knight", "knowitall", "knowledge engine", 281 | "knowledge.com", "koepabot", "koninklijke", "korniki", 282 | "krowler", "ksbot", "kuloko-bot", "kulturarw3", 283 | "kummhttp", "kurzor", "kyluka crawl", "l.webis", 284 | "labhoo", "labourunions411", "lachesis", "lament", 285 | "lamerexterminator", "lapozzbot", "larbin", "lbot", 286 | "leaptag", "leechftp", "leechget", "letscrawl.com", 287 | "lexibot", "lexxebot", "lftp", "libcrawl", 288 | "libiviacore", "libw", "likse", "linguee bot", 289 | "link checker", "link validator", "link_checker", "linkalarm", 290 | "linkbot", "linkcheck by siteimprove.com", "linkcheck scanner", "linkchecker", 291 | "linkdex.com", "linkextractorpro", "linklint", "linklooker", 292 | "linkman", "links sql", "linkscan", "linksmanager.com_bot", 293 | "linksweeper", "linkwalker", "litefinder", "litlrbot", 294 | "little grabber at skanktale.com", "livelapbot", "lm harvester", "lmqueuebot", 295 | "lnspiderguy", "loadtimebot", "localcombot", "locust", 296 | "lolongbot", "lookbot", "lsearch", "lssbot", 297 | "lt scotland checklink", "ltx71.com", "lwp", "lycos_spider", 298 | "lydia entity spider", "lynnbot", "lytranslate", "mag-net", 299 | "magnet", "magpie-crawler", "magus bot", "mail.ru", 300 | "mainseek_bot", "mammoth", "map robot", "markwatch", 301 | "masagool", "masidani_bot_", "mass downloader", "mata hari", 302 | "mata.hari", "matentzn at cs dot man dot ac dot uk", "maxamine.com--robot", "maxamine.com-robot", 303 | "maxomobot", "mcbot", "medrabbit", "megite", 304 | "memacbot", "memo", "mendeleybot", "mercator-", 305 | "mercuryboard_user_agent_sql_injection.nasl", "metacarta", "metaeuro web search", "metager2", 306 | "metagloss", "metal crawler", "metaquerier", "metaspider", 307 | "metaspinner", "metauri", "mfcrawler", "mfhttpscan", 308 | "midown tool", "miixpc", "mini-robot", "minibot", 309 | "minirank", "mirror", "missigua locator", "mister pix", 310 | "mister.pix", "miva", "mj12bot", "mnogosearch", 311 | "mod_accessibility", "moduna.com", "moget", "mojeekbot", 312 | "monkeycrawl", "moses", "mowserbot", "mqbot", 313 | "mse360", "msindianwebcrawl", "msmobot", "msnptc", 314 | "msrbot", "mt-soft", "multitext", "my-heritrix-crawler", 315 | "my_little_searchengine_project", "myapp", "mycompanybot", "mycrawler", 316 | "myengines-us-bot", "myfamilybot", "myra", "nabot", 317 | "najdi.si", "nambu", "nameprotect", "nasa search", 318 | "natchcvs", "natweb-bad-link-mailer", "naver", "navroad", 319 | "nearsite", "nec-meshexplorer", "neosciocrawler", "nerdbynature.bot", 320 | "nerdybot", "nerima-crawl-", "nessus", "nestreader", 321 | "net vampire", "net::trackback", "netants", "netcarta cyberpilot pro", 322 | "netcraft", "netexperts", "netid.com bot", "netmechanic", 323 | "netprospector", "netresearchserver", "netseer", "netshift=", 324 | "netsongbot", "netsparker", "netspider", "netsrcherp", 325 | "netzip", "newmedhunt", "news bot", "news_search_app", 326 | "newsgatherer", "newsgroupreporter", "newstrovebot", "nextgensearchbot", 327 | "nextthing.org", "nicebot", "nicerspro", "niki-bot", 328 | "nimblecrawler", "nimbus-1", "ninetowns", "ninja", 329 | "njuicebot", "nlese", "nogate", "norbert the spider", 330 | "noteworthybot", "npbot", "nrcan intranet crawler", "nsdl_search_bot", 331 | "nu_tch", "nuggetize.com bot", "nusearch spider", "nutch", 332 | "nwspider", "nymesis", "nys-crawler", "objectssearch", 333 | "obot", "obvius external linkcheck", "ocelli", "octopus", 334 | "odp entries t_st", "oegp", "offline navigator", "offline.explorer", 335 | "ogspider", "omiexplorer_bot", "omniexplorer", "omnifind", 336 | "omniweb", "onetszukaj", "online link validator", "oozbot", 337 | "openbot", "openfind", "openintelligencedata", "openisearch", 338 | "openlink virtuoso rdf crawler", "opensearchserver_bot", "opidig", "optidiscover", 339 | "oracle secure enterprise search", "oracle ultra search", "orangebot", "orisbot", 340 | "ornl_crawler", "ornl_mercury", "osis-project.jp", "oso", 341 | "outfoxbot", "outfoxmelonbot", "owler-bot", "owsbot", 342 | "ozelot", "p3p client", "page_verifier", "pagebiteshyperbot", 343 | "pagebull", "pagedown", "pagefetcher", "pagegrabber", 344 | "pagerank monitor", "pamsnbot.htm", "panopy bot", "panscient.com", 345 | "pansophica", "papa foto", "paperlibot", "parasite", 346 | "parsijoo", "pathtraq", "pattern", "patwebbot", 347 | "pavuk", "paxleframework", "pbbot", "pcbrowser", 348 | "pcore-http", "pd-crawler", "penthesila", "perform_crawl", 349 | "perman", "personal ultimate crawler", "php version tracker", "phpcrawl", 350 | "phpdig", "picosearch", "pieno robot", "pipbot", 351 | "pipeliner", "pita", "pixfinder", "piyushbot", 352 | "planetwork bot search", "plucker", "plukkie", "plumtree", 353 | "pockey", "pocohttp", "pogodak.ba", "pogodak.co.yu", 354 | "poirot", "polybot", "pompos", "poodle predictor", 355 | "popscreenbot", "postpost", "privacyfinder", "projectwf-java-test-crawler", 356 | "propowerbot", "prowebwalker", "proxem websearch", "proximic", 357 | "proxy crawler", "psbot", "pss-bot", "psycheclone", 358 | "pub-crawler", "pucl", "pulsebot", "pump", 359 | "pwebot", "python", "qeavis agent", "qfkbot", 360 | "qualidade", "qualidator.com bot", "quepasacreep", "queryn metasearch", 361 | "queryn.metasearch", "quest.durato", "quintura-crw", "qunarbot", 362 | "qwantify", "qweery_robot.txt_checkbot", "qweerybot", "r2ibot", 363 | "r6_commentreader", "r6_feedfetcher", "r6_votereader", "rabot", 364 | "radian6", "radiation retriever", "rampybot", "rankivabot", 365 | "rankur", "rational sitecheck", "rcstartbot", "realdownload", 366 | "reaper", "rebi-shoveler", "recorder", "redbot", 367 | "redcarpet", "reget", "repomonkey", "research robot", 368 | "riddler", "riight", "risenetbot", "riverglassscanner", 369 | "robopal", "robosourcer", "robotek", "robozilla", 370 | "roger", "rome client", "rondello", "rotondo", 371 | "roverbot", "rpt-httpclient", "rtgibot", "rufusbot", 372 | "runnk online rss reader", "runnk rss aggregator", "s2bot", "safaribookmarkchecker", 373 | "safednsbot", "safetynet robot", "saladspoon", "sapienti", 374 | "sapphireweb", "sbider", "sbl-bot", "scfcrawler", 375 | "scich", "scientificcommons.org", "scollspider", "scooperbot", 376 | "scooter", "scoutjet", "scrapebox", "scrapy", 377 | "scrawltest", "scrubby", "scspider", "scumbot", 378 | "search publisher", "search x-bot", "search-channel", "search-engine-studio", 379 | "search.kumkie.com", "search.updated.com", "search.usgs.gov", "searcharoo.net", 380 | "searchblox", "searchbot", "searchengine", "searchhippo.com", 381 | "searchit-bot", "searchmarking", "searchmarks", "searchmee!", 382 | "searchmee_v", "searchmining", "searchnowbot", "searchpreview", 383 | "searchspider.com", "searqubot", "seb spider", "seekbot", 384 | "seeker.lookseek.com", "seeqbot", "seeqpod-vertical-crawler", "selflinkchecker", 385 | "semager", "semanticdiscovery", "semantifire", "semisearch", 386 | "semrushbot", "seoengworldbot", "seokicks", "seznambot", 387 | "shablastbot", "shadowwebanalyzer", "shareaza", "shelob", 388 | "sherlock", "shim-crawler", "shopsalad", "shopwiki", 389 | "showlinks", "showyoubot", "siclab", "silk", 390 | "simplepie", "siphon", "sitebot", "sitecheck", 391 | "sitefinder", "siteguardbot", "siteorbiter", "sitesnagger", 392 | "sitesucker", "sitesweeper", "sitexpert", "skimbot", 393 | "skimwordsbot", "skreemrbot", "skywalker", "sleipnir", 394 | "slow-crawler", "slysearch", "smart-crawler", "smartdownload", 395 | "smarte bot", "smartwit.com", "snake", "snap.com beta crawler", 396 | "snapbot", "snappreviewbot", "snappy", "snookit", 397 | "snooper", "snoopy", "societyrobot", "socscibot", 398 | "soft411 directory", "sogou", "sohu agent", "sohu-search", 399 | "sokitomi crawl", "solbot", "sondeur", "sootle", 400 | "sosospider", "space bison", "space fung", "spacebison", 401 | "spankbot", "spanner", "spatineo monitor controller", "spatineo serval controller", 402 | "spatineo serval getmapbot", "special_archiver", "speedy", "sphere scout", 403 | "sphider", "spider.terranautic.net", "spiderengine", "spiderku", 404 | "spiderman", "spinn3r", "spinne", "sportcrew-bot", 405 | "sproose", "spyder3.microsys.com", "sq webscanner", "sqlmap", 406 | "squid-prefetch", "squidclamav_redirector", "sqworm", "srevbot", 407 | "sslbot", "ssm agent", "stackrambler", "stardownloader", 408 | "statbot", "statcrawler", "statedept-crawler", "steeler", 409 | "stegmann-bot", "stero", "stripper", "stumbler", 410 | "suchclip", "sucker", "sumeetbot", "sumitbot", 411 | "summizebot", "summizefeedreader", "sunrise xp", "superbot", 412 | "superhttp", "superlumin downloader", "superpagesbot", "supremesearch.net", 413 | "supybot", "surdotlybot", "surf", "surveybot", 414 | "suzuran", "swebot", "swish-e", "sygolbot", 415 | "synapticwalker", "syntryx ant scout chassis pheromone", "systemsearch-robot", "szukacz", 416 | "s~stremor-crawler", "t-h-u-n-d-e-r-s-t-o-n-e", "tailrank", "takeout", 417 | "talkro web-shot", "tamu_crawler", "tapuzbot", "tarantula", 418 | "targetblaster.com", "targetyournews.com bot", "tausdatabot", "taxinomiabot", 419 | "teamsoft wininet component", "tecomi bot", "teezirbot", "teleport", 420 | "telesoft", "teradex mapper", "teragram_crawler", "terrawizbot", 421 | "testbot", "testing of bot", "textbot", "thatrobotsite.com", 422 | "the dyslexalizer", "the intraformant", "the.intraformant", "thenomad", 423 | "theophrastus", "theusefulbot", "thumbbot", "thumbnail.cz robot", 424 | "thumbshots-de-bot", "tigerbot", "tighttwatbot", "tineye", 425 | "titan", "to-dress_ru_bot_", "to-night-bot", "tocrawl", 426 | "topicalizer", "topicblogs", "toplistbot", "topserver php", 427 | "topyx-crawler", "touche", "tourlentascanner", "tpsystem", 428 | "traazi", "transgenikbot", "travel-search", "travelbot", 429 | "travellazerbot", "treezy", "trendiction", "trex", 430 | "tridentspider", "trovator", "true_robot", "tscholarsbot", 431 | "tsm translation-search-machine", "tswebbot", "tulipchain", "turingos", 432 | "turnitinbot", "tutorgigbot", "tweetedtimes bot", "tweetmemebot", 433 | "tweezler", "twengabot", "twice", "twikle", 434 | "twinuffbot", "twisted pagegetter", "twitturls", "twitturly", 435 | "tygobot", "tygoprowler", "typhoeus", "u.s. government printing office", 436 | "uberbot", "ucb-nutch", "udmsearch", "ufam-crawler-", 437 | "ultraseek", "unchaos", "unidentified", "unisterbot", 438 | "unitek uniengine", "universalsearch", "unwindfetchor", "uoftdb_experiment", 439 | "updated", "uptimebot/1.0", "url control", "url-checker", 440 | "url_gather", "urlappendbot", "urlblaze", "urlchecker", 441 | "urlck", "urldispatcher", "urlspiderpro", "urly warning", 442 | "urly.warning", "usaf afkn k2spider", "usasearch", "uss-cosmix", 443 | "usyd-nlp-spider", "vacobot", "vacuum", "vadixbot", 444 | "vagabondo", "valkyrie", "vbseo", "vci webviewer vci webviewer win32", 445 | "verbstarbot", "vericitecrawler", "verifactrola", "verity-url-gateway", 446 | "vermut", "versus crawler", "versus.integis.ch", "viasarchivinginformation.html", 447 | "vipr", "virus-detector", "virus_detector", "visbot", 448 | "vishal for clia", "visweb", "vital search'n urchin", "vlad", 449 | "vlsearch", "vmbot", "vocusbot", "voideye", 450 | "voil", "voilabot", "vortex", "voyager", 451 | "vspider", "vuhuvbot/1.0", "w3c-webcon", "w3c_unicorn", 452 | "w3search", "wacbot", "wanadoo", "wastrix", 453 | "water conserve portal", "water conserve spider", "watzbot", "wauuu", 454 | "wavefire", "waypath", "wazzup", "wbdbot", 455 | "wbsrch", "web ceo online robot", "web crawler", "web downloader", 456 | "web image collector", "web link validator", "web magnet", "web site downloader", 457 | "web sucker", "web-agent", "web-sniffer", "web.image.collector", 458 | "webaltbot", "webauto", "webbot", "webbul-bot", 459 | "webcapture", "webcheck", "webclipping.com", "webcollage", 460 | "webcopier", "webcopy", "webcorp", "webcrawl.net", 461 | "webcrawler", "webdatacentrebot", "webdownloader for x", "webdup", 462 | "webemailextrac", "webenhancer", "webfetch", "webgather", 463 | "webgo is", "webgobbler", "webimages", "webinator-search2", 464 | "webinator-wbi", "webindex", "weblayers", "webleacher", 465 | "weblexbot", "weblinker", "weblyzard", "webmastercoffee", 466 | "webmasterworld extractor", "webmasterworldforumbot", "webminer", "webmoose", 467 | "webot", "webpix", "webreaper", "webripper", 468 | "websauger", "webscan", "websearchbench", "website", 469 | "webspear", "websphinx", "webspider", "webster", 470 | "webstripper", "webtrafficexpress", "webtrends link analyzer", "webvac", 471 | "webwalk", "webwasher", "webwatch", "webwhacker", 472 | "webxm", "webzip", "weddings.info", "wenbin", 473 | "wep search", "wepa", "werelatebot", "wget", 474 | "whacker", "whirlpool web engine", "whowhere robot", "widow", 475 | "wikiabot", "wikio", "wikiwix-bot-", "winhttp", 476 | "wire", "wisebot", "wisenutbot", "wish-la", 477 | "wish-project", "wisponbot", "wmcai-robot", "wminer", 478 | "wmsbot", "woriobot", "worldshop", "worqmada", 479 | "wotbox", "wume_crawler", "www collector", "www-collector-e", 480 | "www-mechanize", "wwwoffle", "wwwrobot", "wwwster", 481 | "wwwwanderer", "wwwxref", "wysigot", "x-clawler", 482 | "x-crawler", "xaldon", "xenu", "xerka metabot", 483 | "xerka webbot", "xget", "xirq", "xmarksfetch", 484 | "xqrobot", "y!j", "yacy.net", "yacybot", 485 | "yanga worldsearch bot", "yarienavoir.net", "yasaklibot", "yats crawler", 486 | "ybot", "yebolbot", "yellowjacket", "yeti", 487 | "yolinkbot", "yooglifetchagent", "yoono", "yottacars_bot", 488 | "yourls", "z-add link checker", "zagrebin", "zao", 489 | "zedzo.validate", "zermelo", "zeus", "zibber-v", 490 | "zimeno", "zing-bottabot", "zipppbot", "zongbot", 491 | "zoomspider", "zotag search", "zsebot", "zuibot", 492 | }).Build() 493 | 494 | // IsBotUserAgent return if UserAgent is a bot 495 | func IsBotUserAgent(agent string) bool { 496 | matches := trie.MatchString(strings.ToLower(agent)) 497 | if len(matches) > 0 { 498 | log.Debugf("[IsBotUserAgent] ip: %s", matches) 499 | return true 500 | } 501 | return false 502 | } 503 | 504 | var reMobileUserAgent = regexp.MustCompile("(?:hpw|i|web)os|alamofire|alcatel|amoi|android|avantgo|blackberry|blazer|cell|cfnetwork|darwin|dolfin|dolphin|fennec|htc|ip(?:hone|od|ad)|ipaq|j2me|kindle|midp|minimo|mobi|motorola|nec-|netfront|nokia|opera m(ob|in)i|palm|phone|pocket|portable|psp|silk-accelerated|skyfire|sony|ucbrowser|up.browser|up.link|windows ce|xda|zte|zune") 505 | 506 | // IsMobileUserAgent Checks if User-Agent is from mobile device 507 | func IsMobileUserAgent(agent string) bool { 508 | if reMobileUserAgent.MatchString(strings.ToLower(agent)) { 509 | log.Debugf("[IsMobileUserAgent] ip: %s") 510 | return true 511 | } 512 | return false 513 | } 514 | 515 | var reScriptUserAgent = regexp.MustCompile("curl|wget|collectd|python|urllib|java|jakarta|httpclient|phpcrawl|libwww|perl|go-http|okhttp|lua-resty|winhttp|awesomium") 516 | 517 | // IsScriptUserAgent check if UserAgent comes from script 518 | func IsScriptUserAgent(agent string) bool { 519 | if reScriptUserAgent.MatchString(strings.ToLower(agent)) { 520 | log.Debugf("[IsScriptUserAgent] ip: %s") 521 | return true 522 | } 523 | return false 524 | } 525 | -------------------------------------------------------------------------------- /browser/user_agent_test.go: -------------------------------------------------------------------------------- 1 | package browser 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/suite" 7 | ) 8 | 9 | type UseragentSuite struct { 10 | suite.Suite 11 | } 12 | 13 | func (suite *UseragentSuite) TestParseUseragent() { 14 | tests := []struct { 15 | ua string 16 | want *UserAgent 17 | }{ 18 | {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36", &UserAgent{ 19 | Lowercase: "mozilla/5.0 (macintosh; intel mac os x 10_14_4) applewebkit/537.36 (khtml, like gecko) chrome/73.0.3683.86 safari/537.36", 20 | Device: "Computer", 21 | OS: OS{ 22 | Name: "MacOSX", Platform: "Mac", Version: struct { 23 | Major int 24 | Minor int 25 | Patch int 26 | }{Major: 10, Minor: 14, Patch: 4}}, 27 | Browser: Browser{ 28 | Name: BrowserChrome, 29 | Version: struct { 30 | Major int 31 | Minor int 32 | Patch int 33 | }{Major: 73, Minor: 0, Patch: 3683}}, 34 | }}, 35 | {"invalid", &UserAgent{ 36 | Lowercase: "invalid", 37 | Device: "Unknown", 38 | OS: OS{ 39 | Name: "Unknown", Platform: "Unknown"}, 40 | Browser: Browser{ 41 | Name: BrowserUnknown, 42 | }, 43 | }}, 44 | } 45 | 46 | for _, t := range tests { 47 | ua := GetUserAgent(t.ua) 48 | suite.Equal(t.want, ua) 49 | } 50 | } 51 | 52 | func (suite *UseragentSuite) TestParseUseragentOldBrowser() { 53 | tests := []struct { 54 | ua string 55 | isOld bool 56 | }{ 57 | {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36", false}, 58 | {"Mozilla/5.0 (Linux; Android 4.4.2; SAMSUNG-SGH-I337 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.170 Mobile Safari/537.36", true}, 59 | {"Mozilla/5.0 (Linux; U; Android 4.0.3; de-ch; HTC Sensation Build/IML74K) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30", true}, 60 | {"Mozilla/5.0 (Linux; Android 6.0; ALE-L21) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.80 Mobile Safari/537.36", false}, 61 | {"Mozilla/5.0 (Linux; Android 6.0; XT1072 Build/MPBS24.65-34-5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.87 Mobile Safari/537.36", false}, 62 | {"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; MDDRJS; rv:11.0) like Gecko", true}, 63 | {"Mozilla/5.0 (Linux; Android 5.0.2; SAMSUNG SM-T535 Build/LRX22G) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/9.2 Chrome/67.0.3396.87 Safari/537.36", false}, 64 | {"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36", true}, 65 | {"Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9B176 Safari/7534.48.3", true}, 66 | {"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36 OPR/56.0.3051.116", false}, 67 | {"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_2 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A501 Safari/9537.53", true}, 68 | {"Mozilla/5.0 (Windows Phone 10.0; Android 6.0.1; NOKIA; Lumia 830) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Mobile Safari/537.36 Edge/14.14393", true}, 69 | {"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 YaBrowser/19.3.1.828 Yowser/2.5 Safari/537.36", false}, 70 | {"Mozilla/5.0 (Windows NT 6.3; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0", false}, 71 | {"Mozilla/5.0 (Android 7.0; Mobile; rv:58.0) Gecko/58.0 Firefox/58.0", true}, 72 | {"Opera/9.80 (J2ME/iPhone;Opera Mini/5.0.019802/886; U; ja)Presto/2.4.15", true}, 73 | {"Opera/9.80 (J2ME/iPhone;Opera Mini/5.0.019802/886; U; ja)Presto/2.4.15", true}, // duplicate to check if once set properly 74 | {"Mozilla/5.0 (Linux; Android 8.0.0; SM-G930F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.3729.136 Mobile Safari/537.36", true}, 75 | {"Mozilla/5.0 (Linux; Android 7.0.0; SM-G930F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.3729.136 Mobile Safari/537.36", true}, 76 | {"Not a browser UA", false}, 77 | {"", false}, 78 | // should be true - fix uasurfer to support such browsers (now returns as Unknown) 79 | {"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.17) Gecko/20080919 K-Meleon/1.5.1", false}, 80 | {"Mozilla/5.0 (Windows; U; Windows NT 5.1; pt-BR) AppleWebKit/533.3 (KHTML, like Gecko) QtWeb Internet Browser/3.7 http://www.QtWeb.net", false}, 81 | {"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2a1pre) Gecko/20090316 Minefield/3.2a1pre", false}, 82 | } 83 | 84 | for _, t := range tests { 85 | ua := GetUserAgent(t.ua) 86 | suite.Equal(t.isOld, ua.Browser.IsOld, t) 87 | } 88 | } 89 | 90 | func TestUseragentSuite(t *testing.T) { 91 | suite.Run(t, new(UseragentSuite)) 92 | } 93 | -------------------------------------------------------------------------------- /cmd/threatbite/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | 6 | "github.com/labstack/gommon/log" 7 | "github.com/optimatiq/threatbite/api/transport" 8 | "github.com/optimatiq/threatbite/config" 9 | ) 10 | 11 | var ( 12 | date = "unknown" 13 | tag = "dev" 14 | commit = "unknown" 15 | ) 16 | 17 | func main() { 18 | configFile := flag.String("config", "", "a path to the configuration file") 19 | flag.Parse() 20 | 21 | conf, err := config.NewConfig(*configFile) 22 | if err != nil { 23 | log.Fatal(err) 24 | } 25 | 26 | log.SetPrefix("threatbite") 27 | if conf.Debug { 28 | log.SetLevel(log.DEBUG) 29 | } 30 | 31 | log.Debugf("Starting the app, build date: %s, git tag: %s, git commit: %s", date, tag, commit) 32 | 33 | server, err := transport.NewAPI(conf) 34 | if err != nil { 35 | log.Fatal(err) 36 | } 37 | 38 | server.Run() 39 | } 40 | -------------------------------------------------------------------------------- /config.env: -------------------------------------------------------------------------------- 1 | DEBUG=false 2 | PORT=8080 3 | 4 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "net/url" 6 | "os" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/joho/godotenv" 11 | ) 12 | 13 | // Config configuration struct for the project 14 | type Config struct { 15 | Port int 16 | Debug bool 17 | PwnedKey string 18 | MaxmindKey string 19 | SMTPHello string 20 | SMTPFrom string 21 | AutoTLS bool 22 | ProxyList []string 23 | SpamList []string 24 | VPNList []string 25 | DCList []string 26 | EmailDisposalList []string 27 | EmailFreeList []string 28 | } 29 | 30 | // NewConfig returns a new configuration struct or error. 31 | // Configuration is stored in the environment as the one of the tenets of a twelve-factor app. 32 | // config.env or config_local.env files are allowed but are totally optional. 33 | // If config_local.env is present than it's used (only this file), otherwise config.env is checked 34 | // Envs take precedence of envs that are imported from config_local.env or config.env files 35 | func NewConfig(configFile string) (*Config, error) { 36 | config := &Config{ 37 | Port: 8080, 38 | Debug: false, 39 | ProxyList: []string{"https://get.threatbite.com/public/proxy.txt"}, 40 | SpamList: []string{"https://get.threatbite.com/public/spam.txt"}, 41 | VPNList: []string{"https://get.threatbite.com/public/vpn.txt"}, 42 | DCList: []string{"https://get.threatbite.com/public/dc-names.txt"}, 43 | EmailDisposalList: []string{"https://get.threatbite.com/public/disposal.txt"}, 44 | EmailFreeList: []string{"https://get.threatbite.com/public/free.txt"}, 45 | } 46 | 47 | if configFile == "" { 48 | if _, err := os.Stat("config_local.env"); !os.IsNotExist(err) { 49 | configFile = "config_local.env" 50 | } else if _, err := os.Stat("config.env"); !os.IsNotExist(err) { 51 | configFile = "config.env" 52 | } 53 | } 54 | 55 | if configFile != "" { 56 | if err := godotenv.Load(configFile); err != nil { 57 | return nil, err 58 | } 59 | } 60 | 61 | if port := os.Getenv("PORT"); port != "" { 62 | p, err := strconv.Atoi(port) 63 | if err != nil { 64 | return nil, fmt.Errorf("invalid port value: %s, error: %w", port, err) 65 | } 66 | config.Port = p 67 | } 68 | 69 | if debug := os.Getenv("DEBUG"); debug == "true" || debug == "1" { 70 | config.Debug = true 71 | } 72 | 73 | if tls := os.Getenv("AUTO_TLS"); tls == "true" || tls == "1" { 74 | config.AutoTLS = true 75 | } 76 | 77 | config.PwnedKey = os.Getenv("PWNED_KEY") 78 | config.MaxmindKey = os.Getenv("MAXMIND_KEY") 79 | 80 | config.SMTPHello = os.Getenv("SMTP_HELLO") 81 | config.SMTPFrom = os.Getenv("SMTP_FROM") 82 | 83 | lists := map[string]*[]string{ 84 | "PROXY_LIST": &config.ProxyList, 85 | "SPAM_LIST": &config.SpamList, 86 | "VPN_LIST": &config.VPNList, 87 | "DC_LIST": &config.DCList, 88 | "EMAIL_DISPOSAL_LIST": &config.EmailDisposalList, 89 | "EMAIL_FREE_LIST": &config.EmailFreeList, 90 | } 91 | 92 | for env, list := range lists { 93 | if e := os.Getenv(env); e != "" { 94 | *list = []string{} 95 | for _, u := range strings.Fields(e) { 96 | if _, err := url.ParseRequestURI(u); err != nil { 97 | return nil, fmt.Errorf("invalid list URL: %s, error: %w", u, err) 98 | } 99 | *list = append(*list, u) 100 | } 101 | } 102 | } 103 | 104 | return config, nil 105 | } 106 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestNewConfigLists(t *testing.T) { 11 | tests := []struct { 12 | name string 13 | wantErr bool 14 | list string 15 | }{ 16 | { 17 | name: "list default", 18 | wantErr: false, 19 | }, 20 | { 21 | name: "proxy list invalid one", 22 | wantErr: true, 23 | list: "invalid_url", 24 | }, 25 | { 26 | name: "proxy list invalid one of", 27 | wantErr: true, 28 | list: "https://some_url.com invalid_url", 29 | }, 30 | { 31 | name: "proxy list valid one", 32 | wantErr: false, 33 | list: "https://some_url.com", 34 | }, 35 | { 36 | name: "proxy list valid more", 37 | wantErr: false, 38 | list: "https://some_url.com https://next_url_.com", 39 | }, 40 | } 41 | for _, env := range []string{"PROXY_LIST", "SPAM_LIST", "VPN_LIST", "DC_LIST", "EMAIL_DISPOSAL_LIST", "EMAIL_FREE_LIST"} { 42 | for _, tt := range tests { 43 | t.Run(tt.name+"_"+env, func(t *testing.T) { 44 | err := os.Setenv(env, tt.list) 45 | assert.NoError(t, err) 46 | 47 | _, err = NewConfig("") 48 | if (err != nil) != tt.wantErr { 49 | t.Errorf("NewConfig() error = %v, wantErr %v", err, tt.wantErr) 50 | return 51 | } 52 | }) 53 | } 54 | os.Unsetenv(env) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /email/datasource/datasource.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | // ErrNoData no more date in iterator, means that we finished iterating. 8 | var ErrNoData = errors.New("no more data") 9 | 10 | // ErrInvalidData source is available or data provided in the source were not valid IPv4, IPv6 or CIDR. 11 | // When this error is return Next() method is called again. 12 | var ErrInvalidData = errors.New("invalid data") 13 | 14 | // DataSource defines method for accessing stream of addresses. 15 | type DataSource interface { 16 | // Next returns net.IPNet on success or error. 17 | // ErrNoData and ErrInvalidData can be ignored 18 | Next() (string, error) 19 | Reset() error 20 | } 21 | -------------------------------------------------------------------------------- /email/datasource/datasource_empty.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | // EmptyDataSource as the name suggest, this data source contains no data. 4 | type EmptyDataSource struct { 5 | } 6 | 7 | // NewEmptyDataSource returns empty data source 8 | func NewEmptyDataSource() *EmptyDataSource { 9 | return &EmptyDataSource{} 10 | } 11 | 12 | // Reset does nothing. 13 | func (s *EmptyDataSource) Reset() error { 14 | return nil 15 | } 16 | 17 | // Next returns ErrNoData error always 18 | func (s *EmptyDataSource) Next() (string, error) { 19 | return "", ErrNoData 20 | } 21 | -------------------------------------------------------------------------------- /email/datasource/datasource_list.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | // ListDataSource stores current state (counters) of this source. 4 | type ListDataSource struct { 5 | i int 6 | list []string 7 | } 8 | 9 | // NewListDataSource first argument is a list of domains. 10 | // Returns DataSource or error on IP parsing. 11 | func NewListDataSource(list []string) *ListDataSource { 12 | return &ListDataSource{ 13 | list: list, 14 | } 15 | } 16 | 17 | // Reset rewinds source to the beginning. 18 | func (s *ListDataSource) Reset() error { 19 | s.i = 0 20 | return nil 21 | } 22 | 23 | // Next returns domain from the provided list in NewListDataSource method. 24 | // ErrNoData is returned when there is no data, this error indicates that we reached the end. 25 | func (s *ListDataSource) Next() (string, error) { 26 | if s.i >= len(s.list) || len(s.list) <= 0 { 27 | return "", ErrNoData 28 | } 29 | 30 | v := s.list[s.i] 31 | s.i++ 32 | return v, nil 33 | } 34 | -------------------------------------------------------------------------------- /email/datasource/datasource_test.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/suite" 9 | ) 10 | 11 | type DatasourceSuite struct { 12 | suite.Suite 13 | privateRand *rand.Rand 14 | } 15 | 16 | func (suite *DatasourceSuite) SetupTest() { 17 | suite.privateRand = rand.New(rand.NewSource(time.Now().UnixNano())) // #nosec 18 | } 19 | 20 | func (suite *DatasourceSuite) Test_NewURLDataSource() { 21 | ds := NewURLDataSource([]string{ 22 | "https://iplists.firehol.org/files/proxz_1d.ipset", 23 | }) 24 | 25 | ip, err := ds.Next() 26 | suite.NoError(err) 27 | suite.NotEmpty(ip) 28 | 29 | ds = NewURLDataSource([]string{ 30 | "invalid", 31 | }) 32 | 33 | ip, err = ds.Next() 34 | suite.Error(err) 35 | suite.Empty(ip) 36 | } 37 | 38 | func TestDatasourceSuite(t *testing.T) { 39 | suite.Run(t, new(DatasourceSuite)) 40 | } 41 | -------------------------------------------------------------------------------- /email/datasource/datasource_url.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "io/ioutil" 7 | "net" 8 | "net/http" 9 | "strings" 10 | "time" 11 | 12 | "github.com/labstack/gommon/log" 13 | ) 14 | 15 | // URLDataSource stores current state (counters, URLs, scanners) of this source. 16 | type URLDataSource struct { 17 | urls []string 18 | u int 19 | scanner *bufio.Scanner 20 | client *http.Client 21 | } 22 | 23 | // NewURLDataSource returns iterator, which downloads lists from provided URLs and extract addresses. 24 | // Comments are allowed and ignored. Comments start with # at the beginning of the line. 25 | // Some lists have comments after their address, they are also ignored 26 | func NewURLDataSource(urls []string) *URLDataSource { 27 | dataSource := &URLDataSource{ 28 | client: &http.Client{ 29 | Transport: &http.Transport{ 30 | DialContext: (&net.Dialer{ 31 | Timeout: 60 * time.Second, 32 | KeepAlive: 15 * time.Second, 33 | }).DialContext, 34 | TLSHandshakeTimeout: 60 * time.Second, 35 | ExpectContinueTimeout: 10 * time.Second, 36 | ResponseHeaderTimeout: 10 * time.Second, 37 | }, 38 | Timeout: 120 * time.Second, 39 | }, 40 | urls: urls, 41 | } 42 | 43 | return dataSource 44 | } 45 | 46 | // Reset rewinds source to the beginning. 47 | func (s *URLDataSource) Reset() error { 48 | s.u = 0 49 | s.scanner = nil 50 | return nil 51 | } 52 | 53 | func (s *URLDataSource) Next() (string, error) { 54 | if s.u >= len(s.urls) || len(s.urls) <= 0 { 55 | return "", ErrNoData 56 | } 57 | url := s.urls[s.u] 58 | 59 | if s.scanner == nil { 60 | response, err := s.client.Get(url) 61 | if err != nil { 62 | log.Errorf("[datasource] cannot download list from: %s, error: %s", url, err) 63 | s.u++ 64 | return "", ErrInvalidData 65 | } 66 | 67 | body, err := ioutil.ReadAll(response.Body) 68 | if err != nil { 69 | log.Errorf("[datasource] cannot read from: %s, error: %s", url, err) 70 | s.u++ 71 | return "", ErrInvalidData 72 | } 73 | 74 | if err := response.Body.Close(); err != nil { 75 | log.Errorf("[datasource] cannot close response body from: %s, error: %s", url, err) 76 | } 77 | 78 | s.scanner = bufio.NewScanner(bytes.NewReader(body)) 79 | } 80 | 81 | var line string 82 | for s.scanner.Scan() { 83 | line = s.scanner.Text() 84 | // some lists have address with optional comment as a second argument separated by spaces or tabs 85 | line = strings.ReplaceAll(line, "\t", " ") 86 | line := strings.Split(line, " ")[0] 87 | 88 | // Comment 89 | if strings.Index(line, "#") == 0 { 90 | continue 91 | } 92 | 93 | return line, nil 94 | } 95 | 96 | err := s.scanner.Err() 97 | s.scanner = nil 98 | 99 | if err != nil { 100 | return "", err 101 | } 102 | 103 | s.u++ 104 | return s.Next() 105 | } 106 | -------------------------------------------------------------------------------- /email/datasource/domain.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "github.com/labstack/gommon/log" 8 | ) 9 | 10 | type Domain struct { 11 | domains map[string]bool 12 | ds DataSource 13 | name string 14 | } 15 | 16 | // NewDomain returns a new domain list build on top of map. 17 | // Load method has to be called manually in order to get data from data source 18 | func NewDomain(ds DataSource, name string) *Domain { 19 | return &Domain{ 20 | domains: make(map[string]bool), 21 | ds: ds, 22 | name: name, 23 | } 24 | } 25 | 26 | // Check if lists contains domain from request 27 | func (d *Domain) Check(domain string) bool { 28 | _, found := d.domains[domain] 29 | return found 30 | } 31 | 32 | func (d *Domain) Load() error { 33 | log.Debugf("[list] loading %s list start", d.name) 34 | defer func() { 35 | log.Debugf("[list] loading %s stop; stats domains: %d", d.name, len(d.domains)) 36 | }() 37 | 38 | if err := d.ds.Reset(); err != nil { 39 | return fmt.Errorf("could not reset data source, error: %w", err) 40 | } 41 | 42 | for { 43 | domain, err := d.ds.Next() 44 | if err != nil { 45 | if err == ErrNoData { 46 | return nil 47 | } else if err == ErrInvalidData { 48 | continue 49 | } else { 50 | return fmt.Errorf("could not iterate over data source, error: %w", err) 51 | } 52 | } 53 | 54 | d.domains[strings.ToLower(domain)] = true 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /email/disposal.go: -------------------------------------------------------------------------------- 1 | package email 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/labstack/gommon/log" 7 | "github.com/optimatiq/threatbite/email/datasource" 8 | ) 9 | 10 | type disposal struct { 11 | domain *datasource.Domain 12 | } 13 | 14 | func newDisposal(source datasource.DataSource) *disposal { 15 | return &disposal{domain: datasource.NewDomain(source, "disposal")} 16 | } 17 | 18 | func (d *disposal) isDisposal(email string) bool { 19 | domain := strings.ToLower(strings.Split(email, "@")[1]) 20 | isDisposal := d.domain.Check(domain) 21 | log.Debugf("[isDisposal] domain: %s disposal: %t", domain, isDisposal) 22 | return isDisposal 23 | } 24 | -------------------------------------------------------------------------------- /email/disposal_test.go: -------------------------------------------------------------------------------- 1 | package email 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/optimatiq/threatbite/email/datasource" 9 | ) 10 | 11 | func Test_disposal_isDisposal(t *testing.T) { 12 | type fields struct { 13 | domain *datasource.Domain 14 | } 15 | type args struct { 16 | email string 17 | } 18 | tests := []struct { 19 | name string 20 | fields fields 21 | args args 22 | want bool 23 | }{ 24 | { 25 | name: "on list", 26 | fields: fields{ 27 | domain: datasource.NewDomain(datasource.NewListDataSource([]string{"maildrop.cc"}), "disposal"), 28 | }, 29 | args: args{ 30 | email: "xxx@maildrop.cc", 31 | }, 32 | want: true, 33 | }, 34 | { 35 | name: "on list 2", 36 | fields: fields{ 37 | domain: datasource.NewDomain(datasource.NewListDataSource([]string{"xxx.com", "maildrop.cc"}), "disposal"), 38 | }, 39 | args: args{ 40 | email: "xxx@maildrop.cc", 41 | }, 42 | want: true, 43 | }, 44 | { 45 | name: "on list case sensitive", 46 | fields: fields{ 47 | domain: datasource.NewDomain(datasource.NewListDataSource([]string{"maildrop.cc"}), "disposal"), 48 | }, 49 | args: args{ 50 | email: "xxx@maildrop.CC", 51 | }, 52 | want: true, 53 | }, 54 | { 55 | name: "not on list", 56 | fields: fields{ 57 | domain: datasource.NewDomain(datasource.NewListDataSource([]string{"maildrop.cc"}), "disposal"), 58 | }, 59 | args: args{ 60 | email: "xxx@xxx.com", 61 | }, 62 | want: false, 63 | }, 64 | } 65 | 66 | for _, tt := range tests { 67 | t.Run(tt.name, func(t *testing.T) { 68 | d := &disposal{ 69 | domain: tt.fields.domain, 70 | } 71 | 72 | err := d.domain.Load() 73 | assert.NoError(t, err) 74 | 75 | if got := d.isDisposal(tt.args.email); got != tt.want { 76 | t.Errorf("isDisposal() = %v, want %v", got, tt.want) 77 | } 78 | }) 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /email/email.go: -------------------------------------------------------------------------------- 1 | package email 2 | 3 | import ( 4 | "context" 5 | "crypto/md5" // #nosec 6 | "encoding/hex" 7 | "io/ioutil" 8 | "net" 9 | "net/http" 10 | "net/smtp" 11 | "regexp" 12 | "strings" 13 | "time" 14 | 15 | "github.com/optimatiq/threatbite/email/datasource" 16 | 17 | isd "github.com/jbenet/go-is-domain" 18 | "github.com/labstack/gommon/log" 19 | "golang.org/x/sync/errgroup" 20 | ) 21 | 22 | const pwnedAPI = "https://haveibeenpwned.com/api/v3/breachedaccount/" 23 | 24 | // Info a struct, which contains information about email address. 25 | type Info struct { 26 | EmailScoring uint8 27 | IsDisposal bool 28 | IsDefaultUser bool 29 | IsFree bool 30 | IsValid bool 31 | IsCatchAll bool 32 | IsExistingAccount bool 33 | IsLeaked bool 34 | } 35 | 36 | // Email container for email service. 37 | type Email struct { 38 | pwnedKey string 39 | smtpHello string 40 | smtpFrom string 41 | disposal *disposal 42 | free *free 43 | } 44 | 45 | // NewEmail returns email service, which is used to get detailed information about email address. 46 | func NewEmail(pwnedKey, smtpHello, smtpFrom string, disposalSources, freeSources datasource.DataSource) *Email { 47 | if pwnedKey == "" { 48 | log.Infof("[email] Haveibeenpwned license is not present, reputation accuracy is degraded.") 49 | } 50 | 51 | if smtpFrom == "" || smtpHello == "" { 52 | log.Infof("[email] SMTP configuration is not present, reputation accuracy is degraded.") 53 | } 54 | 55 | return &Email{ 56 | pwnedKey: pwnedKey, 57 | smtpFrom: smtpFrom, 58 | smtpHello: smtpHello, 59 | disposal: newDisposal(disposalSources), 60 | free: newFree(freeSources), 61 | } 62 | } 63 | 64 | // GetInfo returns computed information (Info struct) for given email address. 65 | func (e *Email) GetInfo(email string) Info { 66 | var g errgroup.Group 67 | 68 | var isDisposal bool 69 | g.Go(func() (err error) { 70 | isDisposal = e.disposal.isDisposal(email) 71 | return 72 | }) 73 | 74 | var isUserDefault bool 75 | g.Go(func() (err error) { 76 | isUserDefault = e.isUserDefault(email) 77 | return 78 | }) 79 | 80 | var isFree bool 81 | g.Go(func() (err error) { 82 | isFree = e.free.isFree(email) 83 | return 84 | }) 85 | 86 | // isRFC used in isValid 87 | var isRFC bool 88 | g.Go(func() (err error) { 89 | isRFC = e.isRFC(email) 90 | return 91 | }) 92 | 93 | // isDomainIANA used in isValid 94 | var isDomainIANA bool 95 | g.Go(func() (err error) { 96 | isDomainIANA = e.isDomainIANA(email) 97 | return 98 | }) 99 | 100 | var isCatchAll bool 101 | g.Go(func() (err error) { 102 | isCatchAll = e.isCatchAll(email) 103 | return 104 | }) 105 | 106 | var isExisting bool 107 | g.Go(func() (err error) { 108 | isExisting = e.isExisting(email) 109 | return 110 | }) 111 | 112 | var isPwned bool 113 | g.Go(func() (err error) { 114 | isPwned = e.isPwned(email) 115 | return 116 | }) 117 | 118 | _ = g.Wait() // none of the goroutines return error, so we don't need to check it. 119 | 120 | var isValid bool 121 | if isRFC && isDomainIANA { 122 | isValid = true 123 | } 124 | 125 | // Calculate scoring 0-100 (worst-best) 126 | var scoring uint8 = 80 127 | 128 | // Free email accounts have reduced trust 129 | if isFree { 130 | scoring -= 10 131 | } 132 | if !isFree { 133 | scoring += 10 134 | } 135 | 136 | // Default users have very low trust 137 | if isUserDefault { 138 | scoring -= 35 139 | } 140 | if !isUserDefault { 141 | scoring += 3 142 | } 143 | 144 | if isDisposal { 145 | scoring -= 45 146 | } 147 | if !isDisposal { 148 | scoring += 4 149 | } 150 | 151 | if isCatchAll { 152 | scoring -= 30 153 | } 154 | if !isCatchAll { 155 | scoring += 8 156 | } 157 | 158 | // If email has leaked it means that it has history and exists 159 | if isPwned { 160 | scoring += 3 161 | } 162 | if !isPwned { 163 | scoring-- 164 | } 165 | 166 | if isDomainIANA { 167 | scoring++ 168 | } 169 | if !isDomainIANA { 170 | scoring = 0 171 | } 172 | 173 | if isExisting { 174 | scoring += 2 175 | } 176 | if !isExisting { 177 | scoring = 0 178 | } 179 | 180 | // If email is not proper set to zero 181 | if isRFC { 182 | scoring++ 183 | } 184 | if !isRFC { 185 | scoring = 0 186 | } 187 | 188 | // Maximum value is 100 189 | if scoring > 100 { 190 | scoring = 100 191 | } 192 | 193 | return Info{ 194 | EmailScoring: scoring, 195 | IsDisposal: isDisposal, 196 | IsDefaultUser: isUserDefault, 197 | IsFree: isFree, 198 | IsValid: isValid, 199 | IsCatchAll: isCatchAll, 200 | IsExistingAccount: isExisting, 201 | IsLeaked: isPwned, 202 | } 203 | } 204 | 205 | // isRFC checks if the length of "local part" (before the "@") which maximum is 64 characters (octets) 206 | // and the length of domain part (after the "@") which maximum is 255 characters (octets) 207 | func (e *Email) isRFC(email string) bool { 208 | valid := len(strings.Split(email, "@")[0]) <= 64 && len(strings.Split(email, "@")[1]) <= 255 209 | log.Debugf("[isRFC] email: %s - %t", email, valid) 210 | return valid 211 | } 212 | 213 | // isDomainIANA checks if TLD is on IANA http://data.iana.org/TLD/tlds-alpha-by-domain.txt 214 | func (e *Email) isDomainIANA(email string) bool { 215 | valid := isd.IsDomain(strings.Split(email, "@")[1]) 216 | log.Debugf("[isDomainIANA] email: %s - %t", email, valid) 217 | return valid 218 | } 219 | 220 | // isUserDefault checks is user Name is default, suspicious or commonly used as spamtrap 221 | func (e *Email) isUserDefault(email string) bool { 222 | user := strings.Split(email, "@")[0] 223 | _, found := defaultUsernames[strings.ToLower(user)] 224 | log.Debugf("[isUserDefault] email: %s - %t", email, found) 225 | return found 226 | } 227 | 228 | // checkDomainMX checks if domain have configured MX record and returns IP with the highest priority 229 | func (e *Email) getDomainMX(email string) (string, error) { 230 | mxRecords, err := lookupMXWithTimeout(strings.Split(email, "@")[1], 1*time.Second) 231 | log.Debugf("[checkDomainMX] email: %s mxRecords: %s, error: %s", email, mxRecords, err) 232 | if err != nil { 233 | return "", err 234 | } 235 | return mxRecords[0].Host, nil 236 | } 237 | 238 | // getDomainIP checks if domain has at least one A record and return it 239 | func (e *Email) getDomainIP(email string) (string, error) { 240 | records, err := lookupIPWithTimeout(strings.Split(email, "@")[1], 1*time.Second) 241 | log.Debugf("[getDomainIP] email: %s records: %v, error: %s", email, records, err) 242 | if err != nil { 243 | return "", err 244 | } 245 | 246 | return records[0].String(), nil 247 | } 248 | 249 | // getRandomUser generates random user name 250 | func (e *Email) getRandomUser() string { 251 | now := time.Now() 252 | h := md5.New() // #nosec 253 | _, err := h.Write([]byte(now.String())) 254 | if err != nil { 255 | return now.Format("01-02-2006") 256 | } 257 | return hex.EncodeToString(h.Sum(nil)) 258 | } 259 | 260 | // isCatchAll checks if remote server is configured as Catch all 261 | func (e *Email) isCatchAll(email string) bool { 262 | domain := strings.ToLower(strings.Split(email, "@")[1]) 263 | return e.isExisting(e.getRandomUser() + "@" + domain) 264 | } 265 | 266 | var reSMTP4xx = regexp.MustCompile("^4") 267 | var reSMTP5xx = regexp.MustCompile("^5") 268 | 269 | // isExisting checks if account exists on remote server 270 | func (e *Email) isExisting(email string) bool { 271 | if e.smtpHello == "" || e.smtpFrom == "" { 272 | log.Debug("[isExisting] SMTP is not configured, not checking") 273 | return false 274 | } 275 | 276 | /* 277 | We are working as MTA (server) so by default we are connecting to 25/tcp without TLS 278 | SMTP session example: 279 | 280 | $ telnet example.org 25 281 | S: 220 example.org ESMTP Sendmail 8.13.1/8.13.1; Wed, 30 Aug 2006 07:36:42 -0400 282 | C: HELO mailout1.phrednet.com 283 | S: 250 example.org Hello ip068.subnet71.gci-net.com [216.183.71.68], pleased to meet you 284 | C: MAIL FROM: 285 | S: 250 2.1.0 ... Sender ok 286 | C: RCPT TO: 287 | S: 250 2.1.5 ... Recipient ok 288 | C: DATA 289 | S: 354 Enter mail, end with "." on a line by itself 290 | From: Dave\r\nTo: David\r\nSubject: Its not SPAM\r\n\r\nThis is message 1.\r\n.\r\n 291 | S: 250 2.0.0 k7TKIBYb024731 Message accepted for delivery 292 | C: QUIT 293 | S: 221 2.0.0 example.org closing connection 294 | Connection closed by foreign host. 295 | $ 296 | 297 | In this implementation we are sending only 'MAIL FROM' and 'RCPT TO' to get 250 response. In this case recipient 298 | will not receive any notification because we are not sending 'DATA' command. 299 | 300 | Proper SMTP session ends with 'QUIT' command. To avoid logging on the remote server, we close the connection 301 | without sending the 'QUIT' command. 302 | 303 | - First 'HELO' command must have FQDN argument. Many servers are checking (to block spammers) if this name exists 304 | and is directed to the IP address from which the connection comes. 305 | 306 | - Email used in 'MAIL FROM' must have a domain that exists. Many anti-spam systems also check SPF and DKIM if they 307 | are configured in DNS. 308 | 309 | - IP address from which the connection comes must have configured PTR (revDNS) and domain. 310 | 311 | - RevDNS shouldn't have strings like `vps`, `cloud`, `virtual` or `static` in the name. Such connection can be 312 | blocked by the remote server (like mail.ru). 313 | 314 | - Before starting using IP check if it's not listed on remote RBL 315 | */ 316 | 317 | lowerEmail := strings.ToLower(email) 318 | 319 | var connHost string 320 | 321 | connMX, errMX := e.getDomainMX(lowerEmail) 322 | if errMX != nil { 323 | connIP, errIP := e.getDomainIP(lowerEmail) 324 | if errIP != nil { 325 | return false 326 | } 327 | if connIP != "" { 328 | connHost = connIP 329 | } 330 | } 331 | if connMX != "" { 332 | connHost = connMX 333 | } 334 | 335 | // TODO(PG) Check 465, 587 and STARTTLS 336 | connDial, err := net.DialTimeout("tcp", connHost+":25", time.Duration(3)*time.Second) 337 | if err != nil { 338 | log.Debugf("[isExisting] email: %s host: %v, error: %s", email, connHost, err) 339 | return false 340 | } 341 | 342 | connSMTP, err := smtp.NewClient(connDial, connHost+":25") 343 | if err != nil { 344 | log.Debugf("[isExisting] email: %s host: %v, error: %s", email, connHost, err) 345 | return false 346 | } 347 | 348 | if err := connSMTP.Hello(e.smtpHello); err != nil { 349 | log.Debugf("[isExisting] email: %s host: %v, error: %s", email, connHost, err) 350 | return false 351 | } 352 | 353 | if err := connSMTP.Mail(e.smtpFrom); err != nil { 354 | log.Debugf("[isExisting] email: %s host: %v, error: %s", email, connHost, err) 355 | return false 356 | } 357 | 358 | resp := connSMTP.Rcpt(lowerEmail) 359 | if resp != nil { 360 | if reSMTP5xx.MatchString(resp.Error()) { 361 | log.Debugf("[isExisting] email: %s host: %v, error: %s", email, connHost, err) 362 | return false 363 | } 364 | 365 | if reSMTP4xx.MatchString(resp.Error()) { 366 | // We can make another test after 1 min to bypass Grey Listing 367 | // But we need to implement tokens and repeat this test from the same IP 368 | log.Debugf("[isExisting] email: %s host: %v, error: %s", email, connHost, err) 369 | } 370 | } 371 | 372 | err = connSMTP.Close() 373 | if err != nil { 374 | log.Debugf("[isExisting] email: %s host: %v, error: %s", email, connHost, err) 375 | return false 376 | } 377 | 378 | return true 379 | } 380 | 381 | func (e *Email) isPwned(email string) bool { 382 | var netClient = &http.Client{ 383 | Timeout: time.Second * 30, 384 | } 385 | 386 | request, err := http.NewRequest("GET", pwnedAPI+email, nil) 387 | if err != nil { 388 | log.Debugf("[isPwned] cannot prepare request, error: %s", err) 389 | return false 390 | } 391 | request.Header.Set("hibp-api-key", e.pwnedKey) 392 | 393 | response, err := netClient.Do(request) 394 | if err != nil { 395 | log.Errorf("[isPwned] cannot make request, error: %s", err) 396 | return false 397 | } 398 | 399 | if response.StatusCode != http.StatusOK { 400 | log.Errorf("[isPwned] invalid status code %d", response.StatusCode) 401 | return false 402 | } 403 | 404 | defer response.Body.Close() 405 | body, err := ioutil.ReadAll(response.Body) 406 | if err != nil { 407 | log.Debugf("[isPwned] cannot read response, error: %s", err) 408 | return false 409 | } 410 | 411 | if len(body) > 0 { 412 | log.Debugf("[isPwned] email: %s - %s", email, body) 413 | return true 414 | } 415 | 416 | return false 417 | } 418 | 419 | // RunUpdates schedules and runs updates. 420 | // Update interval is defined for each source individually. 421 | func (e *Email) RunUpdates() { 422 | ctx := context.Background() 423 | 424 | runAndSchedule(ctx, 24*time.Hour, func() { 425 | if err := e.disposal.domain.Load(); err != nil { 426 | log.Error(err) 427 | } 428 | }) 429 | 430 | runAndSchedule(ctx, 24*time.Hour, func() { 431 | if err := e.free.domain.Load(); err != nil { 432 | log.Error(err) 433 | } 434 | }) 435 | } 436 | 437 | func runAndSchedule(ctx context.Context, interval time.Duration, f func()) { 438 | go func() { 439 | t := time.NewTimer(0) // first run - immediately 440 | for { 441 | select { 442 | case <-t.C: 443 | f() 444 | t = time.NewTimer(interval) // next runs according to the schedule 445 | case <-ctx.Done(): 446 | return 447 | } 448 | } 449 | }() 450 | } 451 | -------------------------------------------------------------------------------- /email/email_test.go: -------------------------------------------------------------------------------- 1 | package email 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/optimatiq/threatbite/email/datasource" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func Test_checkLocalLength(t *testing.T) { 12 | tests := map[string]bool{ 13 | "mail@example.com": true, 14 | "m_il.MaIl@exam_le.com": true, 15 | "63.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.@ExapLe.CoM": true, 16 | "64.M.aI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.@ExapLe.CoM": true, 17 | "65.M.aI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.L@ExapLe.CoM": false, 18 | "66.M.aI.M.aI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.MaI.L@ExapLe.CoM": false, 19 | } 20 | 21 | e := NewEmail("", "D", "", nil, nil) 22 | for mail, v := range tests { 23 | assert.Equal(t, v, e.isRFC(mail), mail) 24 | } 25 | } 26 | 27 | func Test_checkDomainLength(t *testing.T) { 28 | tests := map[string]bool{ 29 | "mail@example.com": true, 30 | "m_il.MaIl@exam-le.com": true, 31 | "Mail@255e.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.N.com": true, 32 | "Mail@256e.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Name.Na.com": false, 33 | } 34 | 35 | e := NewEmail("", "D", "", nil, nil) 36 | for mail, v := range tests { 37 | assert.Equal(t, v, e.isRFC(mail), mail) 38 | } 39 | 40 | assert.Panics(t, func() { e.isRFC("invalid_mail") }) 41 | } 42 | 43 | func Test_checkDomainIANA(t *testing.T) { 44 | tests := map[string]bool{ 45 | "mail@example.com": true, 46 | "m_il.MaIl@exam-le.com": true, 47 | "Mail@example.Com": true, 48 | "Mail@example.Pizza1": false, 49 | "Mail@0.0": false, 50 | } 51 | 52 | e := NewEmail("", "D", "", nil, nil) 53 | for mail, v := range tests { 54 | assert.Equal(t, v, e.isDomainIANA(mail), mail) 55 | } 56 | 57 | assert.Panics(t, func() { e.isDomainIANA("invalid_mail") }) 58 | } 59 | 60 | func Test_checkUserDefault(t *testing.T) { 61 | tests := map[string]bool{ 62 | "mail@example.com": true, 63 | "anti.spam@exam-le.com": false, 64 | "default@example.Com": true, 65 | "DeFAult@example.com": true, 66 | "AntiSpam@example.com": true, 67 | } 68 | 69 | e := NewEmail("", "D", "", nil, nil) 70 | for mail, v := range tests { 71 | assert.Equal(t, e.isUserDefault(mail), v, mail) 72 | } 73 | } 74 | 75 | func Test_checkDisposal(t *testing.T) { 76 | tests := map[string]bool{ 77 | "foo@0-mail.com": true, 78 | "anti.spam@example-reputation.com": false, 79 | "default@niepodam.pl": true, 80 | "DeFAult@NiEpOdam.PL": true, 81 | "AntiSpam@126.COM": true, 82 | } 83 | 84 | e := NewEmail("", "", "", datasource.NewListDataSource([]string{"0-mail.com", "niepodam.pl", "126.com"}), datasource.NewEmptyDataSource()) 85 | err := e.disposal.domain.Load() 86 | assert.NoError(t, err) 87 | 88 | for mail, v := range tests { 89 | assert.Equal(t, v, e.disposal.isDisposal(mail), mail) 90 | } 91 | } 92 | 93 | func Test_checkFree(t *testing.T) { 94 | tests := map[string]bool{ 95 | "foo@wp.pl": true, 96 | "anti.spam@example-reputation.com": false, 97 | "default@gmail.com": true, 98 | "DeFAult@GmAil.Com": true, 99 | "AntiSpam@YAHOO.COM": true, 100 | } 101 | e := NewEmail("", "", "", datasource.NewEmptyDataSource(), datasource.NewListDataSource([]string{"wp.pl", "gmail.com", "YAHOO.COM"})) 102 | err := e.free.domain.Load() 103 | assert.NoError(t, err) 104 | 105 | for mail, v := range tests { 106 | assert.Equal(t, v, e.free.isFree(mail), mail) 107 | } 108 | } 109 | 110 | func Test_isPwned(t *testing.T) { 111 | tests := map[string]bool{ 112 | "default@gmail.com": false, 113 | } 114 | e := NewEmail("invalid_key", "D", "", nil, nil) 115 | for mail, v := range tests { 116 | assert.Equal(t, v, e.isPwned(mail), mail) 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /email/free.go: -------------------------------------------------------------------------------- 1 | package email 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | 7 | "github.com/labstack/gommon/log" 8 | "github.com/optimatiq/threatbite/email/datasource" 9 | ) 10 | 11 | type free struct { 12 | domain *datasource.Domain 13 | } 14 | 15 | func newFree(source datasource.DataSource) *free { 16 | return &free{domain: datasource.NewDomain(source, "disposal")} 17 | } 18 | 19 | var reFreeSubDomains = regexp.MustCompile(".hub.pl$|.int.pl$") 20 | 21 | func (d *free) isFree(email string) bool { 22 | domain := strings.ToLower(strings.Split(email, "@")[1]) 23 | isfree := d.domain.Check(domain) || reFreeSubDomains.MatchString(domain) 24 | log.Debugf("[isFree] domain: %s free: %t", domain, isfree) 25 | return isfree 26 | } 27 | -------------------------------------------------------------------------------- /email/free_test.go: -------------------------------------------------------------------------------- 1 | package email 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/optimatiq/threatbite/email/datasource" 9 | ) 10 | 11 | func Test_free_isFree(t *testing.T) { 12 | type fields struct { 13 | domain *datasource.Domain 14 | } 15 | type args struct { 16 | email string 17 | } 18 | tests := []struct { 19 | name string 20 | fields fields 21 | args args 22 | want bool 23 | }{ 24 | { 25 | name: "on list", 26 | fields: fields{ 27 | domain: datasource.NewDomain(datasource.NewListDataSource([]string{"gmail.com"}), "free"), 28 | }, 29 | args: args{ 30 | email: "xxx@gmail.com", 31 | }, 32 | want: true, 33 | }, 34 | { 35 | name: "on list 2", 36 | fields: fields{ 37 | domain: datasource.NewDomain(datasource.NewListDataSource([]string{"xxx.com", "gmail.com"}), "free"), 38 | }, 39 | args: args{ 40 | email: "xxx@gmail.com", 41 | }, 42 | want: true, 43 | }, 44 | { 45 | name: "on list case sensitive", 46 | fields: fields{ 47 | domain: datasource.NewDomain(datasource.NewListDataSource([]string{"GMAIL.com"}), "free"), 48 | }, 49 | args: args{ 50 | email: "xxx@gmail.COM", 51 | }, 52 | want: true, 53 | }, 54 | { 55 | name: "not on list", 56 | fields: fields{ 57 | domain: datasource.NewDomain(datasource.NewListDataSource([]string{"zzz.com"}), "free"), 58 | }, 59 | args: args{ 60 | email: "xxx@xxx.com", 61 | }, 62 | want: false, 63 | }, 64 | } 65 | for _, tt := range tests { 66 | t.Run(tt.name, func(t *testing.T) { 67 | d := &free{ 68 | domain: tt.fields.domain, 69 | } 70 | 71 | err := d.domain.Load() 72 | assert.NoError(t, err) 73 | 74 | if got := d.isFree(tt.args.email); got != tt.want { 75 | t.Errorf("isFree() = %v, want %v", got, tt.want) 76 | } 77 | }) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /email/net.go: -------------------------------------------------------------------------------- 1 | package email 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "time" 7 | ) 8 | 9 | func lookupMXWithTimeout(name string, timeout time.Duration) ([]*net.MX, error) { 10 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 11 | defer cancel() 12 | 13 | return (&net.Resolver{}).LookupMX(ctx, name) 14 | } 15 | 16 | func lookupIPWithTimeout(host string, timeout time.Duration) ([]net.IP, error) { 17 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 18 | defer cancel() 19 | 20 | addrs, err := (&net.Resolver{}).LookupIPAddr(ctx, host) 21 | if err != nil { 22 | return nil, err 23 | } 24 | ips := make([]net.IP, len(addrs)) 25 | for i, ia := range addrs { 26 | ips[i] = ia.IP 27 | } 28 | return ips, nil 29 | } 30 | -------------------------------------------------------------------------------- /email/username.go: -------------------------------------------------------------------------------- 1 | package email 2 | 3 | var defaultUsernames = map[string]bool{ 4 | "abuse": true, 5 | "admin": true, 6 | "administrator": true, 7 | "antispam": true, 8 | "blackhole": true, 9 | "daemon": true, 10 | "default": true, 11 | "dns": true, 12 | "drop": true, 13 | "dump": true, 14 | "dumper": true, 15 | "dupa": true, 16 | "mail": true, 17 | "maildrop": true, 18 | "mailer-agent": true, 19 | "mailer-daemon": true, 20 | "maileragent": true, 21 | "mailerdaemon": true, 22 | "manager": true, 23 | "newsletter": true, 24 | "no-body": true, 25 | "nobody": true, 26 | "no-reply": true, 27 | "noreply": true, 28 | "null": true, 29 | "postdrop": true, 30 | "postfix": true, 31 | "postmaster": true, 32 | "root": true, 33 | "sendmail": true, 34 | "spam": true, 35 | "spool": true, 36 | "test": true, 37 | "testing": true, 38 | } 39 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/optimatiq/threatbite 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/BobuSumisu/aho-corasick v0.0.0-20190714010706-87defef828b4 7 | github.com/alecthomas/units v0.0.0-20190910110746-680d30ca3117 // indirect 8 | github.com/asergeyev/nradix v0.0.0-20170505151046-3872ab85bb56 9 | github.com/avct/uasurfer v0.0.0-20190821150637-906cc7dc6197 10 | github.com/coinpaprika/ratelimiter v0.2.1 11 | github.com/go-playground/locales v0.12.1 // indirect 12 | github.com/go-playground/universal-translator v0.16.0 // indirect 13 | github.com/go-playground/validator v9.31.0+incompatible 14 | github.com/hashicorp/golang-lru v0.5.4 15 | github.com/jbenet/go-is-domain v1.0.3 16 | github.com/joho/godotenv v1.3.0 17 | github.com/labstack/echo-contrib v0.9.0 18 | github.com/labstack/echo/v4 v4.1.16 19 | github.com/labstack/gommon v0.3.0 20 | github.com/leodido/go-urn v1.1.0 // indirect 21 | github.com/oschwald/geoip2-golang v1.4.0 22 | github.com/patrickmn/go-cache v2.1.0+incompatible 23 | github.com/prometheus/client_golang v1.5.0 // indirect 24 | github.com/prometheus/common v0.9.1 25 | github.com/stretchr/testify v1.5.1 26 | golang.org/x/crypto v0.0.0-20200406173513-056763e48d71 27 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect 28 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e 29 | golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4 // indirect 30 | gopkg.in/go-playground/assert.v1 v1.2.1 // indirect 31 | ) 32 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BobuSumisu/aho-corasick v0.0.0-20190714010706-87defef828b4 h1:QtiAfn7n2K1ZYdqSXdI5wQIffCz+qNrC9xBKZuoDUCc= 2 | github.com/BobuSumisu/aho-corasick v0.0.0-20190714010706-87defef828b4/go.mod h1:Dxyzfcjf0gjM2unxeTb4VCdCb3DOdY2j6lfU9ERnDco= 3 | github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= 4 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= 5 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 6 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= 7 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 8 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= 9 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 10 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 11 | github.com/alecthomas/units v0.0.0-20190910110746-680d30ca3117 h1:aUo+WrWZtRRfc6WITdEKzEczFRlEpfW15NhNeLRc17U= 12 | github.com/alecthomas/units v0.0.0-20190910110746-680d30ca3117/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= 13 | github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4= 14 | github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= 15 | github.com/asergeyev/nradix v0.0.0-20170505151046-3872ab85bb56 h1:Wi5Tgn8K+jDcBYL+dIMS1+qXYH2r7tpRAyBgqrWfQtw= 16 | github.com/asergeyev/nradix v0.0.0-20170505151046-3872ab85bb56/go.mod h1:8BhOLuqtSuT5NZtZMwfvEibi09RO3u79uqfHZzfDTR4= 17 | github.com/avct/uasurfer v0.0.0-20190821150637-906cc7dc6197 h1:E7XoJNlFlrtC6dlRG9SoGBwJpX8vD7fNTKRmgSwAj5I= 18 | github.com/avct/uasurfer v0.0.0-20190821150637-906cc7dc6197/go.mod h1:noBAuukeYOXa0aXGqxr24tADqkwDO2KRD15FsuaZ5a8= 19 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 20 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 21 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 22 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 23 | github.com/casbin/casbin/v2 v2.0.0/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= 24 | github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= 25 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 26 | github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= 27 | github.com/coinpaprika/ratelimiter v0.2.1 h1:bng1NJ4CvGcLyksdRLoDzWoURISYV5O95XPS/gS2FJc= 28 | github.com/coinpaprika/ratelimiter v0.2.1/go.mod h1:kM1nyPuy2WiAjo7nCOCxMF+0xR6tzcrl1bVgiyPvx5g= 29 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 30 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 31 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 32 | github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= 33 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 34 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 35 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 36 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 37 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 38 | github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc= 39 | github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= 40 | github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM= 41 | github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= 42 | github.com/go-playground/validator v9.31.0+incompatible h1:UA72EPEogEnq76ehGdEDp4Mit+3FDh548oRqwVgNsHA= 43 | github.com/go-playground/validator v9.31.0+incompatible/go.mod h1:yrEkQXlcI+PugkyDjY2bRrL/UBU4f3rvrgkN3V8JEig= 44 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 45 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 46 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 47 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 48 | github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= 49 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 50 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 51 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 52 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 53 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 54 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 55 | github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= 56 | github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= 57 | github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= 58 | github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= 59 | github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 60 | github.com/jbenet/go-is-domain v1.0.3 h1:FuRBJ0h79p00eseyaLckJT5KnE8RyqI+HLopvNSyNE0= 61 | github.com/jbenet/go-is-domain v1.0.3/go.mod h1:xbRLRb0S7FgzDBTJlguhDVwLYM/5yNtvktxj2Ttfy7Q= 62 | github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= 63 | github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= 64 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 65 | github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 66 | github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= 67 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 68 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 69 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 70 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 71 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 72 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 73 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 74 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 75 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 76 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 77 | github.com/labstack/echo-contrib v0.9.0 h1:hKBA2SnxdxR7sghH0J04zq/pImnKRmgvmQ6MvY9hug4= 78 | github.com/labstack/echo-contrib v0.9.0/go.mod h1:TsFE5Vv0LRpZLoh4mMmaaAxzcTH+1CBFiUtVhwlegzU= 79 | github.com/labstack/echo/v4 v4.1.6/go.mod h1:kU/7PwzgNxZH4das4XNsSpBSOD09XIF5YEPzjpkGnGE= 80 | github.com/labstack/echo/v4 v4.1.16 h1:8swiwjE5Jkai3RPfZoahp8kjVCRNq+y7Q0hPji2Kz0o= 81 | github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI= 82 | github.com/labstack/gommon v0.2.9/go.mod h1:E8ZTmW9vw5az5/ZyHWCp0Lw4OH2ecsaBP1C/NKavGG4= 83 | github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= 84 | github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= 85 | github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8= 86 | github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= 87 | github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= 88 | github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 89 | github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= 90 | github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= 91 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 92 | github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= 93 | github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= 94 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 95 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 96 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 97 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 98 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 99 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 100 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 101 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 102 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 103 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 104 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 105 | github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 106 | github.com/oschwald/geoip2-golang v1.4.0 h1:5RlrjCgRyIGDz/mBmPfnAF4h8k0IAcRv9PvrpOfz+Ug= 107 | github.com/oschwald/geoip2-golang v1.4.0/go.mod h1:8QwxJvRImBH+Zl6Aa6MaIcs5YdlZSTKtzmPGzQqi9ng= 108 | github.com/oschwald/maxminddb-golang v1.6.0 h1:KAJSjdHQ8Kv45nFIbtoLGrGWqHFajOIm7skTyz/+Dls= 109 | github.com/oschwald/maxminddb-golang v1.6.0/go.mod h1:DUJFucBg2cvqx42YmDa/+xHvb0elJtOm3o4aFQ/nb/w= 110 | github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= 111 | github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= 112 | github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= 113 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 114 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 115 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 116 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 117 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 118 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 119 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 120 | github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= 121 | github.com/prometheus/client_golang v1.5.0 h1:Ctq0iGpCmr3jeP77kbF2UxgvRwzWWz+4Bh9/vJTyg1A= 122 | github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= 123 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 124 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= 125 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 126 | github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= 127 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 128 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 129 | github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= 130 | github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= 131 | github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= 132 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 133 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 134 | github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= 135 | github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= 136 | github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= 137 | github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= 138 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 139 | github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= 140 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 141 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 142 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 143 | github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= 144 | github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= 145 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 146 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 147 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 148 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 149 | github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= 150 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 151 | github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= 152 | github.com/uber/jaeger-client-go v2.19.1-0.20191002155754-0be28c34dabf+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= 153 | github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= 154 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 155 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 156 | github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= 157 | github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= 158 | github.com/valyala/fasttemplate v1.1.0 h1:RZqt0yGBsps8NGvLSGW804QQqCUYYLsaOjTVHy1Ocw4= 159 | github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= 160 | go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 161 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 162 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 163 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 164 | golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= 165 | golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 166 | golang.org/x/crypto v0.0.0-20200406173513-056763e48d71 h1:DOmugCavvUtnUD114C1Wh+UgTgQZ4pMLzXxi1pSt+/Y= 167 | golang.org/x/crypto v0.0.0-20200406173513-056763e48d71/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 168 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 169 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 170 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= 171 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 172 | golang.org/x/net v0.0.0-20190607181551-461777fb6f67/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 173 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 174 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= 175 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 176 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= 177 | golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 178 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 179 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= 180 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 181 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 182 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= 183 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 184 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 185 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 186 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 187 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 188 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 189 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 190 | golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 191 | golang.org/x/sys v0.0.0-20190609082536-301114b31cce/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 192 | golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 193 | golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= 194 | golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 195 | golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 196 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 197 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= 198 | golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 199 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= 200 | golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 201 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 202 | golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4 h1:c1Sgqkh8v6ZxafNGG64r8C8UisIW2TKMJN8P86tKjr0= 203 | golang.org/x/sys v0.0.0-20200406155108-e3b113bbe6a4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 204 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 205 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 206 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 207 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 208 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e h1:FDhOuMEY4JVRztM/gsbk+IKUQ8kj74bxZrgw87eMMVc= 209 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 210 | golang.org/x/tools v0.0.0-20190608022120-eacb66d2a7c3 h1:sU3tSV6wDhWsvf9NjL0FzRjgAmYnQL5NEhdmcN16UEg= 211 | golang.org/x/tools v0.0.0-20190608022120-eacb66d2a7c3/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 212 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 213 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 214 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= 215 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 216 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 217 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 218 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 219 | gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= 220 | gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= 221 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 222 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 223 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 224 | gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= 225 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 226 | gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= 227 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 228 | gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= 229 | gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 230 | -------------------------------------------------------------------------------- /ip/datasource/datasource.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import ( 4 | "errors" 5 | "net" 6 | ) 7 | 8 | // ErrNoData no more date in iterator, means that we finished iterating. 9 | var ErrNoData = errors.New("no more data") 10 | 11 | // ErrInvalidData source is available or data provided in the source were not valid IPv4, IPv6 or CIDR. 12 | // When this error is return Next() method is called again. 13 | var ErrInvalidData = errors.New("invalid data") 14 | 15 | // DataSource defines method for accessing stream of addresses. 16 | type DataSource interface { 17 | // Next returns net.IPNet on success or error. 18 | // ErrNoData and ErrInvalidData can be ignored 19 | Next() (*net.IPNet, error) 20 | Reset() error 21 | } 22 | -------------------------------------------------------------------------------- /ip/datasource/datasource_directory.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "net" 7 | "os" 8 | "path" 9 | "path/filepath" 10 | "strings" 11 | "sync" 12 | ) 13 | 14 | // DirectoryDataSource stores current state (counters, files, scanners) of this source. 15 | type DirectoryDataSource struct { 16 | path string 17 | files []string 18 | f int 19 | fLock sync.Mutex 20 | scanner *bufio.Scanner 21 | file *os.File 22 | } 23 | 24 | // NewDirectoryDataSource returns iterator, which looks for all *.txt files in given directory or error. 25 | // Files should have each IPv4 IPv6 or CIDR in new line. 26 | // Comments are allowed and ignored. Comments start with # at the beginning of the line. 27 | func NewDirectoryDataSource(directory string) (*DirectoryDataSource, error) { 28 | dataSource := &DirectoryDataSource{ 29 | path: path.Join(directory, "*.txt"), 30 | } 31 | 32 | err := dataSource.loadFiles() 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | return dataSource, nil 38 | } 39 | 40 | // Reset rewinds source to the beginning. 41 | func (s *DirectoryDataSource) Reset() error { 42 | return s.loadFiles() 43 | } 44 | 45 | func (s *DirectoryDataSource) loadFiles() error { 46 | s.fLock.Lock() 47 | defer s.fLock.Unlock() 48 | 49 | matches, err := filepath.Glob(s.path) 50 | if err != nil { 51 | return fmt.Errorf("directory: %s, error: %w", s.path, err) 52 | } 53 | 54 | if len(matches) <= 0 { 55 | return fmt.Errorf("no data in the directory: %s, error: %w", s.path, ErrInvalidData) 56 | } 57 | 58 | s.scanner = nil 59 | s.files = matches 60 | s.f = 0 61 | 62 | return nil 63 | } 64 | 65 | // Next returns IP/CIDR, this method knows, which file and line needs to be read. 66 | // ErrNoData is returned when there is no data, this error indicates that we reached the end. 67 | func (s *DirectoryDataSource) Next() (*net.IPNet, error) { 68 | if s.f >= len(s.files) || len(s.files) <= 0 { 69 | return nil, ErrNoData 70 | } 71 | 72 | // #nosec G304 73 | if s.scanner == nil { 74 | filename := s.files[s.f] 75 | file, err := os.Open(filename) 76 | if err != nil { 77 | s.f++ 78 | return nil, fmt.Errorf("filename: %s, error: %w", filename, err) 79 | } 80 | s.scanner = bufio.NewScanner(file) 81 | s.file = file 82 | } 83 | 84 | var line string 85 | for s.scanner.Scan() { 86 | line = s.scanner.Text() 87 | 88 | // Comment 89 | if strings.Index(line, "#") == 0 { 90 | continue 91 | } 92 | 93 | // CIDR 94 | if strings.Contains(line, "/") { 95 | _, ipNet, err := net.ParseCIDR(line) 96 | if err != nil { 97 | return nil, ErrInvalidData 98 | } 99 | return ipNet, nil 100 | } 101 | 102 | // Single IP 103 | ip := net.ParseIP(line) 104 | if ip == nil { 105 | return nil, ErrInvalidData 106 | } 107 | return &net.IPNet{IP: ip, Mask: net.CIDRMask(8*len(ip), 8*len(ip))}, nil 108 | } 109 | 110 | // End of file or error 111 | _ = s.file.Close() 112 | err := s.scanner.Err() 113 | s.scanner = nil 114 | 115 | if err != nil { 116 | return nil, err 117 | } 118 | 119 | s.f++ 120 | return s.Next() 121 | } 122 | -------------------------------------------------------------------------------- /ip/datasource/datasource_empty.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import "net" 4 | 5 | // EmptyDataSource as the name suggest, this data source contains no data. 6 | type EmptyDataSource struct { 7 | } 8 | 9 | // NewEmptyDataSource returns empty data source 10 | func NewEmptyDataSource() *EmptyDataSource { 11 | return &EmptyDataSource{} 12 | } 13 | 14 | // Reset does nothing. 15 | func (s *EmptyDataSource) Reset() error { 16 | return nil 17 | } 18 | 19 | // Next returns ErrNoData error always 20 | func (s *EmptyDataSource) Next() (*net.IPNet, error) { 21 | return nil, ErrNoData 22 | } 23 | -------------------------------------------------------------------------------- /ip/datasource/datasource_list.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net" 7 | "strings" 8 | "sync" 9 | ) 10 | 11 | // ListDataSource stores current state (counters) of this source. 12 | type ListDataSource struct { 13 | i int 14 | iLock sync.Mutex 15 | ips []*net.IPNet 16 | } 17 | 18 | // NewListDataSource first argument is a list of IPs or CIDRs. 19 | // Returns DataSource or error on IP parsing. 20 | func NewListDataSource(list []string) (*ListDataSource, error) { 21 | var ipNets []*net.IPNet 22 | for _, element := range list { 23 | if strings.Contains(element, "/") { 24 | _, ipNet, err := net.ParseCIDR(element) 25 | if err != nil { 26 | return nil, fmt.Errorf("invalid address: %s, error: %w", element, err) 27 | } 28 | 29 | ipNets = append(ipNets, ipNet) 30 | } else { 31 | ip := net.ParseIP(element) 32 | if ip == nil { 33 | return nil, errors.New("invalid address: " + element) 34 | } 35 | 36 | ipNets = append(ipNets, &net.IPNet{IP: ip, Mask: net.CIDRMask(8*len(ip), 8*len(ip))}) 37 | } 38 | } 39 | return &ListDataSource{ 40 | ips: ipNets, 41 | }, nil 42 | } 43 | 44 | // Reset rewinds source to the beginning. 45 | func (s *ListDataSource) Reset() error { 46 | s.iLock.Lock() 47 | defer s.iLock.Unlock() 48 | 49 | s.i = 0 50 | return nil 51 | } 52 | 53 | // Next returns IP/CIDR from the provided list in NewListDataSource method. 54 | // ErrNoData is returned when there is no data, this error indicates that we reached the end. 55 | func (s *ListDataSource) Next() (*net.IPNet, error) { 56 | s.iLock.Lock() 57 | defer s.iLock.Unlock() 58 | 59 | if s.i >= len(s.ips) || len(s.ips) <= 0 { 60 | return nil, ErrNoData 61 | } 62 | 63 | v := s.ips[s.i] 64 | s.i++ 65 | return v, nil 66 | } 67 | -------------------------------------------------------------------------------- /ip/datasource/datasource_test.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "math/rand" 8 | "net" 9 | "os" 10 | "path/filepath" 11 | "strconv" 12 | "strings" 13 | "testing" 14 | "time" 15 | 16 | "github.com/stretchr/testify/suite" 17 | ) 18 | 19 | type DatasourceSuite struct { 20 | suite.Suite 21 | privateRand *rand.Rand 22 | } 23 | 24 | func (suite *DatasourceSuite) SetupTest() { 25 | suite.privateRand = rand.New(rand.NewSource(time.Now().UnixNano())) // #nosec 26 | } 27 | 28 | func (suite *DatasourceSuite) Test_StringDatasource() { 29 | tests := []struct { 30 | list []string 31 | wantErr bool 32 | }{ 33 | { 34 | list: []string{""}, 35 | wantErr: true, 36 | }, 37 | { 38 | list: []string{"a.b.c.d"}, 39 | wantErr: true, 40 | }, 41 | { 42 | list: []string{"127.0.0.1"}, 43 | wantErr: false, 44 | }, 45 | { 46 | list: []string{"127.0.0.1/8"}, 47 | wantErr: false, 48 | }, 49 | { 50 | list: []string{"127.0.0.1/8", "1.2.3.4"}, 51 | wantErr: false, 52 | }, 53 | { 54 | list: []string{"127.0.0.1/8", "1.2.3.4/32"}, 55 | wantErr: false, 56 | }, 57 | { 58 | list: []string{"127.0.0.1/8", "1.2.3.4/32", "invalid"}, 59 | wantErr: true, 60 | }, 61 | { 62 | list: []string{"127.0.0.1/8", "1.2.3.4/32", "invalid_cidr/12"}, 63 | wantErr: true, 64 | }, 65 | } 66 | for _, t := range tests { 67 | _, err := NewListDataSource(t.list) 68 | if t.wantErr { 69 | suite.Error(err) 70 | } else { 71 | suite.NoError(err) 72 | } 73 | } 74 | } 75 | 76 | func (suite *DatasourceSuite) Test_StringDatasourceFunctions() { 77 | ds, err := NewListDataSource([]string{"127.0.0.1/8", "1.2.3.4/32"}) 78 | suite.NoError(err) 79 | ip, err := ds.Next() 80 | suite.NoError(err) 81 | _, expected1, err := net.ParseCIDR("127.0.0.1/8") 82 | suite.NoError(err) 83 | suite.Equal(expected1, ip) 84 | 85 | ip, err = ds.Next() 86 | suite.NoError(err) 87 | _, expected2, err := net.ParseCIDR("1.2.3.4/32") 88 | suite.NoError(err) 89 | suite.Equal(expected2, ip) 90 | 91 | ip, err = ds.Next() 92 | suite.Error(err) 93 | suite.Nil(ip) 94 | 95 | err = ds.Reset() 96 | suite.NoError(err) 97 | 98 | ip, err = ds.Next() 99 | suite.NoError(err) 100 | suite.Equal(expected1, ip) 101 | } 102 | 103 | func (suite *DatasourceSuite) Test_NewDirectoryDatasource() { 104 | dir := suite.createDir(true) 105 | _, err := NewDirectoryDataSource(dir) 106 | suite.NoError(err) 107 | err = os.RemoveAll(dir) 108 | suite.NoError(err) 109 | 110 | dir = suite.createDir(false) 111 | _, err = NewDirectoryDataSource(dir) 112 | suite.Error(err) 113 | err = os.RemoveAll(dir) 114 | suite.NoError(err) 115 | } 116 | 117 | func (suite *DatasourceSuite) Test_NewURLDataSource() { 118 | ds := NewURLDataSource([]string{ 119 | "https://iplists.firehol.org/files/proxz_1d.ipset", 120 | }) 121 | 122 | ip, err := ds.Next() 123 | suite.NoError(err) 124 | suite.NotEmpty(ip) 125 | 126 | ds = NewURLDataSource([]string{ 127 | "invalid", 128 | }) 129 | 130 | suite.NoError(err) 131 | ip, err = ds.Next() 132 | suite.Error(err) 133 | suite.Empty(ip) 134 | } 135 | 136 | func (suite *DatasourceSuite) Test_DirectoryDatasourceNext() { 137 | dir := suite.createDir(true) 138 | ds, err := NewDirectoryDataSource(dir) 139 | suite.NoError(err) 140 | for { 141 | ip, err := ds.Next() 142 | if err == ErrNoData { 143 | break 144 | } else if err == ErrInvalidData { 145 | continue 146 | } 147 | suite.NoError(err) 148 | suite.NotEmpty(ip) 149 | } 150 | err = ds.Reset() 151 | suite.NoError(err) 152 | 153 | for { 154 | ip, err := ds.Next() 155 | if err == ErrNoData { 156 | break 157 | } else if err == ErrInvalidData { 158 | continue 159 | } 160 | suite.NoError(err) 161 | suite.NotEmpty(ip) 162 | } 163 | 164 | err = os.RemoveAll(dir) 165 | suite.NoError(err) 166 | 167 | _, err = NewDirectoryDataSource("\\") 168 | suite.Error(err) 169 | } 170 | 171 | func (suite *DatasourceSuite) createDir(content bool) string { 172 | dir, err := ioutil.TempDir("", "datasource") 173 | suite.NoError(err) 174 | if content { 175 | for _, name := range []string{"1", "2", "3"} { 176 | tmpfn := filepath.Join(dir, name+".txt") 177 | b := bytes.NewBuffer(nil) 178 | for i := 0; i < 100; i++ { 179 | b.Write([]byte("# comment\n")) 180 | b.Write([]byte("invalid data x.x.x.x\n")) 181 | b.Write([]byte(suite.randomIPv4(false) + "\n")) 182 | b.Write([]byte(suite.randomIPv4(true) + "\n")) 183 | b.Write([]byte(suite.randomIPv6() + "\n")) 184 | } 185 | err = ioutil.WriteFile(tmpfn, b.Bytes(), 0600) 186 | suite.NoError(err) 187 | } 188 | } 189 | 190 | return dir 191 | } 192 | 193 | func (suite *DatasourceSuite) randomIPv4(cidr bool) string { 194 | var blocks []string 195 | 196 | for i := 0; i < net.IPv4len; i++ { 197 | number := suite.privateRand.Intn(255) 198 | blocks = append(blocks, strconv.Itoa(number)) 199 | } 200 | 201 | ip := strings.Join(blocks, ".") 202 | if cidr { 203 | return fmt.Sprintf("%s/%d", ip, suite.privateRand.Intn(32)) 204 | } 205 | return ip 206 | } 207 | 208 | func (suite *DatasourceSuite) randomIPv6() string { 209 | var ip net.IP 210 | for i := 0; i < net.IPv6len; i++ { 211 | number := uint8(suite.privateRand.Intn(255)) 212 | ip = append(ip, number) 213 | } 214 | return ip.String() 215 | } 216 | 217 | func TestDatasourceSuite(t *testing.T) { 218 | suite.Run(t, new(DatasourceSuite)) 219 | } 220 | -------------------------------------------------------------------------------- /ip/datasource/datasource_url.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "io/ioutil" 7 | "net" 8 | "net/http" 9 | "strings" 10 | "time" 11 | 12 | "github.com/labstack/gommon/log" 13 | ) 14 | 15 | // URLDataSource stores current state (counters, URLs, scanners) of this source. 16 | type URLDataSource struct { 17 | urls []string 18 | u int 19 | scanner *bufio.Scanner 20 | client *http.Client 21 | } 22 | 23 | // NewURLDataSource returns iterator, which downloads lists from provided URLs and extract addresses. 24 | // Files should have each IPv4 IPv6 or CIDR in new line. 25 | // Comments are allowed and ignored. Comments start with # at the beginning of the line. 26 | // Some lists have comments after their address, they are also ignored 27 | func NewURLDataSource(urls []string) *URLDataSource { 28 | dataSource := &URLDataSource{ 29 | client: &http.Client{ 30 | Transport: &http.Transport{ 31 | DialContext: (&net.Dialer{ 32 | Timeout: 60 * time.Second, 33 | KeepAlive: 15 * time.Second, 34 | }).DialContext, 35 | TLSHandshakeTimeout: 60 * time.Second, 36 | ExpectContinueTimeout: 10 * time.Second, 37 | ResponseHeaderTimeout: 10 * time.Second, 38 | }, 39 | Timeout: 120 * time.Second, 40 | }, 41 | urls: urls, 42 | } 43 | 44 | return dataSource 45 | } 46 | 47 | // Reset rewinds source to the beginning. 48 | func (s *URLDataSource) Reset() error { 49 | s.u = 0 50 | s.scanner = nil 51 | return nil 52 | } 53 | 54 | // Next returns IP/CIDR, this method knows which URL and line needs to be read. 55 | // URLs are downloaded one by one and kept in memory, bufio.NewScanner is used to keep track, which line has to be returned. 56 | // ErrNoData is returned when there is no data, this error indicates that we reached the end. 57 | func (s *URLDataSource) Next() (*net.IPNet, error) { 58 | if s.u >= len(s.urls) || len(s.urls) <= 0 { 59 | return nil, ErrNoData 60 | } 61 | url := s.urls[s.u] 62 | 63 | if s.scanner == nil { 64 | response, err := s.client.Get(url) 65 | if err != nil { 66 | log.Errorf("[datasource] cannot download list from: %s, error: %s", url, err) 67 | s.u++ 68 | return nil, ErrInvalidData 69 | } 70 | 71 | body, err := ioutil.ReadAll(response.Body) 72 | if err != nil { 73 | log.Errorf("[datasource] cannot read from: %s, error: %s", url, err) 74 | s.u++ 75 | return nil, ErrInvalidData 76 | } 77 | 78 | if err := response.Body.Close(); err != nil { 79 | log.Errorf("[datasource] cannot close response body from: %s, error: %s", url, err) 80 | } 81 | 82 | s.scanner = bufio.NewScanner(bytes.NewReader(body)) 83 | } 84 | 85 | var line string 86 | for s.scanner.Scan() { 87 | line = s.scanner.Text() 88 | // some lists have address with optional comment as a second argument separated by spaces or tabs 89 | line = strings.ReplaceAll(line, "\t", " ") 90 | line := strings.Split(line, " ")[0] 91 | 92 | // Comment 93 | if strings.Index(line, "#") == 0 { 94 | continue 95 | } 96 | 97 | // CIDR 98 | if strings.Contains(line, "/") { 99 | _, ipNet, err := net.ParseCIDR(line) 100 | if err != nil { 101 | return nil, ErrInvalidData 102 | } 103 | return ipNet, nil 104 | } 105 | 106 | // Single IP 107 | ip := net.ParseIP(line) 108 | if ip == nil { 109 | return nil, ErrInvalidData 110 | } 111 | return &net.IPNet{IP: ip, Mask: net.CIDRMask(8*len(ip), 8*len(ip))}, nil 112 | } 113 | 114 | err := s.scanner.Err() 115 | s.scanner = nil 116 | 117 | if err != nil { 118 | return nil, err 119 | } 120 | 121 | s.u++ 122 | return s.Next() 123 | } 124 | -------------------------------------------------------------------------------- /ip/datasource/ipnet.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net" 7 | "sync" 8 | "time" 9 | 10 | "github.com/asergeyev/nradix" 11 | "github.com/labstack/gommon/log" 12 | "github.com/patrickmn/go-cache" 13 | ) 14 | 15 | // IPNet container struct for IP/CIDR operations 16 | type IPNet struct { 17 | cache *cache.Cache 18 | cidrs *nradix.Tree 19 | cidrsLock sync.RWMutex 20 | ips map[uint64]bool 21 | ipsLock sync.RWMutex 22 | ds DataSource 23 | name string 24 | } 25 | 26 | // NewIPNet returns a new IP/CIDR list build on top of radix tree (for CIDRS) and go map for IPs. 27 | func NewIPNet(ds DataSource, name string) *IPNet { 28 | return &IPNet{ 29 | cache: cache.New(1*time.Minute, 1*time.Minute), 30 | cidrs: nradix.NewTree(0), 31 | ips: make(map[uint64]bool), 32 | ds: ds, 33 | name: name, 34 | } 35 | } 36 | 37 | // Check if lists contains IP from request 38 | func (l *IPNet) Check(ip net.IP) (bool, error) { 39 | ipString := ip.String() 40 | keyPermBlockIP := "ip_" + ipString 41 | if _, ok := l.cache.Get(keyPermBlockIP); ok { 42 | return true, nil 43 | } 44 | 45 | l.cidrsLock.RLock() 46 | cidrFound, err := l.cidrs.FindCIDR(ipString) 47 | l.cidrsLock.RUnlock() 48 | if err != nil { 49 | return false, fmt.Errorf("could not find element: %s, error: %w", ipString, err) 50 | } 51 | 52 | if cidrFound != nil { 53 | l.cache.Set(keyPermBlockIP, cidrFound, 0) 54 | return true, nil 55 | } 56 | 57 | uip, _ := l.ipToUint64(ip) 58 | l.ipsLock.RLock() 59 | value, ipFound := l.ips[uip] 60 | l.ipsLock.RUnlock() 61 | if ipFound { 62 | l.cache.Set(keyPermBlockIP, value, 0) 63 | return true, nil 64 | } 65 | 66 | return false, nil 67 | } 68 | 69 | // Close clears underlying radix tree. 70 | func (l *IPNet) Close() { 71 | l.ipsLock.Lock() 72 | l.ips = map[uint64]bool{} 73 | l.ipsLock.Unlock() 74 | 75 | l.cidrsLock.Lock() 76 | l.cidrs = nradix.NewTree(0) 77 | l.cidrsLock.Unlock() 78 | 79 | l.cache.Flush() 80 | } 81 | func (l *IPNet) Load() error { 82 | var cidrs int 83 | 84 | cidrsTemp := nradix.NewTree(0) 85 | ipsTemp := map[uint64]bool{} 86 | 87 | log.Debugf("[list] loading %s list start", l.name) 88 | defer func() { 89 | l.ipsLock.Lock() 90 | l.ips = ipsTemp 91 | l.ipsLock.Unlock() 92 | 93 | l.cidrsLock.Lock() 94 | l.cidrs = cidrsTemp 95 | l.cidrsLock.Unlock() 96 | 97 | l.cache.Flush() 98 | 99 | log.Debugf("[list] loading %s stop; stats IPs: %d, CIDRs: %d", l.name, len(l.ips), cidrs) 100 | }() 101 | 102 | if err := l.ds.Reset(); err != nil { 103 | return fmt.Errorf("could not reset data source, error: %w", err) 104 | } 105 | 106 | for { 107 | ipNet, err := l.ds.Next() 108 | if err != nil { 109 | if err == ErrNoData { 110 | return nil 111 | } else if err == ErrInvalidData { 112 | continue 113 | } else { 114 | return fmt.Errorf("could not iterate over data source, error: %w", err) 115 | } 116 | } 117 | 118 | // single IP address, not a CIDR, mask contains only "ones" 119 | if ones, bits := ipNet.Mask.Size(); ones == bits { 120 | i, _ := l.ipToUint64(ipNet.IP) 121 | ipsTemp[i] = true 122 | continue 123 | } 124 | 125 | err = cidrsTemp.AddCIDR(ipNet.String(), true) 126 | if err != nil && err != nradix.ErrNodeBusy { 127 | return fmt.Errorf("could not add IP: %s, error: %w", ipNet.String(), err) 128 | } 129 | cidrs++ 130 | } 131 | } 132 | 133 | func (l *IPNet) ipToUint64(ip net.IP) (uint64, error) { 134 | if to4 := ip.To4(); to4 != nil { 135 | return uint64(to4[0])<<24 | uint64(to4[1])<<16 | uint64(to4[2])<<8 | uint64(to4[3]), nil 136 | } else if to16 := ip.To16(); to16 != nil { 137 | int1 := uint64(0) 138 | for i := 0; i < 8; i++ { 139 | int1 = (int1 << 8) + uint64(ip[i]) 140 | } 141 | return int1, nil 142 | } 143 | return 0, errors.New("could not convert IP address") 144 | } 145 | -------------------------------------------------------------------------------- /ip/datasource/ipnet_test.go: -------------------------------------------------------------------------------- 1 | package datasource 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/suite" 8 | ) 9 | 10 | type ListSuite struct { 11 | suite.Suite 12 | } 13 | 14 | func (suite *ListSuite) Test_CheckMixed() { 15 | tests := []struct { 16 | data []string 17 | check string 18 | want bool 19 | wantErr bool 20 | }{ 21 | 22 | { 23 | []string{"2001:db8:1234::/48", "1.1.1.1/8"}, 24 | "2001:db8:1234:0:0:8a2e:370:7334", 25 | true, 26 | false, 27 | }, 28 | { 29 | []string{"2001:db8:1234::/48", "1.1.1.1/8"}, 30 | "2001:db8:1234:0:0:8a2e:370:7334", 31 | true, 32 | false, 33 | }, 34 | } 35 | for _, t := range tests { 36 | ds, err := NewListDataSource(t.data) 37 | suite.NoError(err) 38 | 39 | ipnet := NewIPNet(ds, "testList") 40 | err = ipnet.Load() 41 | suite.NoError(err) 42 | 43 | v, err := ipnet.Check(net.ParseIP(t.check)) 44 | 45 | if t.wantErr { 46 | suite.Error(err) 47 | } else { 48 | suite.NoError(err) 49 | } 50 | 51 | suite.Equal(t.want, v, t) 52 | } 53 | } 54 | 55 | func (suite *ListSuite) Test_CheckCIDR() { 56 | tests := []struct { 57 | data []string 58 | check string 59 | want bool 60 | wantErr bool 61 | }{ 62 | 63 | { 64 | []string{"2001:db8:1234::/48"}, 65 | "2001:db8:1234:0:0:8a2e:370:7334", 66 | true, 67 | false, 68 | }, 69 | 70 | { 71 | []string{"127.0.0.1/8"}, 72 | "127.0.0.1", 73 | true, 74 | false, 75 | }, 76 | { 77 | []string{"127.0.0.1/8"}, 78 | "127.0.0.2", 79 | true, 80 | false, 81 | }, 82 | { 83 | []string{"127.0.0.1/8"}, 84 | "1.1.1.1", 85 | false, 86 | false, 87 | }, 88 | { 89 | []string{"127.0.0.1/8", "1.1.1.1/16"}, 90 | "1.1.1.1", 91 | true, 92 | false, 93 | }, 94 | { 95 | []string{"127.0.0.1/32", "1.1.1.1/32"}, 96 | "1.1.1.1", 97 | true, 98 | false, 99 | }, 100 | } 101 | for _, t := range tests { 102 | ds, err := NewListDataSource(t.data) 103 | suite.NoError(err) 104 | 105 | ipnet := NewIPNet(ds, "testList") 106 | err = ipnet.Load() 107 | suite.NoError(err) 108 | 109 | v, err := ipnet.Check(net.ParseIP(t.check)) 110 | 111 | if t.wantErr { 112 | suite.Error(err) 113 | } else { 114 | suite.NoError(err) 115 | } 116 | 117 | suite.Equal(t.want, v, t) 118 | } 119 | } 120 | 121 | func (suite *ListSuite) Test_CheckIP() { 122 | tests := []struct { 123 | data []string 124 | check string 125 | want bool 126 | wantErr bool 127 | }{ 128 | { 129 | []string{"127.0.0.1"}, 130 | "127.0.0.1", 131 | true, 132 | false, 133 | }, 134 | { 135 | []string{"127.0.0.1"}, 136 | "127.0.0.2", 137 | false, 138 | false, 139 | }, 140 | { 141 | []string{"127.0.0.1"}, 142 | "1.1.1.1", 143 | false, 144 | false, 145 | }, 146 | { 147 | []string{"127.0.0.1", "1.1.1.1"}, 148 | "1.1.1.1", 149 | true, 150 | false, 151 | }, 152 | } 153 | for _, t := range tests { 154 | ds, err := NewListDataSource(t.data) 155 | suite.NoError(err) 156 | f := NewIPNet(ds, "testList") 157 | 158 | err = f.Load() 159 | suite.NoError(err) 160 | 161 | v, err := f.Check(net.ParseIP(t.check)) 162 | 163 | if t.wantErr { 164 | suite.Error(err) 165 | } else { 166 | suite.NoError(err) 167 | } 168 | 169 | suite.Equal(t.want, v, t) 170 | 171 | // check cache 172 | v, err = f.Check(net.ParseIP(t.check)) 173 | 174 | if t.wantErr { 175 | suite.Error(err) 176 | } else { 177 | suite.NoError(err) 178 | } 179 | 180 | suite.Equal(t.want, v, t) 181 | } 182 | } 183 | 184 | func TestListSuite(t *testing.T) { 185 | suite.Run(t, new(ListSuite)) 186 | } 187 | -------------------------------------------------------------------------------- /ip/dc.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "regexp" 7 | "strings" 8 | "time" 9 | 10 | aho "github.com/BobuSumisu/aho-corasick" 11 | "github.com/labstack/gommon/log" 12 | "github.com/optimatiq/threatbite/ip/datasource" 13 | ) 14 | 15 | type datacenter struct { 16 | ipnet *datasource.IPNet 17 | geoip geoip 18 | } 19 | 20 | func newDC(geoip geoip, source datasource.DataSource) *datacenter { 21 | list := datasource.NewIPNet(source, "datacenter") 22 | return &datacenter{ 23 | ipnet: list, 24 | geoip: geoip, 25 | } 26 | } 27 | 28 | var reIsDC = regexp.MustCompile("server|vps|cloud|web|hosting|virt") 29 | 30 | var trie = aho.NewTrieBuilder(). 31 | AddStrings([]string{ 32 | "1&1", "1and1", "1gb", "21vianet", "23media", "23vnet", "2dayhost", "3nt", 33 | "4rweb", "abdicar", "abelohost", "acceleratebiz", "accelerated", "acenet", "acens", "activewebs.dk", 34 | "adhost", "advancedhosters", "advania", "ainet", "airnet group", "akamai", "alfahosting", "alibaba", 35 | "allhostshop", "almahost", "alog", "alpharacks", "altushost", "alvotech", "amanah", "amazon", 36 | "amerinoc", "anexia", "apollon", "applied", "ardis", "ares", "argeweb", "argon", 37 | "aruba", "arvixe", "atman", "atomohost", "availo.no", "avantehosting", "avguro", "awknet", 38 | "aws", "azar-a", "azure", "b2 net", "basefarm", "beget", "best-hosting", "beyond", 39 | "bhost", "biznes-host", "blackmesh", "blazingfast", "blix", "blue mile", "blueconnex", "bluehost", 40 | "bodhost", "braslink", "brinkster", "budgetbytes", "burstnet", "business", "buyvm", "calpop", 41 | "canaca-com", "carat networks", "cari", "ccpg", "ceu", "ch-center", "choopa", "cinipac", 42 | "cirrus", "cloud", "cloudflare", "cloudsigma", "cloudzilla", "co-location", "codero", "colo", 43 | "colo4dallas", "colo@", "colocall", "colocation", "combell", "comfoplace", "comvive", "conetix", 44 | "confluence", "connectingbytes", "connectingbytse", "connectria", "contabo", "continuum", "coolvds", "corponetsa", 45 | "creanova", "crosspointcolo", "ctrls", "cybernetic-servers", "cyberverse", "cyberwurx", "cyquator", "d-hosting", 46 | "data 102", "data centers", "data foundry", "data shack", "data xata", "data-centr", "data-xata", "database", 47 | "datacenter", "datacenterscanada", "datacheap", "dataclub", "datahata.by", "datahouse.nl", "datapipe", "datapoint", 48 | "datasfera", "datotel", "dedi", "dedibox", "dediserv", "dedizull", "delta bulgaria", "delta-x", 49 | "deltahost", "demos", "deninet hungary", "depo40", "depot", "deziweb", "dfw-datacenter", "dhap", 50 | "digicube", "digital", "digitalocean", "digitalone", "digiweb", "dimenoc", "dinahosting", "directspace", 51 | "directvps", "dominios", "dotster", "dreamhost", "duocast", "duomenu centras lithuania", "e-commercepark", "e24cloud", 52 | "earthlink", "easyhost.be", "easyhost.hk", "easyname", "easyspeedy", "eboundhost", "ecatel", "ecritel", 53 | "edgewebhosting", "edis", "egihosting", "ehostidc", "ehostingusa", "ekvia", "elserver", "elvsoft", 54 | "enzu", "epiohost", "erix-colo", "esc", "esds", "eserver", "esited", "estoxy", 55 | "estroweb", "ethnohosting", "ethr", "eukhost", "euro-web", "eurobyte", "eurohoster", "eurovps", 56 | "everhost", "evovps", "fasthosts", "fastly", "fastmetrics", "fdcservers", "fiberhub", "fibermax", 57 | "finaltek", "firehost", "first colo", "firstvds", "flexwebhosting", "flokinet", "flops", "forpsi", 58 | "forta trust", "fortress", "fsdata", "galahost", "gandi", "gbps", "gearhost", "genesys", 59 | "giga-hosting", "gigahost", "gigenet", "glesys", "go4cloud", "godaddy", "gogrid", "goodnet", 60 | "google", "gorack", "gorilla", "gplhost", "grid", "gyron", "h1 host", "h1host", 61 | "h4hosting", "h88", "heart", "hellovps", "hetzner", "hispaweb", "hitme", "hivelocity", 62 | "home.pl", "homecloud", "homenet", "hopone", "hosixy", "host", "host department", "host virtual", 63 | "host-it", "host1plus", "hosta rica", "hostbasket", "hosteam.pl", "hosted", "hoster", "hosteur", 64 | "hostex", "hostex.lt", "hostgrad", "hosthane", "hostinet", "hosting", "hostinger", "hostkey", 65 | "hostmysite", "hostnet.nl", "hostnoc", "hostpro", "hostrevenda", "hostrocket", "hostventures", "hostway", 66 | "hostwinds", "hqhost", "hugeserver", "hurricane", "hyperhosting", "i3d", "iaas", "icn.bg", 67 | "ideal-solution.org", "idealhosting", "ihc", "ihnetworks", "ikoula", "iliad", "immedion", "imperanet", 68 | "inasset", "incero", "incubatec gmbh - srl", "indiana", "inferno", "infinitetech", "infinitie", "infinys", 69 | "infium-1", "infiumhost", "infobox", "infra", "inline", "inmotion hosting", "integrity", "interhost", 70 | "interracks", "interserver", "iomart", "iomart hosting ltd", "ionity", "ip exchange", "ip server", "ip serverone", 71 | "ipglobe", "iphouse", "ipserver", "ipx", "iqhost", "isppro", "ispserver", "ispsystem", 72 | "itl", "iweb", "iws", "ix-host", "ixam-hosting", "jumpline inc", "justhost", "keyweb", 73 | "kievhosting", "kinx", "knownsrv", "kualo", "kylos.pl", "latisys", "layered", "leaderhost", 74 | "leaseweb", "lightedge", "limelight", "limestone", "link11", "linode", "lionlink", "lippunerhosting", 75 | "liquid", "local", "locaweb", "logicworks", "loopbyte", "loopia", "loose", "lunar", 76 | "main-hosting", "marosnet", "masterhost", "mchost", "media temple", "melbicom", "memset", "mesh", 77 | "mgnhost", "micfo", "micron21", "microsoft", "midphase", "mirahost", "mirohost", "mnogobyte", 78 | "mojohost", "mrhost", "mrhost.biz", "multacom", "mxhost", "my247webhosting", "myh2oservers", "myhost", 79 | "myloc", "nano", "natro", "nbiserv", "ndchost", "nedzone.nl", "neospire", "net4", 80 | "netangels", "netbenefit", "netcup", "netelligent", "netgroup", "netinternet", "netio", "netirons", 81 | "netnation", "netplus", "netriplex", "netrouting", "netsys", "netzozeker.nl", "nforce", "nimbushosting", 82 | "nine.ch", "niobeweb", "nthost", "ntx", "nufuture", "nwt idc", "o2switch", "offshore", 83 | "one", "online", "openhosting", "optimate", "ovh", "ozhosting", "packet", "pair networks", 84 | "panamaserver", "patrikweb", "pce-net", "peak", "peak10", "peer 1", "perfect ip", "peron", 85 | "persona host", "pghosting", "planethoster", "plusserver", "plutex", "portlane", "premia", "prioritycolo", 86 | "private layer", "privatesystems", "profihost", "prohoster", "prolocation", "prometey", "providerdienste", "prq.se", 87 | "psychz", "quadranet", "quasar", "quickweb.nz", "qweb", "qwk", "r01", "rack", 88 | "rackforce", "rackmarkt", "rackplace", "rackspace", "racksrv", "rackvibe", "radore", "rapidhost", 89 | "rapidspeeds", "razor", "readyspace", "realcomm", "rebel", "redehost.br", "redstation", "reflected", 90 | "reg", "register", "regtons", "reliable", "rent", "rijndata.nl", "rimu", "risingnet", 91 | "root", "roya", "rtcomm", "ru-center", "s.r.o.", "saas", "sadecehosting", "safe", 92 | "sakura", "securewebs", "seeweb.it", "seflow", "selectel", "servage", "servenet", "server", 93 | "serverbeach", "serverboost.nl", "servercentral", "servercentric", "serverclub", "serverius", "servermania", "serveroffer", 94 | "serverpronto", "servers", "serverspace", "servhost", "servihosting", "servint", "servinus", "servisweb", 95 | "sevenl", "sharktech", "silicon valley", "simcentric", "simplecloud", "simpliq", "singlehop", "siteserver", 96 | "slask", "small orange", "smart-hosting", "smartape", "snel", "softlayer", "solido", "sologigabit", 97 | "spaceweb", "sparkstation", "sprocket", "staminus", "star-hosting", "steadfast", "steep host", "store", 98 | "strato", "sunnyvision", "superdata", "superhost.pl", "superhosting.bg", "supernetwork", "supreme", "swiftway", 99 | "switch", "switch media", "szervernet", "t-n media", "tagadab", "tailor made", "take 2", "tangram", 100 | "techie media", "technologies", "telecity", "tencent cloud", "tentacle", "teuno", "the bunker", "the endurance", 101 | "theplanet", "thorn", "thrust vps", "tierpoint", "tilaa", "titan internet", "totalin", "trabia", 102 | "tranquil", "transip", "travailsystems", "triple8", "trueserver.nl", "turkiye", "tuxis.nl", "twooit", 103 | "uadomen", "ubiquity", "uk2", "uk2group", "ukwebhosting.ltd.uk", "unbelievable", "unitedcolo", "unithost", 104 | "upcloud", "usonyx", "vautron", "vds64", "veesp", "velia", "velocity", "ventu", 105 | "versaweb", "vexxhost", "vhoster", "virpus", "virtacore", "vnet", "volia", "vooservers", 106 | "voxel", "voxility", "vpls", "vps", "vps4less", "vpscheap", "vpsnet", "vshosting.cz", 107 | "vstoike russia", "web werks", "web2objects", "webair", "webalta", "webaxys", "webcontrol", "webexxpurts", 108 | "webfusion", "webhoster", "webhosting", "webnx", "websitewelcome", "websupport", "webvisions", "wedos", 109 | "weebly", "wehostall", "wehostwebsites", "westhost", "wholesale", "wiredtree", "worldstream", "wow", 110 | "x10hosting", "xentime", "xiolink", "xirra gmbh", "xlhost", "xmission", "xserver", "xservers", 111 | "xt global", "xtraordinary", "yandex", "yeshost", "yisp", "yourcolo", "yourserver", "zare", 112 | "zenlayer", "zet", "zomro", "zservers", 113 | }). 114 | Build() 115 | 116 | func (p *datacenter) isDC(ip net.IP) (bool, error) { 117 | isDC, err := p.ipnet.Check(ip) 118 | if err != nil { 119 | return false, fmt.Errorf("cannot run Check on %s, error: %w", ip, err) 120 | } 121 | if isDC { 122 | log.Debugf("[isDC] ip: %s dc: %t", ip, isDC) 123 | return true, nil 124 | } 125 | 126 | organisation, err := p.geoip.getCompany(ip) 127 | if err != nil { 128 | return false, fmt.Errorf("cannot run getCompanyName on %s, error: %w", ip, err) 129 | } 130 | if organisation != "" { 131 | matches := trie.MatchString(strings.ToLower(organisation)) 132 | if len(matches) > 0 { 133 | return true, nil 134 | } 135 | } 136 | 137 | hostnames, err := lookupAddrWithTimeout(ip.String(), 500*time.Millisecond) 138 | if err != nil { 139 | // errors like "no such host" are normal, we don't need to pollute error logs 140 | log.Debugf("[isDC] ip: %s error: %s", ip, err) 141 | return false, nil 142 | } 143 | 144 | if reIsDC.MatchString(hostnames[0]) { 145 | log.Debugf("[isDC] ip: %s DC match", ip) 146 | return true, nil 147 | } 148 | 149 | return false, nil 150 | } 151 | -------------------------------------------------------------------------------- /ip/dc_test.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | 7 | "github.com/optimatiq/threatbite/ip/datasource" 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/stretchr/testify/mock" 11 | ) 12 | 13 | type mockedGeoip struct { 14 | mock.Mock 15 | } 16 | 17 | func (m *mockedGeoip) getCountry(ip net.IP) (string, error) { 18 | args := m.Called(ip) 19 | return args.String(0), args.Error(1) 20 | } 21 | 22 | func (m *mockedGeoip) getCompany(ip net.IP) (string, error) { 23 | args := m.Called(ip) 24 | return args.String(0), args.Error(1) 25 | } 26 | 27 | func (m *mockedGeoip) update() error { 28 | return nil 29 | } 30 | 31 | func Test_datacenter_isDC(t *testing.T) { 32 | geo := new(mockedGeoip) 33 | geo.On("getCountry", net.ParseIP("1.1.1.1")).Return("PL", nil) 34 | geo.On("getCompany", net.ParseIP("1.1.1.1")).Return("Misc corp.", nil) 35 | 36 | geo.On("getCountry", net.ParseIP("1.1.1.2")).Return("PL", nil) 37 | geo.On("getCompany", net.ParseIP("1.1.1.2")).Return("Misc corp.", nil) 38 | 39 | geo.On("getCountry", net.ParseIP("1.1.1.3")).Return("PL", nil) 40 | geo.On("getCompany", net.ParseIP("1.1.1.3")).Return("OVH corporation", nil) 41 | 42 | type fields struct { 43 | list []string 44 | geoip geoip 45 | } 46 | type args struct { 47 | ip net.IP 48 | } 49 | tests := []struct { 50 | name string 51 | fields fields 52 | args args 53 | want bool 54 | wantErr bool 55 | }{ 56 | { 57 | name: "on list", 58 | fields: fields{ 59 | list: []string{"1.1.1.1"}, 60 | geoip: geo, 61 | }, 62 | args: args{ip: net.ParseIP("1.1.1.1")}, 63 | want: true, 64 | }, 65 | 66 | { 67 | name: "not on list", 68 | fields: fields{ 69 | list: []string{"1.1.1.1"}, 70 | geoip: geo, 71 | }, 72 | args: args{ip: net.ParseIP("1.1.1.2")}, 73 | want: false, 74 | }, 75 | 76 | { 77 | name: "invalid IP", 78 | fields: fields{ 79 | list: []string{}, 80 | geoip: geo, 81 | }, 82 | args: args{ip: net.ParseIP("invalid")}, 83 | wantErr: true, 84 | }, 85 | 86 | { 87 | name: "company on list", 88 | fields: fields{ 89 | list: []string{"1.1.1.1"}, 90 | geoip: geo, 91 | }, 92 | args: args{ip: net.ParseIP("1.1.1.3")}, 93 | want: true, 94 | }, 95 | } 96 | for _, tt := range tests { 97 | t.Run(tt.name, func(t *testing.T) { 98 | ds, err := datasource.NewListDataSource(tt.fields.list) 99 | assert.NoError(t, err) 100 | d := &datacenter{ 101 | ipnet: datasource.NewIPNet(ds, "vpn"), 102 | geoip: tt.fields.geoip, 103 | } 104 | err = d.ipnet.Load() 105 | assert.NoError(t, err) 106 | 107 | got, err := d.isDC(tt.args.ip) 108 | if (err != nil) != tt.wantErr { 109 | t.Errorf("isDC() error = %v, wantErr %v", err, tt.wantErr) 110 | return 111 | } 112 | if got != tt.want { 113 | t.Errorf("isDC() got = %v, want %v", got, tt.want) 114 | } 115 | }) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /ip/geoip.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import "net" 4 | 5 | type geoip interface { 6 | getCountry(ip net.IP) (string, error) 7 | getCompany(ip net.IP) (string, error) 8 | update() error 9 | } 10 | -------------------------------------------------------------------------------- /ip/ip.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "time" 7 | 8 | "github.com/optimatiq/threatbite/ip/datasource" 9 | 10 | "github.com/labstack/gommon/log" 11 | "golang.org/x/sync/errgroup" 12 | ) 13 | 14 | // Info a struct, which contains information about IP address. 15 | type Info struct { 16 | Company string 17 | Country string 18 | Hostnames []string 19 | IsProxy bool 20 | IsSearchEngine bool 21 | IsTor bool 22 | IsPrivate bool 23 | IsDatacenter bool 24 | IsSpam bool 25 | IsVpn bool 26 | IPScoring uint8 27 | } 28 | 29 | // IP container struct for IP service. 30 | type IP struct { 31 | tor *tor 32 | geoip geoip 33 | engine *searchEngine 34 | proxy *proxy 35 | dc *datacenter 36 | spam *spam 37 | vpn *vpn 38 | } 39 | 40 | // NewIP creates a service for getting information about IP address. 41 | func NewIP(maxmindKey string, proxyDs, spamDs, vpnDs, dcDs datasource.DataSource) *IP { 42 | geo := newMaxmind(maxmindKey) 43 | 44 | return &IP{ 45 | geoip: geo, 46 | tor: newTor(), 47 | proxy: newProxy(proxyDs), 48 | engine: newSearchEngine(geo), 49 | dc: newDC(geo, dcDs), 50 | spam: newSpam(spamDs), 51 | vpn: newVpn(vpnDs), 52 | } 53 | } 54 | 55 | // GetInfo returns computed information (Info struct) for given IP address. 56 | // Error is returned on critical condition, everything else is logged with debug level. 57 | func (i *IP) GetInfo(ip net.IP) (*Info, error) { 58 | country, err := i.geoip.getCountry(ip) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | var g errgroup.Group 64 | 65 | var company string 66 | g.Go(func() (err error) { 67 | company, err = i.geoip.getCompany(ip) 68 | return 69 | }) 70 | 71 | var isSearch bool 72 | g.Go(func() (err error) { 73 | isSearch, err = i.engine.isSearchEngine(ip) 74 | return 75 | }) 76 | 77 | var isTor bool 78 | g.Go(func() (err error) { 79 | isTor, err = i.tor.isTor(ip) 80 | return 81 | }) 82 | 83 | var isProxy bool 84 | g.Go(func() (err error) { 85 | isProxy, err = i.proxy.isProxy(ip) 86 | return 87 | }) 88 | 89 | var isDC bool 90 | g.Go(func() (err error) { 91 | isDC, err = i.dc.isDC(ip) 92 | return 93 | }) 94 | 95 | var isSpam bool 96 | g.Go(func() (err error) { 97 | isSpam, err = i.spam.isSpam(ip) 98 | return 99 | }) 100 | 101 | var isVpn bool 102 | g.Go(func() (err error) { 103 | isVpn, err = i.vpn.isVpn(ip) 104 | return 105 | }) 106 | 107 | var isPrivateAddr bool 108 | g.Go(func() (err error) { 109 | isPrivateAddr = isPrivateIP(ip) 110 | return 111 | }) 112 | 113 | if err := g.Wait(); err != nil { 114 | return nil, err 115 | } 116 | 117 | // error here can happen, and it's normal 118 | hostnames, _ := lookupAddrWithTimeout(ip.String(), 500*time.Millisecond) 119 | 120 | // Calculate scoring 0-100 (worst-best) 121 | var score uint8 = 86 122 | 123 | if isProxy { 124 | score -= 53 125 | } 126 | if !isProxy { 127 | score += 2 128 | } 129 | 130 | if isSearch { 131 | score++ 132 | } 133 | 134 | if isTor { 135 | score -= 59 136 | } 137 | 138 | if isDC { 139 | score -= 16 140 | } 141 | 142 | if isSpam { 143 | score -= 24 144 | } 145 | 146 | if isVpn { 147 | score -= 13 148 | } 149 | 150 | if len(hostnames) == 0 { 151 | score -= 3 152 | } 153 | 154 | if isPrivateAddr { 155 | score = 0 156 | } 157 | 158 | if score > 100 { 159 | score = 100 160 | } 161 | 162 | return &Info{ 163 | Company: company, 164 | Country: country, 165 | IsProxy: isProxy, 166 | IsSearchEngine: isSearch, 167 | IsTor: isTor, 168 | Hostnames: hostnames, 169 | IsPrivate: isPrivateAddr, 170 | IsDatacenter: isDC, 171 | IsSpam: isSpam, 172 | IsVpn: isVpn, 173 | IPScoring: score, 174 | }, nil 175 | } 176 | 177 | // RunUpdates schedules and runs updates. 178 | // Update interval is defined for each source individually. 179 | func (i *IP) RunUpdates() { 180 | ctx := context.Background() 181 | 182 | runAndSchedule(ctx, 15*time.Minute, func() { 183 | if err := i.tor.update(); err != nil { 184 | log.Error(err) 185 | } 186 | }) 187 | 188 | runAndSchedule(ctx, 24*time.Hour, func() { 189 | if err := i.geoip.update(); err != nil { 190 | log.Error(err) 191 | } 192 | }) 193 | 194 | runAndSchedule(ctx, 12*time.Hour, func() { 195 | if err := i.proxy.ipnet.Load(); err != nil { 196 | log.Error(err) 197 | } 198 | }) 199 | 200 | runAndSchedule(ctx, 12*time.Hour, func() { 201 | if err := i.dc.ipnet.Load(); err != nil { 202 | log.Error(err) 203 | } 204 | }) 205 | 206 | runAndSchedule(ctx, 12*time.Hour, func() { 207 | if err := i.spam.ipnet.Load(); err != nil { 208 | log.Error(err) 209 | } 210 | }) 211 | 212 | runAndSchedule(ctx, 12*time.Hour, func() { 213 | if err := i.vpn.ipnet.Load(); err != nil { 214 | log.Error(err) 215 | } 216 | }) 217 | } 218 | 219 | func runAndSchedule(ctx context.Context, interval time.Duration, f func()) { 220 | go func() { 221 | t := time.NewTimer(0) // first run - immediately 222 | for { 223 | select { 224 | case <-t.C: 225 | f() 226 | t = time.NewTimer(interval) // next runs according to the schedule 227 | case <-ctx.Done(): 228 | return 229 | } 230 | } 231 | }() 232 | } 233 | 234 | // isPrivateIP CHeck if IP belongs to private networks 235 | func isPrivateIP(ip net.IP) bool { 236 | // Eliminate by default multicast and loopback for IPv4 and IPv6 237 | global := ip.IsGlobalUnicast() 238 | if !global { 239 | return true 240 | } 241 | 242 | privateIPs := []string{ 243 | "10.0.0.0/8", // RFC1918 244 | "172.16.0.0/12", // RFC1918 245 | "192.168.0.0/16", // RFC1918 246 | "::1/128", // IPv6 loopback 247 | "fe80::/10", // IPv6 link-local 248 | "fc00::/7", // IPv6 unique local addr 249 | } 250 | 251 | for _, i := range privateIPs { 252 | _, network, _ := net.ParseCIDR(i) 253 | if network.Contains(ip) { 254 | return true 255 | } 256 | } 257 | 258 | return false 259 | } 260 | -------------------------------------------------------------------------------- /ip/ip_test.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func Test_checkPrivateIP(t *testing.T) { 11 | tests := map[string]bool{ 12 | "1.1.1.1": false, 13 | "255.255.255.255": true, 14 | "8.8.8.8": false, 15 | "127.0.0.1": true, 16 | "192.168.10.10": true, 17 | "10.0.0.1": true, 18 | "123.123.123.12": false, 19 | } 20 | 21 | for ip, v := range tests { 22 | assert.Equal(t, isPrivateIP(net.ParseIP(ip)), v, ip) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /ip/maxmind.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "archive/tar" 5 | "bytes" 6 | "compress/gzip" 7 | "crypto/md5" // #nosec 8 | "errors" 9 | "fmt" 10 | "io" 11 | "io/ioutil" 12 | "net" 13 | "os" 14 | "path/filepath" 15 | 16 | "github.com/labstack/gommon/log" 17 | "github.com/oschwald/geoip2-golang" 18 | ) 19 | 20 | const maxmindDir = "./resources/maxmind/" 21 | 22 | var maxmindFiles = []struct { 23 | url string 24 | md5 string 25 | file string 26 | t string 27 | }{ 28 | { 29 | url: "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-ASN&suffix=tar.gz", 30 | md5: "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-ASN&suffix=tar.gz.md5", 31 | file: "GeoLite2-ASN.mmdb", 32 | t: "asn", 33 | }, 34 | { 35 | url: "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&suffix=tar.gz", 36 | md5: "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&suffix=tar.gz.md5", 37 | file: "GeoLite2-Country.mmdb", 38 | t: "country", 39 | }, 40 | } 41 | 42 | type maxmind struct { 43 | license string 44 | country *geoip2.Reader 45 | asn *geoip2.Reader 46 | } 47 | 48 | func newMaxmind(license string) *maxmind { 49 | if license == "" { 50 | log.Infof("[geoip] MaxMind license is not present, reputation accuracy is degraded.") 51 | } 52 | 53 | return &maxmind{ 54 | license: license, 55 | } 56 | } 57 | 58 | func (g *maxmind) getCountry(ip net.IP) (string, error) { 59 | if g.country == nil { 60 | return "-", nil 61 | } 62 | 63 | country, err := g.country.Country(ip) 64 | if err != nil { 65 | return "-", fmt.Errorf("cannot get city for: %s , error: %w", ip, err) 66 | } 67 | 68 | log.Debugf("[geoip] IP: %s country: %s", ip, country.Country.IsoCode) 69 | return country.Country.IsoCode, nil 70 | } 71 | 72 | func (g *maxmind) getCompany(ip net.IP) (string, error) { 73 | if g.asn == nil { 74 | return "-", nil 75 | } 76 | 77 | asn, err := g.asn.ASN(ip) 78 | if err != nil { 79 | return "-", fmt.Errorf("cannot get ASN for: %s , error: %w", ip, err) 80 | } 81 | 82 | log.Debugf("[geoip] IP: %s ASN: %s", ip, asn.AutonomousSystemOrganization) 83 | return asn.AutonomousSystemOrganization, nil 84 | } 85 | 86 | func (g *maxmind) update() error { 87 | log.Debug("[geoip] update start") 88 | defer log.Debug("[geoip] update finished") 89 | 90 | if g.license == "" { 91 | log.Debug("[geoip] no license, skip update") 92 | return nil 93 | } 94 | 95 | if err := os.MkdirAll(maxmindDir, 0750); err != nil { 96 | return fmt.Errorf("cannot create directory %s , error: %w", maxmindDir, err) 97 | } 98 | 99 | for _, m := range maxmindFiles { 100 | license := "&license_key=" + g.license 101 | 102 | if err := g.download(m.url+license, m.md5+license); err != nil { 103 | return err 104 | } 105 | 106 | dbFile := filepath.Join(maxmindDir, m.file) 107 | db, err := geoip2.Open(dbFile) 108 | if err != nil { 109 | return fmt.Errorf("cannot open maxmind file %s, error: %w", dbFile, err) 110 | } 111 | 112 | if m.t == "country" { 113 | g.country = db 114 | } else if m.t == "asn" { 115 | g.asn = db 116 | } else { 117 | return errors.New("invalid type") 118 | } 119 | } 120 | return nil 121 | } 122 | 123 | func (g *maxmind) download(url string, md5Url string) error { 124 | response, err := defaultHTTPClient.Get(url) 125 | if err != nil { 126 | return fmt.Errorf("cannot download url: %s, error: %w", url, err) 127 | } 128 | defer response.Body.Close() 129 | 130 | var md5FileResponse bytes.Buffer 131 | body := io.TeeReader(response.Body, &md5FileResponse) 132 | 133 | gzr, err := gzip.NewReader(body) 134 | if err != nil { 135 | return fmt.Errorf("cannot open GZIP reader url: %s, error: %w", url, err) 136 | } 137 | defer gzr.Close() 138 | 139 | tr := tar.NewReader(gzr) 140 | for { 141 | header, err := tr.Next() 142 | 143 | if err != nil { 144 | if err == io.EOF { 145 | break 146 | } 147 | return fmt.Errorf("error while reading from TAR url: %s , error: %w", url, err) 148 | } 149 | 150 | if header.Typeflag == tar.TypeReg { 151 | target := filepath.Join(maxmindDir, header.FileInfo().Name()) 152 | 153 | f, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_RDWR, os.FileMode(header.Mode)) 154 | if err != nil { 155 | return fmt.Errorf("cannot create file %s, error: %w", target, err) 156 | } 157 | 158 | if _, err := io.CopyN(f, tr, int64(100_000_000)); err != nil { 159 | if err != io.EOF { 160 | return fmt.Errorf("cannot copy to file %s, error: %w", target, err) 161 | } 162 | } 163 | 164 | if err := f.Close(); err != nil { 165 | return fmt.Errorf("cannot copy close file %s, error: %w", target, err) 166 | } 167 | } 168 | } 169 | 170 | response, err = defaultHTTPClient.Get(md5Url) 171 | if err != nil { 172 | return fmt.Errorf("cannot download url: %s, error: %w", md5Url, err) 173 | } 174 | defer response.Body.Close() 175 | 176 | md5CheckResponse, err := ioutil.ReadAll(response.Body) 177 | if err != nil { 178 | return fmt.Errorf("can read md5 sum from url: %s, error: %w", md5Url, err) 179 | } 180 | 181 | md5File := fmt.Sprintf("%x", md5.Sum(md5FileResponse.Bytes())) // #nosec 182 | md5Checksum := string(md5CheckResponse) 183 | if md5Checksum != md5File { 184 | return fmt.Errorf("url: %s, md5(file): %s, md5(checksum): %s error: %w", 185 | url, md5File, md5Checksum, errors.New("invalid md5 checksum")) 186 | } 187 | 188 | return nil 189 | } 190 | -------------------------------------------------------------------------------- /ip/net.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "context" 5 | "net" 6 | "net/http" 7 | "time" 8 | ) 9 | 10 | var defaultHTTPClient = &http.Client{ 11 | Transport: &http.Transport{ 12 | DialContext: (&net.Dialer{ 13 | Timeout: 60 * time.Second, 14 | KeepAlive: 15 * time.Second, 15 | }).DialContext, 16 | TLSHandshakeTimeout: 60 * time.Second, 17 | ExpectContinueTimeout: 10 * time.Second, 18 | ResponseHeaderTimeout: 10 * time.Second, 19 | }, 20 | Timeout: 120 * time.Second, 21 | } 22 | 23 | func lookupAddrWithTimeout(addr string, timeout time.Duration) ([]string, error) { 24 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 25 | defer cancel() 26 | 27 | return (&net.Resolver{}).LookupAddr(ctx, addr) 28 | } 29 | 30 | func lookupIPWithTimeout(host string, timeout time.Duration) ([]net.IP, error) { 31 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 32 | defer cancel() 33 | 34 | addrs, err := (&net.Resolver{}).LookupIPAddr(ctx, host) 35 | if err != nil { 36 | return nil, err 37 | } 38 | ips := make([]net.IP, len(addrs)) 39 | for i, ia := range addrs { 40 | ips[i] = ia.IP 41 | } 42 | return ips, nil 43 | } 44 | -------------------------------------------------------------------------------- /ip/proxy.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "net" 5 | "regexp" 6 | "time" 7 | 8 | "github.com/labstack/gommon/log" 9 | "github.com/optimatiq/threatbite/ip/datasource" 10 | ) 11 | 12 | type proxy struct { 13 | ipnet *datasource.IPNet 14 | } 15 | 16 | func newProxy(source datasource.DataSource) *proxy { 17 | return &proxy{ipnet: datasource.NewIPNet(source, "proxy")} 18 | } 19 | 20 | var reIsProxy = regexp.MustCompile("proxy|sock|anon") 21 | 22 | // isProxy check if IP belongs to proxy list or have defined string in reverse name 23 | func (p *proxy) isProxy(ip net.IP) (bool, error) { 24 | isProxy, err := p.ipnet.Check(ip) 25 | if isProxy { 26 | log.Debugf("[isProxy] ip: %s tor: %t", ip, isProxy) 27 | return isProxy, err 28 | } 29 | 30 | reverse, err := lookupAddrWithTimeout(ip.String(), 500*time.Millisecond) 31 | if err != nil { 32 | // errors like "no such host" are normal, we don't need to pollute error logs 33 | log.Debugf("[isProxy] ip: %s error: %s", ip, err) 34 | return false, nil 35 | } 36 | 37 | if reIsProxy.MatchString(reverse[0]) { 38 | return true, nil 39 | } 40 | 41 | return false, nil 42 | } 43 | -------------------------------------------------------------------------------- /ip/search_engine.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "net" 5 | "regexp" 6 | "time" 7 | 8 | "github.com/labstack/gommon/log" 9 | ) 10 | 11 | type searchEngine struct { 12 | geoip *maxmind 13 | } 14 | 15 | func newSearchEngine(geoip *maxmind) *searchEngine { 16 | return &searchEngine{geoip: geoip} 17 | } 18 | 19 | var searchHosts = regexp.MustCompile("googlebot.com|google.com|yandex.com|search.msn.com|yahoo.net|yahoo.com|yahoo-net.jp|yahoo.co.jp|crawl.baidu.com|opera-mini.net|seznam.cz|mail.ru|pinterest.com|archive.org") 20 | var searchASNs = regexp.MustCompile("Google|Seznam.cz|Microsoft|Yahoo|Yandex|Opera Software|Facebook|Mail.Ru|Apple|LinkedIn|Twitter Inc.|Internet Archive") 21 | 22 | func (s *searchEngine) isSearchEngine(ip net.IP) (bool, error) { 23 | asn, err := s.geoip.getCompany(ip) 24 | if err != nil { 25 | return false, err 26 | } 27 | 28 | if searchASNs.MatchString(asn) { 29 | log.Debugf("[isEngine] ip: %s Company: %s %t", ip, asn, true) 30 | return true, nil 31 | } 32 | 33 | hostnames, err := lookupAddrWithTimeout(ip.String(), 500*time.Millisecond) 34 | if err != nil { 35 | // errors like "no such host" are normal, we don't need to pollute error logs 36 | log.Debugf("[isEngine] ip: %s error: %s", ip, err) 37 | return false, nil 38 | } 39 | ips, err := lookupIPWithTimeout(hostnames[0], 500*time.Millisecond) 40 | if err != nil { 41 | // errors like "cannot lookup" are normal, we don't need to pollute error logs 42 | log.Debugf("[isEngine] ip: %s error: %s", ip, err) 43 | return false, nil 44 | } 45 | 46 | matchedIP := false 47 | for _, i := range ips { 48 | if i.Equal(ip) { 49 | matchedIP = true 50 | break 51 | } 52 | } 53 | if !matchedIP { 54 | log.Debugf("[isEngine] ip: %s and hosts: %v don't match", ip, hostnames) 55 | return false, nil 56 | } 57 | 58 | for _, h := range hostnames { 59 | if searchHosts.MatchString(h) { 60 | log.Debugf("[isEngine] ip: %s Company: %s %t", ip, asn, true) 61 | return true, nil 62 | } 63 | } 64 | 65 | return false, nil 66 | } 67 | -------------------------------------------------------------------------------- /ip/spam.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "net" 5 | 6 | "github.com/labstack/gommon/log" 7 | "github.com/optimatiq/threatbite/ip/datasource" 8 | ) 9 | 10 | type spam struct { 11 | ipnet *datasource.IPNet 12 | } 13 | 14 | func newSpam(source datasource.DataSource) *spam { 15 | return &spam{ipnet: datasource.NewIPNet(source, "spam")} 16 | } 17 | 18 | func (s *spam) isSpam(ip net.IP) (bool, error) { 19 | isSpam, err := s.ipnet.Check(ip) 20 | log.Debugf("[isSpam] ip: %s tor: %t", ip, isSpam) 21 | return isSpam, err 22 | } 23 | -------------------------------------------------------------------------------- /ip/tor.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net" 7 | "regexp" 8 | 9 | "github.com/labstack/gommon/log" 10 | "github.com/optimatiq/threatbite/ip/datasource" 11 | ) 12 | 13 | // TODO(RW) we should use different list of exit nodes, official endpoint can contain outdated data. 14 | const torExitNodes = "https://check.torproject.org/exit-addresses" 15 | 16 | type tor struct { 17 | ipnet *datasource.IPNet 18 | } 19 | 20 | func newTor() *tor { 21 | return &tor{ipnet: datasource.NewIPNet(datasource.NewEmptyDataSource(), "tor")} 22 | } 23 | 24 | func (t *tor) update() error { 25 | log.Debug("[tor] update start") 26 | defer log.Debug("[tor] update finished") 27 | 28 | response, err := defaultHTTPClient.Get(torExitNodes) 29 | if err != nil { 30 | return fmt.Errorf("cannot download TOR exit nodes, error: %w", err) 31 | } 32 | 33 | content, err := ioutil.ReadAll(response.Body) 34 | if err != nil { 35 | return fmt.Errorf("cannot read body of TOR exit nodes, error: %w", err) 36 | } 37 | 38 | if err := response.Body.Close(); err != nil { 39 | return fmt.Errorf("cannot close response body, error: %w", err) 40 | } 41 | 42 | reExitNode := regexp.MustCompile(`ExitAddress (\d+\.\d+\.\d+\.\d+)`) 43 | var nodes []string 44 | for _, node := range reExitNode.FindAllStringSubmatch(string(content), -1) { 45 | nodes = append(nodes, node[1]) 46 | } 47 | 48 | ds, err := datasource.NewListDataSource(nodes) 49 | if err != nil { 50 | return fmt.Errorf("cannot create datasource for TOR, error: %w", err) 51 | } 52 | 53 | t.ipnet = datasource.NewIPNet(ds, "tor") 54 | return t.ipnet.Load() 55 | } 56 | 57 | func (t *tor) isTor(ip net.IP) (bool, error) { 58 | isTor, err := t.ipnet.Check(ip) 59 | log.Debugf("[checkTor] ip: %s tor: %t", ip, isTor) 60 | return isTor, err 61 | } 62 | -------------------------------------------------------------------------------- /ip/vpn.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "net" 5 | "regexp" 6 | "time" 7 | 8 | "github.com/labstack/gommon/log" 9 | "github.com/optimatiq/threatbite/ip/datasource" 10 | ) 11 | 12 | type vpn struct { 13 | ipnet *datasource.IPNet 14 | } 15 | 16 | func newVpn(source datasource.DataSource) *vpn { 17 | return &vpn{ipnet: datasource.NewIPNet(source, "vpn")} 18 | } 19 | 20 | var reIsVpn = regexp.MustCompile("vpn|ipsec|private|ovudp|l2tp|ovtcp|sstp|expressnetw|anony|hma.rocks|ipvanish|serverlocation.co|world4china|safersoftware.net|dns2use|ivacy|.cstorm.|cryptostorm|boxpnservers|airdns|hide.me|privateinternetaccess|windscribe|lazerpenguin|mullvad") 21 | 22 | // isVpn check if IP belongs to vpn list or have defined string in reverse name 23 | func (v *vpn) isVpn(ip net.IP) (bool, error) { 24 | isVpn, err := v.ipnet.Check(ip) 25 | if isVpn { 26 | log.Debugf("[isVpn] ip: %s tor: %t", ip, isVpn) 27 | return isVpn, err 28 | } 29 | 30 | reverse, err := lookupAddrWithTimeout(ip.String(), 500*time.Millisecond) 31 | if err != nil { 32 | // errors like "no such host" are normal, we don't need to pollute error logs 33 | log.Debugf("[isVpn] ip: %s error: %s", ip, err) 34 | return false, nil 35 | } 36 | 37 | if reIsVpn.MatchString(reverse[0]) { 38 | return true, nil 39 | } 40 | 41 | return false, nil 42 | } 43 | -------------------------------------------------------------------------------- /ip/vpn_test.go: -------------------------------------------------------------------------------- 1 | package ip 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | 9 | "github.com/optimatiq/threatbite/ip/datasource" 10 | ) 11 | 12 | func Test_vpn_isVpn(t *testing.T) { 13 | type args struct { 14 | ip net.IP 15 | } 16 | tests := []struct { 17 | name string 18 | list []string 19 | args args 20 | want bool 21 | wantErr bool 22 | }{ 23 | { 24 | name: "on list", 25 | list: []string{"192.168.0.1"}, 26 | args: args{ 27 | ip: net.ParseIP("192.168.0.1"), 28 | }, 29 | want: true, 30 | }, 31 | { 32 | name: "on list cidr", 33 | list: []string{"192.168.0.0/24"}, 34 | args: args{ 35 | ip: net.ParseIP("192.168.0.1"), 36 | }, 37 | want: true, 38 | }, 39 | { 40 | name: "not on list", 41 | list: []string{"192.168.0.0/24"}, 42 | args: args{ 43 | ip: net.ParseIP("191.168.0.1"), 44 | }, 45 | want: false, 46 | }, 47 | } 48 | for _, tt := range tests { 49 | t.Run(tt.name, func(t *testing.T) { 50 | ds, err := datasource.NewListDataSource(tt.list) 51 | assert.NoError(t, err) 52 | v := &vpn{ 53 | ipnet: datasource.NewIPNet(ds, "vpn"), 54 | } 55 | err = v.ipnet.Load() 56 | assert.NoError(t, err) 57 | 58 | got, err := v.isVpn(tt.args.ip) 59 | if (err != nil) != tt.wantErr { 60 | t.Errorf("isVpn() error = %v, wantErr %v", err, tt.wantErr) 61 | return 62 | } 63 | if got != tt.want { 64 | t.Errorf("isVpn() got = %v, want %v", got, tt.want) 65 | } 66 | }) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /resources/static/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Reputation 7 | 8 | 9 | 10 | 11 | 12 | 15 | 29 | 30 | 31 |
32 |
33 | 34 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /resources/static/swagger.yml: -------------------------------------------------------------------------------- 1 | openapi: 3.0.0 2 | info: 3 | description: Reputation API 4 | version: "1.0.3" 5 | title: Reputation API 6 | tags: 7 | - name: client 8 | description: Secured and authorized direct calls from client application 9 | - name: score 10 | description: Scoring call 11 | - name: info 12 | description: Information call 13 | paths: 14 | /v1/stats/: 15 | get: 16 | tags: 17 | - client 18 | - info 19 | summary: Information about account 20 | description: Show detail information about token/account, statistics, expiration date 21 | responses: 22 | '200': 23 | description: successful 24 | content: 25 | application/json: 26 | schema: 27 | type: array 28 | items: 29 | $ref: '#/components/schemas/Stats' 30 | '400': 31 | description: Invalid Input 32 | '401': 33 | description: Invalid ApiKey 34 | '402': 35 | description: Payment Required 36 | '429': 37 | description: Too Many Requests 38 | security: 39 | - headerKey: [] 40 | /v1/score/ip/{IP}: 41 | get: 42 | tags: 43 | - client 44 | - score 45 | summary: Get informations about IP 46 | description: Returns information about the scoring and suggested action based on the given IP address 47 | parameters: 48 | - in: path 49 | name: IP 50 | required: true 51 | schema: 52 | type: string 53 | format: ipv4 54 | description: IP address to test 55 | responses: 56 | '200': 57 | description: successful 58 | content: 59 | application/json: 60 | schema: 61 | $ref: '#/components/schemas/ScoreInfoIP' 62 | '400': 63 | description: Invalid Input 64 | '401': 65 | description: Invalid ApiKey 66 | '402': 67 | description: Payment Required 68 | '429': 69 | description: Too Many Requests 70 | security: 71 | - headerKey: [] 72 | /v1/score/request/: 73 | post: 74 | tags: 75 | - client 76 | - score 77 | summary: Get informations about request 78 | description: Returns information about the scoring and suggested action based on the given detail data 79 | requestBody: 80 | content: 81 | application/json: 82 | schema: 83 | $ref: '#/components/schemas/GetScoreRequest' 84 | responses: 85 | '200': 86 | description: successful 87 | content: 88 | application/json: 89 | schema: 90 | $ref: '#/components/schemas/ScoreInfoRequest' 91 | '400': 92 | description: Invalid Input 93 | '401': 94 | description: Invalid ApiKey 95 | '402': 96 | description: Payment Required 97 | '429': 98 | description: Too Many Requests 99 | security: 100 | - headerKey: [] 101 | /v1/score/email/{EMAIL}: 102 | get: 103 | tags: 104 | - client 105 | - score 106 | summary: Get informations about e-mail address 107 | description: Returns information about e-mail address 108 | parameters: 109 | - in: path 110 | name: EMAIL 111 | required: true 112 | schema: 113 | type: string 114 | format: email 115 | description: Email address to test 116 | responses: 117 | '200': 118 | description: successful 119 | content: 120 | application/json: 121 | schema: 122 | $ref: '#/components/schemas/ScoreInfoEmail' 123 | '400': 124 | description: Invalid Input 125 | '401': 126 | description: Invalid ApiKey 127 | '402': 128 | description: Payment Required 129 | '429': 130 | description: Too Many Requests 131 | security: 132 | - headerKey: [] 133 | components: 134 | schemas: 135 | Stats: 136 | type: object 137 | required: 138 | - type 139 | properties: 140 | type: 141 | type: string 142 | example: demo 143 | enum: [demo, payed, free] 144 | description: Type of account 145 | req1h: 146 | type: number 147 | example: 10 148 | description: Number of API requests in last 1 hour 149 | req12h: 150 | type: number 151 | example: 1200 152 | description: Number of API requests in last 12 hours 153 | req1d: 154 | type: number 155 | example: 2400 156 | description: Number of API requests in last 1 day (24h) 157 | req7d: 158 | type: number 159 | example: 16800 160 | description: Number of API requests in last 7 days (168h) 161 | req30d: 162 | type: number 163 | example: 72000 164 | description: Number of API requests in last 30 days (720h) 165 | ScoreInfoIP: 166 | type: object 167 | required: 168 | - requestid 169 | - scoring 170 | - action 171 | properties: 172 | requestid: 173 | type: string 174 | example: 4c4712a4141d261ec0ca8f9037950685 175 | description: Unique query ID. 176 | scoring: 177 | type: number 178 | example: 51 179 | minimum: 0 180 | maximum: 100 181 | description: Scoring information. Ihe higher the number, the greater the potential threat. 182 | action: 183 | type: string 184 | example: captcha 185 | enum: [accept, verify, block] 186 | description: Suggested action that can be performed in relation to the verified source. 187 | tor: 188 | type: boolean 189 | example: true 190 | description: Source IP belongs to Tor network. 191 | proxy: 192 | type: boolean 193 | example: true 194 | description: Source IP is listed as proxy. 195 | dc: 196 | type: boolean 197 | example: false 198 | description: Source IP belongs datacenter. 199 | vpn: 200 | type: boolean 201 | example: false 202 | description: Source IP is listed as VPN. 203 | spam: 204 | type: boolean 205 | example: false 206 | description: Source IP is listed as spam source. 207 | private: 208 | type: boolean 209 | example: false 210 | description: Source IP belongs to private network. 211 | bad: 212 | type: boolean 213 | example: false 214 | description: Source IP has bad reputation. 215 | se: 216 | type: boolean 217 | example: false 218 | description: Source IP belongs to popular search engine (Google, Bing etc.). 219 | bot: 220 | type: boolean 221 | example: false 222 | description: Source IP is used by bot. 223 | country: 224 | type: string 225 | example: US 226 | description: Source IP country code. 227 | company: 228 | type: string 229 | example: Optimatiq Sp. z o.o. 230 | description: Name of network owner. 231 | ScoreInfoRequest: 232 | type: object 233 | required: 234 | - requestid 235 | - scoring 236 | - action 237 | properties: 238 | requestid: 239 | type: string 240 | example: 4c4712a4141d261ec0ca8f9037950685 241 | description: Unique query ID. 242 | scoring: 243 | type: number 244 | example: 51 245 | minimum: 0 246 | maximum: 100 247 | description: Scoring information. Ihe higher the number, the greater the potential threat. 248 | action: 249 | type: string 250 | example: captcha 251 | enum: [accept, verify, block] 252 | description: Suggested action that can be performed in relation to the verified source. 253 | isTor: 254 | type: boolean 255 | example: true 256 | description: Request source belongs to Tor network. 257 | isProxy: 258 | type: boolean 259 | example: true 260 | description: Request source is listed as proxy. 261 | isDC: 262 | type: boolean 263 | example: false 264 | description: Request source belongs datacenter. 265 | isBadReputation: 266 | type: boolean 267 | example: false 268 | description: Request source has bad reputation. 269 | isSearchEngine: 270 | type: boolean 271 | example: false 272 | description: Request source belongs to popular search engine (Google, Bing etc.). 273 | isBot: 274 | type: boolean 275 | example: false 276 | description: Request source is used by bot. 277 | isMobile: 278 | type: boolean 279 | example: false 280 | description: Request source is Mobile device. 281 | Country: 282 | type: string 283 | example: US 284 | description: Source IP country code. 285 | ScoreInfoEmail: 286 | type: object 287 | required: 288 | - requestid 289 | - scoring 290 | - isValid 291 | - isPresent 292 | - isFree 293 | - isTemporary 294 | - isCatchAll 295 | properties: 296 | requestid: 297 | type: string 298 | example: 4c4712a4141d261ec0ca8f9037950685 299 | description: Unique query ID. 300 | scoring: 301 | type: number 302 | example: 51 303 | minimum: 0 304 | maximum: 100 305 | description: Scoring information. Ihe higher the number, the greater the potential threat. 306 | valid: 307 | type: boolean 308 | example: true 309 | description: E-mail string is valid 310 | exists: 311 | type: boolean 312 | example: true 313 | description: E-mail account exists on remote server. 314 | free: 315 | type: boolean 316 | example: false 317 | description: E-mail address belongs to free e-mail account provider. 318 | disposal: 319 | type: boolean 320 | example: false 321 | description: E-mail address belongs to disposable email provider. 322 | catchall: 323 | type: boolean 324 | example: false 325 | description: E-mail account is a part of CatchAll. 326 | leaked: 327 | type: boolean 328 | example: false 329 | description: E-mail account has been compromised in a data breach. 330 | default: 331 | type: boolean 332 | example: false 333 | description: E-mail account belongs to group of administrative accounts. 334 | GetScoreIp: 335 | type: object 336 | required: 337 | - ip 338 | properties: 339 | ip: 340 | type: string 341 | format: ipv4 342 | example: 8.8.8.8 343 | description: IPv4 addres of the verified source. 344 | GetScoreEmail: 345 | type: object 346 | required: 347 | - email 348 | properties: 349 | email: 350 | type: string 351 | format: email 352 | example: mail@example.com 353 | description: E-mail address to test 354 | GetScoreRequest: 355 | description: Informations about request that should be scored. Remember that any change in the order of headings or the size of characters may affect the result. 356 | type: object 357 | required: 358 | - remote_ip 359 | - request_domain 360 | - request_uri 361 | - request_method 362 | - user_agent 363 | properties: 364 | remote_ip: 365 | type: string 366 | format: ipv4 367 | example: 8.8.8.8 368 | description: IPv4 addres of the verified source (required). 369 | request_domain: 370 | type: string 371 | format: hostname 372 | example: example.com 373 | description: The domain that recieved HTTP query (required). 374 | request_uri: 375 | type: string 376 | format: uri 377 | example: /data?foo=bar 378 | description: URI request with all arguments (required). 379 | request_method: 380 | type: string 381 | example: GET 382 | description: HTTP method 383 | server_protocol: 384 | type: string 385 | example: HTTP/1.1 386 | description: Full protocol name with version 387 | request_scheme: 388 | type: string 389 | enum: [http, https] 390 | example: https 391 | description: Scheme of HTTP request. 392 | remote_port: 393 | type: integer 394 | minimum: 0 395 | maximum: 65535 396 | example: 1357 397 | description: Remote TCP port of HTTP request. 398 | user_agent: 399 | type: string 400 | example: 'Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405' 401 | description: Value of User-Agent header. 402 | all_headers: 403 | type: object 404 | additionalProperties: 405 | type: string 406 | example: { 407 | "X-Forwarded-For": "1.1.1.1", 408 | "Host": "example.com", 409 | "Content-Type": "text/html; charset=utf-8", 410 | } 411 | description: All HTTP headers send in request. 412 | securitySchemes: 413 | headerKey: 414 | type: apiKey 415 | in: header 416 | name: X-Score-API-KEY -------------------------------------------------------------------------------- /resources/static/threatbite_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/optimatiq/threatbite/6718d6174a312c600c2ab10a67e78bebd4656d43/resources/static/threatbite_diagram.png --------------------------------------------------------------------------------