├── .github ├── ISSUE_TEMPLATE │ ├── BUG_REPORT.md │ ├── FEATURE_REQUEST.md │ └── SUPPORT_QUESTION.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml └── workflows │ ├── docker-build.yml │ ├── fetch.yml │ ├── golangci.yml │ ├── goreleaser.yml │ └── test.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── .revive.toml ├── Dockerfile ├── GNUmakefile ├── LICENSE ├── README.md ├── commands ├── fetch-cti.go ├── fetch.go ├── root.go ├── search.go ├── server.go └── version.go ├── config └── config.go ├── db ├── db.go ├── rdb.go └── redis.go ├── fetcher ├── attack │ ├── attack.go │ ├── attack_test.go │ ├── testdata │ │ └── enterprise-attack.json │ └── types.go ├── capec │ ├── capec.go │ ├── capec_test.go │ ├── testdata │ │ └── stix-capec.json │ └── types.go ├── cti.go ├── cti_test.go ├── cwe │ ├── cwe.go │ ├── cwe_test.go │ ├── testdata │ │ ├── v4.14 │ │ │ └── cwec_latest.xml.zip │ │ ├── v4.6 │ │ │ └── cwec.xml.zip │ │ └── v5.0 │ │ │ └── cwec_latest.xml.zip │ └── types.go └── nvd │ ├── nvd.go │ ├── nvd_test.go │ ├── testdata │ └── go-cti00001 │ │ └── vuls-data-raw-nvd-api-cve │ │ └── 2020 │ │ └── CVE-2020-0002.json │ └── types.go ├── go.mod ├── go.sum ├── integration ├── .gitignore ├── README.md ├── cveid.txt └── diff_server_mode.py ├── main.go ├── models ├── models.go └── models_test.go ├── server └── server.go └── utils └── utils.go /.github/ISSUE_TEMPLATE/BUG_REPORT.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | labels: bug 4 | about: If something isn't working as expected. 5 | --- 6 | 7 | # What did you do? (required. The issue will be **closed** when not provided.) 8 | 9 | 10 | # What did you expect to happen? 11 | 12 | 13 | # What happened instead? 14 | 15 | * Current Output 16 | 17 | Please re-run the command using ```-debug``` and provide the output below. 18 | 19 | # Steps to reproduce the behaviour 20 | 21 | 22 | # Configuration (**MUST** fill this out): 23 | 24 | * Go version (`go version`): 25 | 26 | * Go environment (`go env`): 27 | 28 | * go-cti environment: 29 | 30 | Hash : ____ 31 | 32 | To check the commit hash of HEAD 33 | $ go-cti version 34 | 35 | or 36 | 37 | $ cd $GOPATH/src/github.com/vulsio/go-cti 38 | $ git rev-parse --short HEAD 39 | 40 | * command: 41 | 42 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | labels: enhancement 4 | about: I have a suggestion (and might want to implement myself)! 5 | --- 6 | 7 | 10 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/SUPPORT_QUESTION.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Support Question 3 | labels: question 4 | about: If you have a question about Vuls. 5 | --- 6 | 7 | 11 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | If this Pull Request is work in progress, Add a prefix of “[WIP]” in the title. 3 | 4 | # What did you implement: 5 | 6 | Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. 7 | 8 | Fixes # (issue) 9 | 10 | ## Type of change 11 | 12 | Please delete options that are not relevant. 13 | 14 | - [ ] Bug fix (non-breaking change which fixes an issue) 15 | - [ ] New feature (non-breaking change which adds functionality) 16 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 17 | - [ ] This change requires a documentation update 18 | 19 | # How Has This Been Tested? 20 | 21 | Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. 22 | 23 | # Checklist: 24 | You don't have to satisfy all of the following. 25 | 26 | - [ ] Write tests 27 | - [ ] Write documentation 28 | - [ ] Check that there aren't other open pull requests for the same issue/feature 29 | - [ ] Format your source code by `make fmt` 30 | - [ ] Pass the test by `make test` 31 | - [ ] Provide verification config / commands 32 | - [ ] Enable "Allow edits from maintainers" for this PR 33 | - [ ] Update the messages below 34 | 35 | ***Is this ready for review?:*** NO 36 | 37 | # Reference 38 | 39 | * https://blog.github.com/2015-01-21-how-to-write-the-perfect-pull-request/ 40 | 41 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | target-branch: "master" 13 | - package-ecosystem: "github-actions" # See documentation for possible values 14 | directory: "/" # Location of package manifests 15 | schedule: 16 | interval: "weekly" 17 | target-branch: "master" 18 | -------------------------------------------------------------------------------- /.github/workflows/docker-build.yml: -------------------------------------------------------------------------------- 1 | name: Publish Docker image 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'master' 7 | tags: 8 | - '*' 9 | 10 | jobs: 11 | docker: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v4 16 | 17 | - name: Set up QEMU 18 | uses: docker/setup-qemu-action@v3 19 | 20 | - name: Set up Docker Buildx 21 | uses: docker/setup-buildx-action@v3 22 | 23 | - name: Login to DockerHub 24 | uses: docker/login-action@v3 25 | with: 26 | username: ${{ secrets.DOCKERHUB_USERNAME }} 27 | password: ${{ secrets.DOCKERHUB_TOKEN }} 28 | 29 | - 30 | name: Docker meta 31 | id: meta 32 | uses: docker/metadata-action@v5 33 | with: 34 | images: vuls/go-cti 35 | tags: | 36 | type=ref,event=tag 37 | 38 | - name: Build and push 39 | uses: docker/build-push-action@v6 40 | with: 41 | push: true 42 | context: . 43 | tags: | 44 | vuls/go-cti:latest 45 | ${{ steps.meta.outputs.tags }} 46 | secrets: | 47 | "github_token=${{ secrets.GITHUB_TOKEN }}" 48 | platforms: linux/amd64,linux/arm64 49 | -------------------------------------------------------------------------------- /.github/workflows/fetch.yml: -------------------------------------------------------------------------------- 1 | name: Fetch Test 2 | 3 | on: 4 | pull_request: 5 | schedule: 6 | - cron: '0 0 * * *' 7 | 8 | jobs: 9 | fetch-threat: 10 | name: fetch-threat 11 | runs-on: ubuntu-latest 12 | services: 13 | mysql: 14 | image: mysql 15 | ports: 16 | - 3306:3306 17 | env: 18 | MYSQL_ROOT_PASSWORD: password 19 | MYSQL_DATABASE: test 20 | options: >- 21 | --health-cmd "mysqladmin ping" 22 | --health-interval 10s 23 | --health-timeout 5s 24 | --health-retries 5 25 | postgres: 26 | image: postgres 27 | ports: 28 | - 5432:5432 29 | env: 30 | POSTGRES_PASSWORD: password 31 | POSTGRES_DB: test 32 | options: >- 33 | --health-cmd pg_isready 34 | --health-interval 10s 35 | --health-timeout 5s 36 | --health-retries 5 37 | redis: 38 | image: redis 39 | ports: 40 | - 6379:6379 41 | options: >- 42 | --health-cmd "redis-cli ping" 43 | --health-interval 10s 44 | --health-timeout 5s 45 | --health-retries 5 46 | steps: 47 | - name: Check out code into the Go module directory 48 | uses: actions/checkout@v4 49 | - name: Set up Go 50 | uses: actions/setup-go@v5 51 | with: 52 | go-version-file: go.mod 53 | - name: build 54 | id: build 55 | run: make build 56 | - name: fetch sqlite3 57 | if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} 58 | run: ./go-cti fetch --dbtype sqlite3 threat 59 | - name: fetch mysql 60 | if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} 61 | run: ./go-cti fetch --dbtype mysql --dbpath "root:password@tcp(127.0.0.1:3306)/test?parseTime=true" threat 62 | - name: fetch postgres 63 | if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} 64 | run: ./go-cti fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" threat 65 | - name: fetch redis 66 | if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} 67 | run: ./go-cti fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" threat 68 | -------------------------------------------------------------------------------- /.github/workflows/golangci.yml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | on: 3 | push: 4 | tags: 5 | - v* 6 | branches: 7 | - master 8 | pull_request: 9 | jobs: 10 | golangci: 11 | name: lint 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Check out code into the Go module directory 15 | uses: actions/checkout@v4 16 | - name: Set up Go 1.x 17 | uses: actions/setup-go@v5 18 | with: 19 | go-version-file: go.mod 20 | - name: golangci-lint 21 | uses: golangci/golangci-lint-action@v7 22 | with: 23 | # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version 24 | version: v2.0.2 25 | 26 | # Optional: working directory, useful for monorepos 27 | # working-directory: somedir 28 | 29 | # Optional: golangci-lint command line arguments. 30 | # args: --issues-exit-code=0 31 | 32 | # Optional: show only new issues if it's a pull request. The default value is `false`. 33 | # only-new-issues: true 34 | -------------------------------------------------------------------------------- /.github/workflows/goreleaser.yml: -------------------------------------------------------------------------------- 1 | name: goreleaser 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | goreleaser: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - 13 | name: Checkout 14 | uses: actions/checkout@v4 15 | - 16 | name: Unshallow 17 | run: git fetch --prune --unshallow 18 | - 19 | name: Set up Go 20 | uses: actions/setup-go@v5 21 | with: 22 | go-version-file: go.mod 23 | - 24 | name: Run GoReleaser 25 | uses: goreleaser/goreleaser-action@v6 26 | with: 27 | distribution: goreleaser 28 | version: latest 29 | args: release --clean 30 | env: 31 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 32 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: [pull_request] 4 | 5 | jobs: 6 | build: 7 | name: Build 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Check out code into the Go module directory 11 | uses: actions/checkout@v4 12 | - name: Set up Go 1.x 13 | uses: actions/setup-go@v5 14 | with: 15 | go-version-file: go.mod 16 | - name: Test 17 | run: make test 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | 17 | # Priject's ignore 18 | go-cti 19 | *.sqlite3 20 | *.sqlite3-shm 21 | *.sqlite3-wal 22 | data 23 | cache 24 | .DS_Store 25 | .vscode 26 | 27 | # Loging files 28 | *.log 29 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | 3 | linters: 4 | default: none 5 | enable: 6 | - errcheck 7 | - govet 8 | - ineffassign 9 | - misspell 10 | - prealloc 11 | - revive 12 | - staticcheck 13 | settings: 14 | revive: # https://golangci-lint.run/usage/linters/#revive 15 | rules: 16 | - name: blank-imports 17 | - name: context-as-argument 18 | - name: context-keys-type 19 | - name: dot-imports 20 | - name: empty-block 21 | - name: error-naming 22 | - name: error-return 23 | - name: error-strings 24 | - name: errorf 25 | - name: exported 26 | - name: if-return 27 | - name: increment-decrement 28 | - name: indent-error-flow 29 | - name: package-comments 30 | disabled: true 31 | - name: range 32 | - name: receiver-naming 33 | - name: redefines-builtin-id 34 | - name: superfluous-else 35 | - name: time-naming 36 | - name: unexported-return 37 | - name: unreachable-code 38 | - name: unused-parameter 39 | - name: var-declaration 40 | - name: var-naming 41 | staticcheck: # https://golangci-lint.run/usage/linters/#staticcheck 42 | checks: 43 | - all 44 | - -ST1000 # at least one file in a package should have a package comment 45 | - -ST1005 # error strings should not be capitalized 46 | exclusions: 47 | rules: 48 | - source: "defer .+\\.Close\\(\\)" 49 | linters: 50 | - errcheck 51 | - source: "defer os.RemoveAll\\(.+\\)" 52 | linters: 53 | - errcheck 54 | 55 | formatters: 56 | enable: 57 | - goimports 58 | 59 | run: 60 | timeout: 10m 61 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: go-cti 2 | release: 3 | github: 4 | owner: vulsio 5 | name: go-cti 6 | env: 7 | - CGO_ENABLED=0 8 | builds: 9 | - id: go-cti 10 | goos: 11 | - linux 12 | - windows 13 | - darwin 14 | goarch: 15 | - amd64 16 | - arm64 17 | main: . 18 | ldflags: -s -w -X github.com/vulsio/go-cti/config.Version={{.Version}} -X github.com/vulsio/go-cti/config.Revision={{.Commit}} 19 | binary: go-cti 20 | archives: 21 | - name_template: '{{ .Binary }}_{{.Version}}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' 22 | format: tar.gz 23 | files: 24 | - LICENSE 25 | - README* 26 | snapshot: 27 | name_template: SNAPSHOT-{{ .Commit }} 28 | -------------------------------------------------------------------------------- /.revive.toml: -------------------------------------------------------------------------------- 1 | ignoreGeneratedHeader = false 2 | severity = "warning" 3 | confidence = 0.8 4 | errorCode = 0 5 | warningCode = 0 6 | 7 | [rule.blank-imports] 8 | [rule.context-as-argument] 9 | [rule.context-keys-type] 10 | [rule.dot-imports] 11 | [rule.error-return] 12 | [rule.error-strings] 13 | [rule.error-naming] 14 | [rule.exported] 15 | [rule.if-return] 16 | [rule.increment-decrement] 17 | [rule.var-naming] 18 | [rule.var-declaration] 19 | [rule.package-comments] 20 | [rule.range] 21 | [rule.receiver-naming] 22 | [rule.time-naming] 23 | [rule.unexported-return] 24 | [rule.indent-error-flow] 25 | [rule.errorf] 26 | [rule.empty-block] 27 | [rule.superfluous-else] 28 | [rule.unused-parameter] 29 | [rule.unreachable-code] 30 | [rule.redefines-builtin-id] -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine as builder 2 | 3 | RUN apk add --no-cache \ 4 | git \ 5 | make \ 6 | gcc \ 7 | musl-dev 8 | 9 | ENV REPOSITORY github.com/vulsio/go-cti 10 | COPY . $GOPATH/src/$REPOSITORY 11 | RUN cd $GOPATH/src/$REPOSITORY && make install 12 | 13 | 14 | FROM alpine:3.15 15 | 16 | ENV LOGDIR /var/log/go-cti 17 | ENV WORKDIR /go-cti 18 | 19 | RUN apk add --no-cache ca-certificates git \ 20 | && mkdir -p $WORKDIR $LOGDIR 21 | 22 | COPY --from=builder /go/bin/go-cti /usr/local/bin/ 23 | 24 | VOLUME ["$WORKDIR", "$LOGDIR"] 25 | WORKDIR $WORKDIR 26 | ENV PWD $WORKDIR 27 | 28 | ENTRYPOINT ["go-cti"] 29 | CMD ["help"] -------------------------------------------------------------------------------- /GNUmakefile: -------------------------------------------------------------------------------- 1 | .PHONY: \ 2 | build \ 3 | install \ 4 | all \ 5 | vendor \ 6 | lint \ 7 | golangci \ 8 | vet \ 9 | fmt \ 10 | fmtcheck \ 11 | pretest \ 12 | test \ 13 | cov \ 14 | clean \ 15 | build-integration \ 16 | clean-integration \ 17 | fetch-rdb \ 18 | fetch-redis \ 19 | diff-cveid \ 20 | diff-package \ 21 | diff-server-rdb \ 22 | diff-server-redis \ 23 | diff-server-rdb-redis 24 | 25 | SRCS = $(shell git ls-files '*.go') 26 | PKGS = $(shell go list ./...) 27 | VERSION := $(shell git describe --tags --abbrev=0) 28 | REVISION := $(shell git rev-parse --short HEAD) 29 | BUILDTIME := $(shell date "+%Y%m%d_%H%M%S") 30 | LDFLAGS := -X 'github.com/vulsio/go-cti/config.Version=$(VERSION)' \ 31 | -X 'github.com/vulsio/go-cti/config.Revision=$(REVISION)' 32 | GO := CGO_ENABLED=0 go 33 | 34 | all: build test 35 | 36 | build: main.go 37 | $(GO) build -a -ldflags "$(LDFLAGS)" -o go-cti $< 38 | 39 | install: main.go 40 | $(GO) install -ldflags "$(LDFLAGS)" 41 | 42 | lint: 43 | go install github.com/mgechev/revive@latest 44 | revive -config ./.revive.toml -formatter plain $(PKGS) 45 | 46 | golangci: 47 | go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest 48 | golangci-lint run 49 | 50 | vet: 51 | echo $(PKGS) | xargs env $(GO) vet || exit; 52 | 53 | fmt: 54 | gofmt -w $(SRCS) 55 | 56 | fmtcheck: 57 | $(foreach file,$(SRCS),gofmt -d $(file);) 58 | 59 | pretest: lint vet fmtcheck 60 | 61 | test: pretest 62 | $(GO) test -cover -v ./... || exit; 63 | 64 | cov: 65 | @ go get -v github.com/axw/gocov/gocov 66 | @ go get golang.org/x/tools/cmd/cover 67 | gocov test | gocov report 68 | 69 | clean: 70 | $(foreach pkg,$(PKGS),go clean $(pkg) || exit;) 71 | 72 | BRANCH := $(shell git symbolic-ref --short HEAD) 73 | build-integration: 74 | @ git stash save 75 | $(GO) build -ldflags "$(LDFLAGS)" -o integration/go-cti.new 76 | git checkout $(shell git describe --tags --abbrev=0) 77 | @git reset --hard 78 | $(GO) build -ldflags "$(LDFLAGS)" -o integration/go-cti.old 79 | git checkout $(BRANCH) 80 | -@ git stash apply stash@{0} && git stash drop stash@{0} 81 | 82 | clean-integration: 83 | -pkill go-cti.old 84 | -pkill go-cti.new 85 | -rm integration/go-cti.old integration/go-cti.new integration/go-cti.old.sqlite3 integration/go-cti.new.sqlite3 86 | -rm -rf integration/diff 87 | -docker kill redis-old redis-new 88 | -docker rm redis-old redis-new 89 | 90 | fetch-rdb: 91 | integration/go-cti.old fetch threat --dbpath=integration/go-cti.old.sqlite3 92 | integration/go-cti.new fetch threat --dbpath=integration/go-cti.new.sqlite3 93 | 94 | fetch-redis: 95 | docker run --name redis-old -d -p 127.0.0.1:6379:6379 redis 96 | docker run --name redis-new -d -p 127.0.0.1:6380:6379 redis 97 | 98 | integration/go-cti.old fetch threat --dbtype redis --dbpath "redis://127.0.0.1:6379/0" 99 | integration/go-cti.new fetch threat --dbtype redis --dbpath "redis://127.0.0.1:6380/0" 100 | 101 | diff-cves: 102 | @ python integration/diff_server_mode.py cves --sample_rate 0.01 103 | @ python integration/diff_server_mode.py multi-cves --sample_rate 0.01 104 | 105 | diff-server-rdb: 106 | integration/go-cti.old server --dbpath=integration/go-cti.old.sqlite3 --port 1325 > /dev/null 2>&1 & 107 | integration/go-cti.new server --dbpath=integration/go-cti.new.sqlite3 --port 1326 > /dev/null 2>&1 & 108 | make diff-cves 109 | pkill go-cti.old 110 | pkill go-cti.new 111 | 112 | diff-server-redis: 113 | integration/go-cti.old server --dbtype redis --dbpath "redis://127.0.0.1:6379/0" --port 1325 > /dev/null 2>&1 & 114 | integration/go-cti.new server --dbtype redis --dbpath "redis://127.0.0.1:6380/0" --port 1326 > /dev/null 2>&1 & 115 | make diff-cves 116 | pkill go-cti.old 117 | pkill go-cti.new 118 | 119 | diff-server-rdb-redis: 120 | integration/go-cti.new server --dbpath=integration/go-cti.new.sqlite3 --port 1325 > /dev/null 2>&1 & 121 | integration/go-cti.new server --dbtype redis --dbpath "redis://127.0.0.1:6380/0" --port 1326 > /dev/null 2>&1 & 122 | make diff-cves 123 | pkill go-cti.new -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright © 2020 Takuya Sawada 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-cti 2 | 3 | `go-cti` build a local copy of MITRE ATT&CK and CAPEC. 4 | 5 | ## Usage 6 | ```console 7 | $ go-cti help 8 | Go collect Cyber Threat Intelligence 9 | 10 | Usage: 11 | go-cti [command] 12 | 13 | Available Commands: 14 | completion Generate the autocompletion script for the specified shell 15 | fetch Fetch the data of mitre/cti 16 | help Help about any command 17 | search Search the data of mitre/cti form DB 18 | server Start go-cti HTTP server 19 | version Show version 20 | 21 | Flags: 22 | --config string config file (default is $HOME/.go-cti.yaml) 23 | --dbpath string /path/to/sqlite3 or SQL connection string (default "$PWD/go-cti.sqlite3") 24 | --dbtype string Database type to store data in (sqlite3, mysql, postgres or redis supported) (default "sqlite3") 25 | --debug debug mode (default: false) 26 | --debug-sql SQL debug mode 27 | -h, --help help for go-cti 28 | --http-proxy string http://proxy-url:port (default: empty) 29 | --log-dir string /path/to/log (default "/var/log/go-cti") 30 | --log-json output log as JSON 31 | --log-to-file output log to file 32 | 33 | Use "go-cti [command] --help" for more information about a command. 34 | ``` 35 | 36 | ## Fetch MITRE ATT&CK and CAPEC 37 | ```console 38 | $ go-cti fetch threat 39 | INFO[04-20|11:39:27] Fetching Cyber Threat Intelligence and CVE-ID to CTI-ID Mappings 40 | INFO[04-20|11:39:27] Fetching MITRE ATT&CK... 41 | INFO[04-20|11:39:30] Fetching CAPEC... 42 | INFO[04-20|11:39:31] Fetching CWE... 43 | INFO[04-20|11:39:34] Fetching NVD CVE... year=recent 44 | INFO[04-20|11:39:35] Fetching NVD CVE... year=modified 45 | INFO[04-20|11:39:37] Fetching NVD CVE... year=2002 46 | INFO[04-20|11:39:39] Fetching NVD CVE... year=2003 47 | INFO[04-20|11:39:40] Fetching NVD CVE... year=2004 48 | INFO[04-20|11:39:42] Fetching NVD CVE... year=2005 49 | INFO[04-20|11:39:43] Fetching NVD CVE... year=2006 50 | INFO[04-20|11:39:46] Fetching NVD CVE... year=2007 51 | INFO[04-20|11:39:48] Fetching NVD CVE... year=2008 52 | INFO[04-20|11:39:51] Fetching NVD CVE... year=2009 53 | INFO[04-20|11:39:53] Fetching NVD CVE... year=2010 54 | INFO[04-20|11:39:55] Fetching NVD CVE... year=2011 55 | INFO[04-20|11:39:58] Fetching NVD CVE... year=2012 56 | INFO[04-20|11:40:00] Fetching NVD CVE... year=2013 57 | INFO[04-20|11:40:03] Fetching NVD CVE... year=2014 58 | INFO[04-20|11:40:05] Fetching NVD CVE... year=2015 59 | INFO[04-20|11:40:08] Fetching NVD CVE... year=2016 60 | INFO[04-20|11:40:11] Fetching NVD CVE... year=2017 61 | INFO[04-20|11:40:15] Fetching NVD CVE... year=2018 62 | INFO[04-20|11:40:19] Fetching NVD CVE... year=2019 63 | INFO[04-20|11:40:23] Fetching NVD CVE... year=2020 64 | INFO[04-20|11:40:28] Fetching NVD CVE... year=2021 65 | INFO[04-20|11:40:33] Fetching NVD CVE... year=2022 66 | INFO[04-20|11:40:35] Fetched Cyber Threat Intelligence and CVE-ID to CTI-ID Mappings techniques=1112 mappings=98011 attackers=672 67 | INFO[04-20|11:40:35] Insert Cyber Threat Intelligences and CVE-ID to CTI-ID Mappings into go-cti. db=redis 68 | INFO[04-20|11:40:35] Inserting Cyber Threat Intelligences... 69 | INFO[04-20|11:43:29] Inserting Techniques... 70 | 1112 / 1112 [------------------------------------------------] 100.00% 3530 p/s 71 | INFO[04-20|11:43:30] Inserting CVE-ID to CTI-ID CveToTechniques... 72 | 98011 / 98011 [----------------------------------------------] 100.00% 9900 p/s 73 | INFO[04-20|11:43:40] Inserting Attackers... 74 | 672 / 672 [-----------------------------------------------------] 100.00% ? p/s 75 | ``` 76 | 77 | ## Search by CVE-ID 78 | ``` 79 | $ go-cti search cti T1037 80 | { 81 | "type": "Technique", 82 | "technique": { 83 | "technique_id": "T1037", 84 | "type": "MITRE-ATTACK", 85 | "name": "T1037: Boot or Logon Initialization Scripts", 86 | ... 87 | } 88 | } 89 | 90 | $ go-cti search cve CVE-2017-15131 91 | [ 92 | "T1037", 93 | "CAPEC-578", 94 | "T1562.001", 95 | "T1014", 96 | "CAPEC-502", 97 | "CAPEC-551", 98 | "T1547.006", 99 | "T1080", 100 | "CAPEC-563", 101 | "T1546.004", 102 | "T1574.011", 103 | "CAPEC-536", 104 | "CAPEC-550", 105 | "T1542.003", 106 | "CAPEC-19", 107 | "T1543.002", 108 | "CAPEC-503", 109 | "T1553.004", 110 | "T1546.001", 111 | "CAPEC-564", 112 | "T1547", 113 | "CAPEC-478", 114 | "CAPEC-558", 115 | "CAPEC-562", 116 | "CAPEC-546", 117 | "T1543.004", 118 | "CAPEC-552", 119 | "CAPEC-556", 120 | "CAPEC-479", 121 | "T1543.003", 122 | "T1546.008", 123 | "T1543.001", 124 | "CAPEC-441" 125 | ] 126 | 127 | $ search attacker T1078 T1550.002 T1588.002 128 | [ 129 | "S0122", // T1550.002 130 | "G0011" // T1078, T1588.002 131 | ] 132 | ``` 133 | 134 | ## Sever mode 135 | ```console 136 | $ go-cti server 137 | INFO[04-15|00:23:43] Starting HTTP Server... 138 | INFO[04-15|00:23:43] Listening... URL=127.0.0.1:1329 139 | 140 | ____ __ 141 | / __/___/ / ___ 142 | / _// __/ _ \/ _ \ 143 | /___/\__/_//_/\___/ v3.3.10-dev 144 | High performance, minimalist Go web framework 145 | https://echo.labstack.com 146 | ____________________________________O/_______ 147 | O\ 148 | ⇨ http server started on 127.0.0.1:1329 149 | {"time":"2022-04-15T00:24:23.773648507+09:00","id":"","remote_ip":"127.0.0.1","host":"127.0.0.1:1329","method":"GET","uri":"/cves/CVE-2021-46628","user_agent":"curl/7.68.0","status":200,"error":"","latency":143229557,"latency_human":"143.229557ms","bytes_in":0,"bytes_out":358064} 150 | {"time":"2022-04-15T00:26:34.068344126+09:00","id":"","remote_ip":"127.0.0.1","host":"127.0.0.1:1329","method":"POST","uri":"/multi-cves","user_agent":"curl/7.68.0","status":200,"error":"","latency":137130582,"latency_human":"137.130582ms","bytes_in":28,"bytes_out":358083} 151 | 152 | $ curl http://127.0.0.1:1329/ctis/CAPEC-540 | jq . 153 | { 154 | "type": "Technique", 155 | "technique": { 156 | "technique_id": "CAPEC-540", 157 | "type": "CAPEC", 158 | "name": "CAPEC-540: Overread Buffers", 159 | // ... 160 | } 161 | } 162 | 163 | $ curl http://127.0.0.1:1329/cves/CVE-2021-46628 | jq . 164 | [ 165 | "CAPEC-540" 166 | ] 167 | 168 | 169 | $ curl -d "{\"args\": [\"CVE-2021-46628\"]}" -H "Content-Type: application/json" 127.0.0.1:1329/multi-cves | jq . 170 | { 171 | "CVE-2021-46628": [ 172 | "CAPEC-540" 173 | ] 174 | } 175 | ``` 176 | 177 | ## How to generate the Technique Dictionary for Vuls 178 | - main.go 179 | ```go 180 | package main 181 | 182 | import ( 183 | "fmt" 184 | "os" 185 | "slices" 186 | "strings" 187 | 188 | "gorm.io/driver/sqlite" 189 | "gorm.io/gorm" 190 | 191 | ctiDB "github.com/vulsio/go-cti/db" 192 | "github.com/vulsio/go-cti/models" 193 | ) 194 | 195 | func main() { 196 | db, err := gorm.Open(sqlite.Open("go-cti.sqlite3")) 197 | if err != nil { 198 | fmt.Fprintf(os.Stderr, "failed to open DB. err: %s\n", err) 199 | os.Exit(1) 200 | } 201 | techniqueIDs := []string{} 202 | if err := db.Model(&models.Technique{}).Select("technique_id").Find(&techniqueIDs).Error; err != nil { 203 | fmt.Fprintf(os.Stderr, "failed to get techniqueIDs. err: %s\n", err) 204 | os.Exit(1) 205 | } 206 | sqlDB, err := db.DB() 207 | if err != nil { 208 | fmt.Fprintf(os.Stderr, "failed to get sqlDB. err: %s\n", err) 209 | os.Exit(1) 210 | } 211 | if err := sqlDB.Close(); err != nil { 212 | fmt.Fprintf(os.Stderr, "failed to close sqlDB. err: %s\n", err) 213 | os.Exit(1) 214 | } 215 | 216 | driver, locked, err := ctiDB.NewDB("sqlite3", "go-cti.sqlite3", false, ctiDB.Option{}) 217 | if locked || err != nil { 218 | fmt.Fprintf(os.Stderr, "failed to new DB. locked: %t, err: %s\n", locked, err) 219 | os.Exit(1) 220 | } 221 | 222 | fmt.Println("// Technique has MITER ATT&CK Technique or CAPEC information") 223 | fmt.Printf("type Technique struct {\n Name string `json:\"name\"`\n Platforms []string `json:\"platforms\"`\n}\n\n") 224 | fmt.Println("// TechniqueDict is the MITRE ATT&CK Technique and CAPEC dictionary") 225 | fmt.Printf("var TechniqueDict = map[string]Technique{\n") 226 | slices.Sort(techniqueIDs) 227 | for _, techniqueID := range techniqueIDs { 228 | cti, err := driver.GetCtiByCtiID(techniqueID) 229 | if err != nil { 230 | fmt.Fprintf(os.Stderr, "failed to get CTI. err: %s\n", err) 231 | os.Exit(1) 232 | } 233 | 234 | if cti.Technique.Type == models.MitreAttackType { 235 | tactics := []string{} 236 | for _, phase := range cti.Technique.MitreAttack.KillChainPhases { 237 | tactics = append(tactics, phase.Tactic) 238 | } 239 | slices.Sort(tactics) 240 | 241 | platforms := []string{} 242 | for _, platform := range cti.Technique.MitreAttack.Platforms { 243 | platforms = append(platforms, fmt.Sprintf("\"%s\"", platform.Platform)) 244 | } 245 | slices.Sort(platforms) 246 | 247 | fmt.Printf("\"%s\": {\n Name: %q,\n Platforms: []string{%s},\n},\n", 248 | cti.Technique.TechniqueID, 249 | fmt.Sprintf("%s => %s", strings.Join(tactics, ", "), cti.Technique.Name), 250 | strings.Join(platforms, ", "), 251 | ) 252 | } else { 253 | fmt.Printf("\"%s\": {\n Name: %q,\n},\n", 254 | cti.Technique.TechniqueID, 255 | cti.Technique.Name, 256 | ) 257 | } 258 | } 259 | fmt.Println("}") 260 | 261 | if err := driver.CloseDB(); err != nil { 262 | fmt.Fprintf(os.Stderr, "failed to close DB. err: %s", err) 263 | os.Exit(1) 264 | } 265 | } 266 | ``` 267 | 268 | ```console 269 | $ ls 270 | go-cti.sqlite3 go.mod go.sum main.go 271 | 272 | $ go run main.go 273 | ``` 274 | 275 | ## License 276 | MIT 277 | 278 | ## Author 279 | [MaineK00n](https://twitter.com/MaineK00n) -------------------------------------------------------------------------------- /commands/fetch-cti.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "errors" 5 | "time" 6 | 7 | "github.com/inconshreveable/log15" 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | "golang.org/x/xerrors" 11 | 12 | "github.com/vulsio/go-cti/db" 13 | "github.com/vulsio/go-cti/fetcher" 14 | "github.com/vulsio/go-cti/models" 15 | "github.com/vulsio/go-cti/utils" 16 | ) 17 | 18 | var fetchMitreCtiCmd = &cobra.Command{ 19 | Use: "threat", 20 | Short: "Fetch the data of mitre/cti cve's list", 21 | Long: `Fetch the data of mitre/cti cve's list`, 22 | RunE: fetchMitreCti, 23 | } 24 | 25 | func init() { 26 | fetchCmd.AddCommand(fetchMitreCtiCmd) 27 | } 28 | 29 | func fetchMitreCti(_ *cobra.Command, _ []string) (err error) { 30 | if err := utils.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil { 31 | return xerrors.Errorf("Failed to SetLogger. err: %w", err) 32 | } 33 | 34 | driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{}) 35 | if err != nil { 36 | if errors.Is(err, db.ErrDBLocked) { 37 | return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err) 38 | } 39 | return xerrors.Errorf("Failed to open DB. err: %w", err) 40 | } 41 | 42 | fetchMeta, err := driver.GetFetchMeta() 43 | if err != nil { 44 | return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err) 45 | } 46 | if fetchMeta.OutDated() { 47 | return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion}) 48 | } 49 | // If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion. 50 | if err := driver.UpsertFetchMeta(fetchMeta); err != nil { 51 | return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err) 52 | } 53 | 54 | log15.Info("Fetching Cyber Threat Intelligence and CVE-ID to CTI-ID Mappings") 55 | techniques, mappings, attackers, err := fetcher.FetchCti() 56 | if err != nil { 57 | return xerrors.Errorf("Failed to fetch Cyber Threat Intelligence. err: %w", err) 58 | } 59 | log15.Info("Fetched Cyber Threat Intelligence and CVE-ID to CTI-ID Mappings", "techniques", len(techniques), "mappings", len(mappings), "attackers", len(attackers)) 60 | 61 | log15.Info("Insert Cyber Threat Intelligences and CVE-ID to CTI-ID Mappings into go-cti.", "db", driver.Name()) 62 | if err := driver.InsertCti(techniques, mappings, attackers); err != nil { 63 | return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err) 64 | } 65 | 66 | fetchMeta.LastFetchedAt = time.Now() 67 | if err := driver.UpsertFetchMeta(fetchMeta); err != nil { 68 | return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err) 69 | } 70 | 71 | return nil 72 | } 73 | -------------------------------------------------------------------------------- /commands/fetch.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | "github.com/spf13/viper" 6 | ) 7 | 8 | // fetchCmd represents the fetch command 9 | var fetchCmd = &cobra.Command{ 10 | Use: "fetch", 11 | Short: "Fetch the data of mitre/cti", 12 | Long: `Fetch the data of mitre/cti`, 13 | } 14 | 15 | func init() { 16 | RootCmd.AddCommand(fetchCmd) 17 | 18 | fetchCmd.PersistentFlags().Int("batch-size", 50, "The number of batch size to insert.") 19 | _ = viper.BindPFlag("batch-size", fetchCmd.PersistentFlags().Lookup("batch-size")) 20 | } 21 | -------------------------------------------------------------------------------- /commands/root.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/inconshreveable/log15" 9 | homedir "github.com/mitchellh/go-homedir" 10 | "github.com/spf13/cobra" 11 | "github.com/spf13/viper" 12 | 13 | "github.com/vulsio/go-cti/utils" 14 | ) 15 | 16 | var cfgFile string 17 | 18 | // RootCmd represents the base command when called without any subcommands 19 | var RootCmd = &cobra.Command{ 20 | Use: "go-cti", 21 | Short: "Go collect Cyber Threat Intelligence", 22 | Long: `Go collect Cyber Threat Intelligence`, 23 | SilenceErrors: true, 24 | SilenceUsage: true, 25 | } 26 | 27 | func init() { 28 | cobra.OnInitialize(initConfig) 29 | 30 | RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.go-cti.yaml)") 31 | 32 | RootCmd.PersistentFlags().Bool("log-to-file", false, "output log to file") 33 | _ = viper.BindPFlag("log-to-file", RootCmd.PersistentFlags().Lookup("log-to-file")) 34 | 35 | RootCmd.PersistentFlags().String("log-dir", utils.GetDefaultLogDir(), "/path/to/log") 36 | _ = viper.BindPFlag("log-dir", RootCmd.PersistentFlags().Lookup("log-dir")) 37 | 38 | RootCmd.PersistentFlags().Bool("log-json", false, "output log as JSON") 39 | _ = viper.BindPFlag("log-json", RootCmd.PersistentFlags().Lookup("log-json")) 40 | 41 | RootCmd.PersistentFlags().Bool("debug", false, "debug mode (default: false)") 42 | _ = viper.BindPFlag("debug", RootCmd.PersistentFlags().Lookup("debug")) 43 | 44 | RootCmd.PersistentFlags().Bool("debug-sql", false, "SQL debug mode") 45 | _ = viper.BindPFlag("debug-sql", RootCmd.PersistentFlags().Lookup("debug-sql")) 46 | 47 | RootCmd.PersistentFlags().String("dbpath", filepath.Join(os.Getenv("PWD"), "go-cti.sqlite3"), "/path/to/sqlite3 or SQL connection string") 48 | _ = viper.BindPFlag("dbpath", RootCmd.PersistentFlags().Lookup("dbpath")) 49 | 50 | RootCmd.PersistentFlags().String("dbtype", "sqlite3", "Database type to store data in (sqlite3, mysql, postgres or redis supported)") 51 | _ = viper.BindPFlag("dbtype", RootCmd.PersistentFlags().Lookup("dbtype")) 52 | 53 | // proxy support 54 | RootCmd.PersistentFlags().String("http-proxy", "", "http://proxy-url:port (default: empty)") 55 | _ = viper.BindPFlag("http-proxy", RootCmd.PersistentFlags().Lookup("http-proxy")) 56 | } 57 | 58 | // initConfig reads in config file and ENV variables if set. 59 | func initConfig() { 60 | if cfgFile != "" { 61 | viper.SetConfigFile(cfgFile) 62 | } else { 63 | // Find home directory. 64 | home, err := homedir.Dir() 65 | if err != nil { 66 | log15.Error("Failed to find home directory.", "err", err) 67 | os.Exit(1) 68 | } 69 | 70 | // Search config in home directory with name ".go-cti" (without extension). 71 | viper.AddConfigPath(home) 72 | viper.SetConfigName(".go-cti") 73 | } 74 | 75 | viper.AutomaticEnv() // read in environment variables that match 76 | 77 | // If a config file is found, read it in. 78 | if err := viper.ReadInConfig(); err == nil { 79 | fmt.Println("Using config file:", viper.ConfigFileUsed()) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /commands/search.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | "golang.org/x/xerrors" 11 | 12 | "github.com/vulsio/go-cti/db" 13 | "github.com/vulsio/go-cti/models" 14 | "github.com/vulsio/go-cti/utils" 15 | ) 16 | 17 | // fetchCmd represents the fetch command 18 | var searchCmd = &cobra.Command{ 19 | Use: "search", 20 | Short: "Search the data of mitre/cti form DB", 21 | Long: `Search the data of mitrc/cti form DB`, 22 | Args: func(_ *cobra.Command, args []string) error { 23 | if len(args) < 2 { 24 | fmt.Println("[usage] $ go-cti search (cti|cve|attacker) $id1(, $id2...)") 25 | return xerrors.New("Failed to search. err: argument is missing") 26 | } 27 | switch args[0] { 28 | case "cti", "cve", "attacker": 29 | return nil 30 | default: 31 | fmt.Println("[usage] $ go-cti search (cti|cve|attacker) $id1(, $id2...)") 32 | return xerrors.New(`Failed to search. err: search target is inappropriate, select "cti", "cve" or "attacker".`) 33 | } 34 | }, 35 | RunE: searchCti, 36 | } 37 | 38 | func init() { 39 | RootCmd.AddCommand(searchCmd) 40 | } 41 | 42 | func searchCti(_ *cobra.Command, args []string) error { 43 | if err := utils.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil { 44 | return xerrors.Errorf("Failed to SetLogger. err: %w", err) 45 | } 46 | 47 | driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{}) 48 | if err != nil { 49 | if errors.Is(err, db.ErrDBLocked) { 50 | return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err) 51 | } 52 | return xerrors.Errorf("Failed to open DB. err: %w", err) 53 | } 54 | 55 | fetchMeta, err := driver.GetFetchMeta() 56 | if err != nil { 57 | return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err) 58 | } 59 | if fetchMeta.OutDated() { 60 | return xerrors.Errorf("Failed to search command. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion}) 61 | } 62 | 63 | var result []byte 64 | switch args[0] { 65 | case "cti": 66 | if len(args[1:]) == 1 { 67 | cti, err := driver.GetCtiByCtiID(args[1]) 68 | if err != nil { 69 | return xerrors.Errorf("Failed to search CTI. err: %w", err) 70 | } 71 | result, err = json.MarshalIndent(cti, "", " ") 72 | if err != nil { 73 | return xerrors.Errorf("Failed to marshal json. err: %w", err) 74 | } 75 | } else { 76 | ctis, err := driver.GetCtisByMultiCtiID(args[1:]) 77 | if err != nil { 78 | return xerrors.Errorf("Failed to search CTIs. err: %w", err) 79 | } 80 | result, err = json.MarshalIndent(ctis, "", " ") 81 | if err != nil { 82 | return xerrors.Errorf("Failed to marshal json. err: %w", err) 83 | } 84 | } 85 | case "cve": 86 | if len(args[1:]) == 1 { 87 | techniques, err := driver.GetTechniqueIDsByCveID(args[1]) 88 | if err != nil { 89 | return xerrors.Errorf("Failed to get CTI. err: %w", err) 90 | } 91 | if len(techniques) == 0 { 92 | return nil 93 | } 94 | result, err = json.MarshalIndent(techniques, "", " ") 95 | if err != nil { 96 | return xerrors.Errorf("Failed to marshal json. err: %w", err) 97 | } 98 | } else { 99 | techniques, err := driver.GetTechniqueIDsByMultiCveID(args[1:]) 100 | if err != nil { 101 | return xerrors.Errorf("Failed to get CTI. err: %w", err) 102 | } 103 | if len(techniques) == 0 { 104 | return nil 105 | } 106 | result, err = json.MarshalIndent(techniques, "", " ") 107 | if err != nil { 108 | return xerrors.Errorf("Failed to marshal json. err: %w", err) 109 | } 110 | } 111 | case "attacker": 112 | attackers, err := driver.GetAttackerIDsByTechniqueIDs(args[1:]) 113 | if err != nil { 114 | return xerrors.Errorf("Failed to get attackers. err: %w", err) 115 | } 116 | if len(attackers) == 0 { 117 | return nil 118 | } 119 | result, err = json.MarshalIndent(attackers, "", " ") 120 | if err != nil { 121 | return xerrors.Errorf("Failed to marshal json. err: %w", err) 122 | } 123 | } 124 | fmt.Printf("%s\n", string(result)) 125 | return nil 126 | } 127 | -------------------------------------------------------------------------------- /commands/server.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/inconshreveable/log15" 7 | "github.com/spf13/cobra" 8 | "github.com/spf13/viper" 9 | "golang.org/x/xerrors" 10 | 11 | "github.com/vulsio/go-cti/db" 12 | "github.com/vulsio/go-cti/models" 13 | "github.com/vulsio/go-cti/server" 14 | "github.com/vulsio/go-cti/utils" 15 | ) 16 | 17 | // serverCmd represents the server command 18 | var serverCmd = &cobra.Command{ 19 | Use: "server", 20 | Short: "Start go-cti HTTP server", 21 | Long: `Start go-cti HTTP server`, 22 | RunE: executeServer, 23 | } 24 | 25 | func init() { 26 | RootCmd.AddCommand(serverCmd) 27 | 28 | serverCmd.PersistentFlags().String("bind", "", "HTTP server bind to IP address") 29 | _ = viper.BindPFlag("bind", serverCmd.PersistentFlags().Lookup("bind")) 30 | viper.SetDefault("bind", "127.0.0.1") 31 | 32 | serverCmd.PersistentFlags().String("port", "", "HTTP server port number") 33 | _ = viper.BindPFlag("port", serverCmd.PersistentFlags().Lookup("port")) 34 | viper.SetDefault("port", "1329") 35 | 36 | } 37 | 38 | func executeServer(_ *cobra.Command, _ []string) (err error) { 39 | if err := utils.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil { 40 | return xerrors.Errorf("Failed to SetLogger. err: %w", err) 41 | } 42 | 43 | driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{}) 44 | if err != nil { 45 | if errors.Is(err, db.ErrDBLocked) { 46 | return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err) 47 | } 48 | return xerrors.Errorf("Failed to open DB. err: %w", err) 49 | } 50 | 51 | fetchMeta, err := driver.GetFetchMeta() 52 | if err != nil { 53 | return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err) 54 | } 55 | if fetchMeta.OutDated() { 56 | return xerrors.Errorf("Failed to start server. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion}) 57 | } 58 | 59 | log15.Info("Starting HTTP Server...") 60 | if err = server.Start(viper.GetBool("log-to-file"), viper.GetString("log-dir"), driver); err != nil { 61 | return xerrors.Errorf("Failed to start server. err: %w", err) 62 | } 63 | 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /commands/version.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/cobra" 7 | 8 | "github.com/vulsio/go-cti/config" 9 | ) 10 | 11 | func init() { 12 | RootCmd.AddCommand(versionCmd) 13 | } 14 | 15 | var versionCmd = &cobra.Command{ 16 | Use: "version", 17 | Short: "Show version", 18 | Long: `Show version`, 19 | Run: func(_ *cobra.Command, _ []string) { 20 | fmt.Printf("go-cti %s %s\n", config.Version, config.Revision) 21 | }, 22 | } 23 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // Version of go-cti 4 | var Version = "`make build` or `make install` will show the version" 5 | 6 | // Revision of Git 7 | var Revision string 8 | -------------------------------------------------------------------------------- /db/db.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | "golang.org/x/xerrors" 8 | 9 | "github.com/vulsio/go-cti/models" 10 | ) 11 | 12 | // DB : 13 | type DB interface { 14 | Name() string 15 | OpenDB(dbType, dbPath string, debugSQL bool, option Option) error 16 | MigrateDB() error 17 | CloseDB() error 18 | 19 | IsGoCTIModelV1() (bool, error) 20 | GetFetchMeta() (*models.FetchMeta, error) 21 | UpsertFetchMeta(*models.FetchMeta) error 22 | 23 | InsertCti([]models.Technique, []models.CveToTechniques, []models.Attacker) error 24 | GetCtiByCtiID(string) (models.CTI, error) 25 | GetCtisByMultiCtiID([]string) ([]models.CTI, error) 26 | GetTechniqueIDsByCveID(string) ([]string, error) 27 | GetTechniqueIDsByMultiCveID([]string) (map[string][]string, error) 28 | GetAttackerIDsByTechniqueIDs([]string) ([]string, error) 29 | } 30 | 31 | // Option : 32 | type Option struct { 33 | RedisTimeout time.Duration 34 | } 35 | 36 | // NewDB : 37 | func NewDB(dbType string, dbPath string, debugSQL bool, option Option) (driver DB, err error) { 38 | if driver, err = newDB(dbType); err != nil { 39 | return driver, xerrors.Errorf("Failed to new db. err: %w", err) 40 | } 41 | 42 | if err := driver.OpenDB(dbType, dbPath, debugSQL, option); err != nil { 43 | return nil, xerrors.Errorf("Failed to open db. err: %w", err) 44 | } 45 | 46 | isV1, err := driver.IsGoCTIModelV1() 47 | if err != nil { 48 | return nil, xerrors.Errorf("Failed to IsGoCTIModelV1. err: %w", err) 49 | } 50 | if isV1 { 51 | return nil, xerrors.New("Failed to NewDB. Since SchemaVersion is incompatible, delete Database and fetch again.") 52 | } 53 | 54 | if err := driver.MigrateDB(); err != nil { 55 | return driver, xerrors.Errorf("Failed to migrate db. err: %w", err) 56 | } 57 | return driver, nil 58 | } 59 | 60 | func newDB(dbType string) (DB, error) { 61 | switch dbType { 62 | case dialectSqlite3, dialectMysql, dialectPostgreSQL: 63 | return &RDBDriver{name: dbType}, nil 64 | case dialectRedis: 65 | return &RedisDriver{name: dbType}, nil 66 | } 67 | return nil, xerrors.Errorf("Invalid database dialect, %s", dbType) 68 | } 69 | 70 | func classCtiIDs(ctiIDs []string) ([]string, []string, error) { 71 | techniques := []string{} 72 | attackers := []string{} 73 | for _, ctiID := range ctiIDs { 74 | if strings.HasPrefix(ctiID, "T") || strings.HasPrefix(ctiID, "CAPEC") { 75 | techniques = append(techniques, ctiID) 76 | } else if strings.HasPrefix(ctiID, "G") || strings.HasPrefix(ctiID, "S") { 77 | attackers = append(attackers, ctiID) 78 | } else { 79 | return nil, nil, xerrors.Errorf(`Failed to class CTI-IDs. err: invalid CTI-ID(%s). The prefix of the CTI-ID must be "T", "CAPEC", "G", or "S".`, ctiID) 80 | } 81 | } 82 | return techniques, attackers, nil 83 | } 84 | -------------------------------------------------------------------------------- /db/rdb.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | "errors" 7 | "io" 8 | "log" 9 | "os" 10 | "slices" 11 | "time" 12 | 13 | "github.com/cheggaaa/pb/v3" 14 | "github.com/glebarez/sqlite" 15 | "github.com/inconshreveable/log15" 16 | "github.com/spf13/viper" 17 | "golang.org/x/xerrors" 18 | "gorm.io/driver/mysql" 19 | "gorm.io/driver/postgres" 20 | "gorm.io/gorm" 21 | "gorm.io/gorm/clause" 22 | "gorm.io/gorm/logger" 23 | 24 | "github.com/vulsio/go-cti/config" 25 | "github.com/vulsio/go-cti/models" 26 | ) 27 | 28 | const ( 29 | dialectSqlite3 = "sqlite3" 30 | dialectMysql = "mysql" 31 | dialectPostgreSQL = "postgres" 32 | ) 33 | 34 | // RDBDriver : 35 | type RDBDriver struct { 36 | name string 37 | conn *gorm.DB 38 | } 39 | 40 | // https://github.com/mattn/go-sqlite3/blob/edc3bb69551dcfff02651f083b21f3366ea2f5ab/error.go#L18-L66 41 | type errNo int 42 | 43 | type sqliteError struct { 44 | Code errNo /* The error code returned by SQLite */ 45 | } 46 | 47 | // result codes from http://www.sqlite.org/c3ref/c_abort.html 48 | var ( 49 | errBusy = errNo(5) /* The database file is locked */ 50 | errLocked = errNo(6) /* A table in the database is locked */ 51 | ) 52 | 53 | // ErrDBLocked : 54 | var ErrDBLocked = xerrors.New("database is locked") 55 | 56 | // Name return db name 57 | func (r *RDBDriver) Name() string { 58 | return r.name 59 | } 60 | 61 | // OpenDB opens Database 62 | func (r *RDBDriver) OpenDB(dbType, dbPath string, debugSQL bool, _ Option) (err error) { 63 | gormConfig := gorm.Config{ 64 | DisableForeignKeyConstraintWhenMigrating: true, 65 | Logger: logger.New( 66 | log.New(os.Stderr, "\r\n", log.LstdFlags), 67 | logger.Config{ 68 | LogLevel: logger.Silent, 69 | }, 70 | ), 71 | } 72 | 73 | if debugSQL { 74 | gormConfig.Logger = logger.New( 75 | log.New(os.Stderr, "\r\n", log.LstdFlags), 76 | logger.Config{ 77 | SlowThreshold: time.Second, 78 | LogLevel: logger.Info, 79 | Colorful: true, 80 | }, 81 | ) 82 | } 83 | 84 | switch r.name { 85 | case dialectSqlite3: 86 | r.conn, err = gorm.Open(sqlite.Open(dbPath), &gormConfig) 87 | if err != nil { 88 | parsedErr, marshalErr := json.Marshal(err) 89 | if marshalErr != nil { 90 | return xerrors.Errorf("Failed to marshal err. err: %w", marshalErr) 91 | } 92 | 93 | var errMsg sqliteError 94 | if unmarshalErr := json.Unmarshal(parsedErr, &errMsg); unmarshalErr != nil { 95 | return xerrors.Errorf("Failed to unmarshal. err: %w", unmarshalErr) 96 | } 97 | 98 | switch errMsg.Code { 99 | case errBusy, errLocked: 100 | return xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, ErrDBLocked) 101 | default: 102 | return xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, err) 103 | } 104 | } 105 | 106 | r.conn.Exec("PRAGMA foreign_keys = ON") 107 | case dialectMysql: 108 | r.conn, err = gorm.Open(mysql.Open(dbPath), &gormConfig) 109 | if err != nil { 110 | return xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, err) 111 | } 112 | case dialectPostgreSQL: 113 | r.conn, err = gorm.Open(postgres.Open(dbPath), &gormConfig) 114 | if err != nil { 115 | return xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, err) 116 | } 117 | default: 118 | return xerrors.Errorf("Not Supported DB dialects. r.name: %s", r.name) 119 | } 120 | return nil 121 | } 122 | 123 | // CloseDB close Database 124 | func (r *RDBDriver) CloseDB() (err error) { 125 | if r.conn == nil { 126 | return 127 | } 128 | 129 | var sqlDB *sql.DB 130 | if sqlDB, err = r.conn.DB(); err != nil { 131 | return xerrors.Errorf("Failed to get DB Object. err : %w", err) 132 | } 133 | if err = sqlDB.Close(); err != nil { 134 | return xerrors.Errorf("Failed to close DB. Type: %s. err: %w", r.name, err) 135 | } 136 | return 137 | } 138 | 139 | // MigrateDB migrates Database 140 | func (r *RDBDriver) MigrateDB() error { 141 | if err := r.conn.AutoMigrate( 142 | &models.FetchMeta{}, 143 | 144 | &models.CveToTechniques{}, 145 | &models.CveToTechniqueID{}, 146 | 147 | &models.Technique{}, 148 | &models.TechniqueReference{}, 149 | &models.Mitigation{}, 150 | 151 | &models.MitreAttack{}, 152 | &models.CapecID{}, 153 | &models.KillChainPhase{}, 154 | &models.DataSource{}, 155 | &models.Procedure{}, 156 | &models.TechniquePlatform{}, 157 | &models.PermissionRequired{}, 158 | &models.EffectivePermission{}, 159 | &models.DefenseBypassed{}, 160 | &models.ImpactType{}, 161 | &models.SubTechnique{}, 162 | 163 | &models.Capec{}, 164 | &models.AttackID{}, 165 | &models.Relationship{}, 166 | &models.Domain{}, 167 | &models.AlternateTerm{}, 168 | &models.ExampleInstance{}, 169 | &models.Prerequisite{}, 170 | &models.ResourceRequired{}, 171 | &models.SkillRequired{}, 172 | &models.Consequence{}, 173 | &models.RelatedWeakness{}, 174 | 175 | &models.Attacker{}, 176 | &models.TechniqueUsed{}, 177 | &models.AttackerReference{}, 178 | 179 | &models.AttackerGroup{}, 180 | &models.AssociatedGroup{}, 181 | &models.SoftwareUsed{}, 182 | 183 | &models.AttackerSoftware{}, 184 | &models.AssociatedSoftware{}, 185 | &models.SoftwarePlatform{}, 186 | &models.GroupUsed{}, 187 | ); err != nil { 188 | switch r.name { 189 | case dialectSqlite3: 190 | if r.name == dialectSqlite3 { 191 | parsedErr, marshalErr := json.Marshal(err) 192 | if marshalErr != nil { 193 | return xerrors.Errorf("Failed to marshal err. err: %w", marshalErr) 194 | } 195 | 196 | var errMsg sqliteError 197 | if unmarshalErr := json.Unmarshal(parsedErr, &errMsg); unmarshalErr != nil { 198 | return xerrors.Errorf("Failed to unmarshal. err: %w", unmarshalErr) 199 | } 200 | 201 | switch errMsg.Code { 202 | case errBusy, errLocked: 203 | return xerrors.Errorf("Failed to migrate. err: %w", ErrDBLocked) 204 | default: 205 | return xerrors.Errorf("Failed to migrate. err: %w", err) 206 | } 207 | } 208 | case dialectMysql, dialectPostgreSQL: 209 | return xerrors.Errorf("Failed to migrate. err: %w", err) 210 | default: 211 | return xerrors.Errorf("Not Supported DB dialects. r.name: %s", r.name) 212 | } 213 | } 214 | 215 | return nil 216 | } 217 | 218 | // IsGoCTIModelV1 determines if the DB was created at the time of go-cti Model v1 219 | func (r *RDBDriver) IsGoCTIModelV1() (bool, error) { 220 | if r.conn.Migrator().HasTable(&models.FetchMeta{}) { 221 | return false, nil 222 | } 223 | 224 | var ( 225 | count int64 226 | err error 227 | ) 228 | switch r.name { 229 | case dialectSqlite3: 230 | err = r.conn.Table("sqlite_master").Where("type = ?", "table").Count(&count).Error 231 | case dialectMysql: 232 | err = r.conn.Table("information_schema.tables").Where("table_schema = ?", r.conn.Migrator().CurrentDatabase()).Count(&count).Error 233 | case dialectPostgreSQL: 234 | err = r.conn.Table("pg_tables").Where("schemaname = ?", "public").Count(&count).Error 235 | } 236 | 237 | if count > 0 { 238 | return true, nil 239 | } 240 | return false, err 241 | } 242 | 243 | // GetFetchMeta get FetchMeta from Database 244 | func (r *RDBDriver) GetFetchMeta() (fetchMeta *models.FetchMeta, err error) { 245 | if err = r.conn.Take(&fetchMeta).Error; err != nil { 246 | if !errors.Is(err, gorm.ErrRecordNotFound) { 247 | return nil, err 248 | } 249 | return &models.FetchMeta{GoCTIRevision: config.Revision, SchemaVersion: models.LatestSchemaVersion, LastFetchedAt: time.Date(1000, time.January, 1, 0, 0, 0, 0, time.UTC)}, nil 250 | } 251 | 252 | return fetchMeta, nil 253 | } 254 | 255 | // UpsertFetchMeta upsert FetchMeta to Database 256 | func (r *RDBDriver) UpsertFetchMeta(fetchMeta *models.FetchMeta) error { 257 | fetchMeta.GoCTIRevision = config.Revision 258 | fetchMeta.SchemaVersion = models.LatestSchemaVersion 259 | return r.conn.Save(fetchMeta).Error 260 | } 261 | 262 | // InsertCti : 263 | func (r *RDBDriver) InsertCti(techniques []models.Technique, mappings []models.CveToTechniques, attackers []models.Attacker) (err error) { 264 | return r.deleteAndInsertCti(r.conn, techniques, mappings, attackers) 265 | } 266 | 267 | func (r *RDBDriver) deleteAndInsertCti(conn *gorm.DB, techniques []models.Technique, mappings []models.CveToTechniques, attackers []models.Attacker) (err error) { 268 | tx := conn.Begin() 269 | defer func() { 270 | if err != nil { 271 | tx.Rollback() 272 | return 273 | } 274 | tx.Commit() 275 | }() 276 | 277 | // Delete all old records 278 | for _, table := range []interface{}{ 279 | models.GroupUsed{}, models.SoftwarePlatform{}, models.AssociatedSoftware{}, models.AttackerSoftware{}, 280 | models.SoftwareUsed{}, models.AssociatedGroup{}, models.AttackerGroup{}, 281 | models.AttackerReference{}, models.TechniqueUsed{}, models.Attacker{}, 282 | models.RelatedWeakness{}, models.Consequence{}, models.SkillRequired{}, models.ResourceRequired{}, models.Prerequisite{}, models.ExampleInstance{}, models.AlternateTerm{}, models.Domain{}, models.Relationship{}, models.AttackID{}, models.Capec{}, 283 | models.SubTechnique{}, models.ImpactType{}, models.DefenseBypassed{}, models.EffectivePermission{}, models.PermissionRequired{}, models.TechniquePlatform{}, models.Procedure{}, models.DataSource{}, models.KillChainPhase{}, models.CapecID{}, models.MitreAttack{}, 284 | models.Mitigation{}, models.TechniqueReference{}, models.Technique{}, 285 | models.CveToTechniqueID{}, models.CveToTechniques{}, 286 | } { 287 | if err := tx.Session(&gorm.Session{AllowGlobalUpdate: true}).Unscoped().Delete(table).Error; err != nil { 288 | return xerrors.Errorf("Failed to delete old records. err: %w", err) 289 | } 290 | } 291 | 292 | batchSize := viper.GetInt("batch-size") 293 | if batchSize < 1 { 294 | return xerrors.New("Failed to set batch-size. err: batch-size option is not set properly") 295 | } 296 | 297 | log15.Info("Inserting Cyber Threat Intelligences...") 298 | 299 | log15.Info("Inserting Techniques...") 300 | bar := pb.StartNew(len(techniques)).SetWriter(func() io.Writer { 301 | if viper.GetBool("log-json") { 302 | return io.Discard 303 | } 304 | return os.Stderr 305 | }()) 306 | for chunk := range slices.Chunk(techniques, batchSize) { 307 | if err = tx.Create(chunk).Error; err != nil { 308 | return xerrors.Errorf("Failed to insert. err: %w", err) 309 | } 310 | bar.Add(len(chunk)) 311 | } 312 | bar.Finish() 313 | 314 | log15.Info("Inserting CVE-ID to CTI-ID CveToTechniques...") 315 | bar = pb.StartNew(len(mappings)).SetWriter(func() io.Writer { 316 | if viper.GetBool("log-json") { 317 | return io.Discard 318 | } 319 | return os.Stderr 320 | }()) 321 | for chunk := range slices.Chunk(mappings, batchSize) { 322 | if err = tx.Create(chunk).Error; err != nil { 323 | return xerrors.Errorf("Failed to insert. err: %w", err) 324 | } 325 | bar.Add(len(chunk)) 326 | } 327 | bar.Finish() 328 | 329 | log15.Info("Inserting Attackers...") 330 | bar = pb.StartNew(len(attackers)).SetWriter(func() io.Writer { 331 | if viper.GetBool("log-json") { 332 | return io.Discard 333 | } 334 | return os.Stderr 335 | }()) 336 | for chunk := range slices.Chunk(attackers, batchSize) { 337 | if err = tx.Create(chunk).Error; err != nil { 338 | return xerrors.Errorf("Failed to insert. err: %w", err) 339 | } 340 | bar.Add(len(chunk)) 341 | } 342 | bar.Finish() 343 | 344 | return nil 345 | } 346 | 347 | // GetCtiByCtiID : 348 | func (r *RDBDriver) GetCtiByCtiID(ctiID string) (models.CTI, error) { 349 | techniqueIDs, attackerIDs, err := classCtiIDs([]string{ctiID}) 350 | if err != nil { 351 | return models.CTI{}, xerrors.Errorf("Failed to classCtiIDs. err: %w", err) 352 | } 353 | 354 | cti := models.CTI{} 355 | if len(techniqueIDs) > 0 { 356 | cti.Type = models.TechniqueType 357 | 358 | if err := r.conn. 359 | Preload("References"). 360 | Preload("Mitigations"). 361 | Where(&models.Technique{TechniqueID: techniqueIDs[0]}). 362 | Take(&cti.Technique).Error; err != nil { 363 | return models.CTI{}, xerrors.Errorf("Failed to get Technique by CTI-ID. err: %w", err) 364 | } 365 | 366 | switch cti.Technique.Type { 367 | case models.MitreAttackType: 368 | if err := r.conn. 369 | Preload(clause.Associations). 370 | Where(&models.MitreAttack{TechniqueID: cti.Technique.ID}). 371 | Take(&cti.Technique.MitreAttack).Error; err != nil { 372 | if errors.Is(err, gorm.ErrRecordNotFound) { 373 | return models.CTI{}, xerrors.Errorf("Failed to get MitreAttack. DB relationship may be broken, use `$ go-cti fetch threat` to recreate DB. err: %w", err) 374 | } 375 | return models.CTI{}, xerrors.Errorf("Failed to get MitreAttack. err: %w", err) 376 | } 377 | case models.CAPECType: 378 | if err := r.conn. 379 | Preload(clause.Associations). 380 | Where(&models.Capec{TechniqueID: cti.Technique.ID}). 381 | Take(&cti.Technique.Capec).Error; err != nil { 382 | if errors.Is(err, gorm.ErrRecordNotFound) { 383 | return models.CTI{}, xerrors.Errorf("Failed to get Capec. DB relationship may be broken, use `$ go-cti fetch threat` to recreate DB. err: %w", err) 384 | } 385 | return models.CTI{}, xerrors.Errorf("Failed to get Capec. err: %w", err) 386 | } 387 | } 388 | } else { 389 | cti.Type = models.AttackerType 390 | 391 | if err := r.conn. 392 | Preload("TechniquesUsed"). 393 | Preload("References"). 394 | Where("attacker_id IN ?", attackerIDs). 395 | Take(&cti.Attacker).Error; err != nil { 396 | return models.CTI{}, xerrors.Errorf("Failed to get Attacker by CTI-ID. err: %w", err) 397 | } 398 | 399 | switch cti.Attacker.Type { 400 | case models.GroupType: 401 | if err := r.conn. 402 | Preload(clause.Associations). 403 | Where(&models.AttackerGroup{AttackerID: cti.Attacker.ID}). 404 | Take(&cti.Attacker.Group).Error; err != nil { 405 | if errors.Is(err, gorm.ErrRecordNotFound) { 406 | return models.CTI{}, xerrors.Errorf("Failed to get Group. DB relationship may be broken, use `$ go-cti fetch threat` to recreate DB. err: %w", err) 407 | } 408 | return models.CTI{}, xerrors.Errorf("Failed to get Group. err: %w", err) 409 | } 410 | case models.SoftwareType: 411 | if err := r.conn. 412 | Preload(clause.Associations). 413 | Where(&models.AttackerSoftware{AttackerID: cti.Attacker.ID}). 414 | Take(&cti.Attacker.Software).Error; err != nil { 415 | if errors.Is(err, gorm.ErrRecordNotFound) { 416 | return models.CTI{}, xerrors.Errorf("Failed to get Software. DB relationship may be broken, use `$ go-cti fetch threat` to recreate DB. err: %w", err) 417 | } 418 | return models.CTI{}, xerrors.Errorf("Failed to get Software. err: %w", err) 419 | } 420 | } 421 | } 422 | 423 | return cti, nil 424 | } 425 | 426 | // GetCtisByMultiCtiID : 427 | func (r *RDBDriver) GetCtisByMultiCtiID(ctiIDs []string) ([]models.CTI, error) { 428 | techniqueIDs, attackerIDs, err := classCtiIDs(ctiIDs) 429 | if err != nil { 430 | return nil, xerrors.Errorf("Failed to classCtiIDs. err: %w", err) 431 | } 432 | 433 | ctis := []models.CTI{} 434 | 435 | techniques := []models.Technique{} 436 | if err := r.conn. 437 | Preload("References"). 438 | Preload("Mitigations"). 439 | Where("technique_id IN ?", techniqueIDs). 440 | Find(&techniques).Error; err != nil { 441 | return nil, xerrors.Errorf("Failed to get Techniques by CTI-IDs. err: %w", err) 442 | } 443 | for i := range techniques { 444 | switch techniques[i].Type { 445 | case models.MitreAttackType: 446 | if err := r.conn. 447 | Preload(clause.Associations). 448 | Where(&models.MitreAttack{TechniqueID: techniques[i].ID}). 449 | Take(&techniques[i].MitreAttack).Error; err != nil { 450 | if errors.Is(err, gorm.ErrRecordNotFound) { 451 | return nil, xerrors.Errorf("Failed to get MitreAttack. DB relationship may be broken, use `$ go-cti fetch threat` to recreate DB. err: %w", err) 452 | } 453 | return nil, xerrors.Errorf("Failed to get MitreAttack. err: %w", err) 454 | } 455 | case models.CAPECType: 456 | if err := r.conn. 457 | Preload(clause.Associations). 458 | Where(&models.Capec{TechniqueID: techniques[i].ID}). 459 | Take(&techniques[i].Capec).Error; err != nil { 460 | if errors.Is(err, gorm.ErrRecordNotFound) { 461 | return nil, xerrors.Errorf("Failed to get Capec. DB relationship may be broken, use `$ go-cti fetch threat` to recreate DB. err: %w", err) 462 | } 463 | return nil, xerrors.Errorf("Failed to get Capec. err: %w", err) 464 | } 465 | } 466 | ctis = append(ctis, models.CTI{ 467 | Type: models.TechniqueType, 468 | Technique: &techniques[i], 469 | }) 470 | } 471 | 472 | attackers := []models.Attacker{} 473 | if err := r.conn. 474 | Preload("TechniquesUsed"). 475 | Preload("References"). 476 | Where("attacker_id IN ?", attackerIDs). 477 | Find(&attackers).Error; err != nil { 478 | return nil, xerrors.Errorf("Failed to get Attackers by CTI-IDs. err: %w", err) 479 | } 480 | 481 | for i := range attackers { 482 | switch attackers[i].Type { 483 | case models.GroupType: 484 | if err := r.conn. 485 | Preload(clause.Associations). 486 | Where(&models.AttackerGroup{AttackerID: attackers[i].ID}). 487 | Take(&attackers[i].Group).Error; err != nil { 488 | if errors.Is(err, gorm.ErrRecordNotFound) { 489 | return nil, xerrors.Errorf("Failed to get Group. DB relationship may be broken, use `$ go-cti fetch threat` to recreate DB. err: %w", err) 490 | } 491 | return nil, xerrors.Errorf("Failed to get Group. err: %w", err) 492 | } 493 | case models.SoftwareType: 494 | if err := r.conn. 495 | Preload(clause.Associations). 496 | Where(&models.AttackerSoftware{AttackerID: attackers[i].ID}). 497 | Take(&attackers[i].Software).Error; err != nil { 498 | if errors.Is(err, gorm.ErrRecordNotFound) { 499 | return nil, xerrors.Errorf("Failed to get Software. DB relationship may be broken, use `$ go-cti fetch threat` to recreate DB. err: %w", err) 500 | } 501 | return nil, xerrors.Errorf("Failed to get Software. err: %w", err) 502 | } 503 | } 504 | ctis = append(ctis, models.CTI{ 505 | Type: models.AttackerType, 506 | Attacker: &attackers[i], 507 | }) 508 | } 509 | return ctis, nil 510 | } 511 | 512 | // GetTechniqueIDsByCveID : 513 | func (r *RDBDriver) GetTechniqueIDsByCveID(cveID string) ([]string, error) { 514 | var mappingID int64 515 | if err := r.conn. 516 | Model(&models.CveToTechniques{}). 517 | Select("id"). 518 | Where(&models.CveToTechniques{CveID: cveID}). 519 | Take(&mappingID).Error; err != nil { 520 | if errors.Is(err, gorm.ErrRecordNotFound) { 521 | return []string{}, nil 522 | } 523 | return nil, xerrors.Errorf("Failed to get ID by CVE-ID. err: %w", err) 524 | } 525 | 526 | techniqueIDs := []string{} 527 | if err := r.conn. 528 | Model(&models.CveToTechniqueID{}). 529 | Select("technique_id"). 530 | Where(&models.CveToTechniqueID{CveToTechniquesID: mappingID}). 531 | Find(&techniqueIDs).Error; err != nil { 532 | return nil, xerrors.Errorf("Failed to get TechniqueIDs by ID. err: %w", err) 533 | } 534 | 535 | return techniqueIDs, nil 536 | } 537 | 538 | // GetTechniqueIDsByMultiCveID : 539 | func (r *RDBDriver) GetTechniqueIDsByMultiCveID(cveIDs []string) (map[string][]string, error) { 540 | var mappings []models.CveToTechniques 541 | if err := r.conn. 542 | Preload("TechniqueIDs"). 543 | Where("cve_id IN ?", cveIDs). 544 | Find(&mappings).Error; err != nil { 545 | return nil, xerrors.Errorf("Failed to get TechniqueIDs by CVE-IDs. err: %w", err) 546 | } 547 | 548 | techniqueIDs := map[string][]string{} 549 | for _, mapping := range mappings { 550 | for _, techniqueID := range mapping.TechniqueIDs { 551 | techniqueIDs[mapping.CveID] = append(techniqueIDs[mapping.CveID], techniqueID.TechniqueID) 552 | } 553 | } 554 | 555 | return techniqueIDs, nil 556 | } 557 | 558 | // GetAttackerIDsByTechniqueIDs : 559 | func (r *RDBDriver) GetAttackerIDsByTechniqueIDs(techniqueIDs []string) ([]string, error) { 560 | attackers := []models.Attacker{} 561 | if err := r.conn. 562 | Preload("TechniquesUsed"). 563 | Find(&attackers).Error; err != nil { 564 | return nil, xerrors.Errorf("Failed to get Attackers by TechniqueIDs. err: %w", err) 565 | } 566 | 567 | attackerIDs := []string{} 568 | for _, attacker := range attackers { 569 | if len(attacker.TechniquesUsed) == 0 { 570 | continue 571 | } 572 | 573 | attackerUsedTechniques := map[string]struct{}{} 574 | for _, attackerUsedTechnique := range attacker.TechniquesUsed { 575 | attackerUsedTechniques[attackerUsedTechnique.TechniqueID] = struct{}{} 576 | } 577 | 578 | for _, techniqueID := range techniqueIDs { 579 | delete(attackerUsedTechniques, techniqueID) 580 | if len(attackerUsedTechniques) == 0 { 581 | break 582 | } 583 | } 584 | if len(attackerUsedTechniques) == 0 { 585 | attackerIDs = append(attackerIDs, attacker.AttackerID) 586 | } 587 | } 588 | 589 | return attackerIDs, nil 590 | } 591 | -------------------------------------------------------------------------------- /db/redis.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "maps" 10 | "os" 11 | "slices" 12 | "strconv" 13 | "strings" 14 | "time" 15 | 16 | "github.com/cheggaaa/pb/v3" 17 | "github.com/go-redis/redis/v8" 18 | "github.com/inconshreveable/log15" 19 | "github.com/spf13/viper" 20 | "golang.org/x/xerrors" 21 | 22 | "github.com/vulsio/go-cti/config" 23 | "github.com/vulsio/go-cti/models" 24 | ) 25 | 26 | /** 27 | # Redis Data Structure 28 | - Strings 29 | ┌───┬────────────────┬────────┬──────────────────────────────────────────────────┐ 30 | │NO │ KEY │ MEMBER │ PURPOSE │ 31 | └───┴────────────────┴────────┴──────────────────────────────────────────────────┘ 32 | ┌───┬────────────────┬────────┬──────────────────────────────────────────────────┐ 33 | │ 1 │ CTI#CTI#$CTIID │ JSON │ TO GET CTI FROM CTIID │ 34 | ├───┼────────────────┼────────┼──────────────────────────────────────────────────┤ 35 | │ 2 │ CTI#DEP │ JSON │ TO DELETE OUTDATED AND UNNEEDED FIELD AND MEMBER │ 36 | └───┴────────────────┴────────┴──────────────────────────────────────────────────┘ 37 | 38 | - Sets 39 | ┌───┬─────────────────────┬──────────────┬─────────────────────────────────────┐ 40 | │NO │ KEY │ MEMBER │ PURPOSE │ 41 | └───┴─────────────────────┴──────────────┴─────────────────────────────────────┘ 42 | ┌───┬─────────────────────┬──────────────┬─────────────────────────────────────┐ 43 | │ 1 │ CTI#CVE#$CVEID │ $TECHNIQUEID │ TO GET TECHNIQUEIDs FROM CVEID │ 44 | ├───┼─────────────────────┼──────────────┼─────────────────────────────────────┤ 45 | │ 2 │ CTI#ATK#$ATTACKERID │ $TECHNIQUEID │ TO GET TECHNIQUEIDs FROM ATTACKERID │ 46 | └───┴─────────────────────┴──────────────┴─────────────────────────────────────┘ 47 | 48 | - Hash 49 | ┌───┬────────────────┬───────────────┬──────────────┬──────────────────────────────┐ 50 | │NO │ KEY │ FIELD │ VALUE │ PURPOSE │ 51 | └───┴────────────────┴───────────────┴──────────────┴──────────────────────────────┘ 52 | ┌───┬────────────────┬───────────────┬──────────────┬──────────────────────────────┐ 53 | │ 1 │ CTI#FETCHMETA │ Revision │ string │ GET Go-CTI Binary Revision │ 54 | ├───┼────────────────┼───────────────┼──────────────┼──────────────────────────────┤ 55 | │ 2 │ CTI#FETCHMETA │ SchemaVersion │ uint │ GET Go-CTI Schema Version │ 56 | ├───┼────────────────┼───────────────┼──────────────┼──────────────────────────────┤ 57 | │ 3 │ CTI#FETCHMETA │ LastFetchedAt │ time.Time │ GET Go-CTI Last Fetched Time │ 58 | └───┴────────────────┴───────────────┴──────────────┴──────────────────────────────┘ 59 | **/ 60 | 61 | const ( 62 | dialectRedis = "redis" 63 | ctiIDKeyFormat = "CTI#CTI#%s" 64 | cveIDKeyFormat = "CTI#CVE#%s" 65 | atkIDKeyFormat = "CTI#ATK#%s" 66 | depKey = "CTI#DEP" 67 | fetchMetaKey = "CTI#FETCHMETA" 68 | ) 69 | 70 | // RedisDriver is Driver for Redis 71 | type RedisDriver struct { 72 | name string 73 | conn *redis.Client 74 | } 75 | 76 | // Name return db name 77 | func (r *RedisDriver) Name() string { 78 | return r.name 79 | } 80 | 81 | // OpenDB opens Database 82 | func (r *RedisDriver) OpenDB(_, dbPath string, _ bool, option Option) error { 83 | if err := r.connectRedis(dbPath, option); err != nil { 84 | return xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dialectRedis, dbPath, err) 85 | } 86 | return nil 87 | } 88 | 89 | func (r *RedisDriver) connectRedis(dbPath string, option Option) error { 90 | ctx := context.Background() 91 | var err error 92 | var opt *redis.Options 93 | if opt, err = redis.ParseURL(dbPath); err != nil { 94 | return xerrors.Errorf("Failed to parse url. err: %w", err) 95 | } 96 | if 0 < option.RedisTimeout.Seconds() { 97 | opt.ReadTimeout = option.RedisTimeout 98 | } 99 | r.conn = redis.NewClient(opt) 100 | return r.conn.Ping(ctx).Err() 101 | } 102 | 103 | // CloseDB close Database 104 | func (r *RedisDriver) CloseDB() (err error) { 105 | if r.conn == nil { 106 | return 107 | } 108 | if err = r.conn.Close(); err != nil { 109 | return xerrors.Errorf("Failed to close DB. Type: %s. err: %w", r.name, err) 110 | } 111 | return 112 | } 113 | 114 | // MigrateDB migrates Database 115 | func (r *RedisDriver) MigrateDB() error { 116 | return nil 117 | } 118 | 119 | // IsGoCTIModelV1 determines if the DB was created at the time of go-cti Model v1 120 | func (r *RedisDriver) IsGoCTIModelV1() (bool, error) { 121 | ctx := context.Background() 122 | 123 | exists, err := r.conn.Exists(ctx, fetchMetaKey).Result() 124 | if err != nil { 125 | return false, xerrors.Errorf("Failed to Exists. err: %w", err) 126 | } 127 | if exists == 0 { 128 | keys, _, err := r.conn.Scan(ctx, 0, "CTI#*", 1).Result() 129 | if err != nil { 130 | return false, xerrors.Errorf("Failed to Scan. err: %w", err) 131 | } 132 | if len(keys) == 0 { 133 | return false, nil 134 | } 135 | return true, nil 136 | } 137 | 138 | return false, nil 139 | } 140 | 141 | // GetFetchMeta get FetchMeta from Database 142 | func (r *RedisDriver) GetFetchMeta() (*models.FetchMeta, error) { 143 | ctx := context.Background() 144 | 145 | exists, err := r.conn.Exists(ctx, fetchMetaKey).Result() 146 | if err != nil { 147 | return nil, xerrors.Errorf("Failed to Exists. err: %w", err) 148 | } 149 | if exists == 0 { 150 | return &models.FetchMeta{GoCTIRevision: config.Revision, SchemaVersion: models.LatestSchemaVersion, LastFetchedAt: time.Date(1000, time.January, 1, 0, 0, 0, 0, time.UTC)}, nil 151 | } 152 | 153 | revision, err := r.conn.HGet(ctx, fetchMetaKey, "Revision").Result() 154 | if err != nil { 155 | return nil, xerrors.Errorf("Failed to HGet Revision. err: %w", err) 156 | } 157 | 158 | verstr, err := r.conn.HGet(ctx, fetchMetaKey, "SchemaVersion").Result() 159 | if err != nil { 160 | return nil, xerrors.Errorf("Failed to HGet SchemaVersion. err: %w", err) 161 | } 162 | version, err := strconv.ParseUint(verstr, 10, 8) 163 | if err != nil { 164 | return nil, xerrors.Errorf("Failed to ParseUint. err: %w", err) 165 | } 166 | 167 | datestr, err := r.conn.HGet(ctx, fetchMetaKey, "LastFetchedAt").Result() 168 | if err != nil { 169 | if !errors.Is(err, redis.Nil) { 170 | return nil, xerrors.Errorf("Failed to HGet LastFetchedAt. err: %w", err) 171 | } 172 | datestr = time.Date(1000, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339) 173 | } 174 | date, err := time.Parse(time.RFC3339, datestr) 175 | if err != nil { 176 | return nil, xerrors.Errorf("Failed to Parse date. err: %w", err) 177 | } 178 | 179 | return &models.FetchMeta{GoCTIRevision: revision, SchemaVersion: uint(version), LastFetchedAt: date}, nil 180 | } 181 | 182 | // UpsertFetchMeta upsert FetchMeta to Database 183 | func (r *RedisDriver) UpsertFetchMeta(fetchMeta *models.FetchMeta) error { 184 | return r.conn.HSet(context.Background(), fetchMetaKey, map[string]interface{}{"Revision": config.Revision, "SchemaVersion": models.LatestSchemaVersion, "LastFetchedAt": fetchMeta.LastFetchedAt}).Err() 185 | } 186 | 187 | // InsertCti : 188 | func (r *RedisDriver) InsertCti(techniques []models.Technique, mappings []models.CveToTechniques, attackers []models.Attacker) error { 189 | ctx := context.Background() 190 | batchSize := viper.GetInt("batch-size") 191 | if batchSize < 1 { 192 | return xerrors.New("Failed to set batch-size. err: batch-size option is not set properly") 193 | } 194 | 195 | // newDeps, oldDeps: {"cti": {"CTI-ID": {}}, "mapping": {"CVE-ID or AttackID": {"TechniqueID": {}}}} 196 | newDeps := map[string]map[string]map[string]struct{}{"cti": {}, "mapping": {}} 197 | oldDepsStr, err := r.conn.Get(ctx, depKey).Result() 198 | if err != nil { 199 | if !errors.Is(err, redis.Nil) { 200 | return xerrors.Errorf("Failed to Get key: %s. err: %w", depKey, err) 201 | } 202 | oldDepsStr = `{"cti": {}, "mapping": {}}` 203 | } 204 | var oldDeps map[string]map[string]map[string]struct{} 205 | if err := json.Unmarshal([]byte(oldDepsStr), &oldDeps); err != nil { 206 | return xerrors.Errorf("Failed to unmarshal JSON. err: %w", err) 207 | } 208 | 209 | log15.Info("Inserting Cyber Threat Intelligences...") 210 | 211 | log15.Info("Inserting Techniques...") 212 | bar := pb.StartNew(len(techniques)).SetWriter(func() io.Writer { 213 | if viper.GetBool("log-json") { 214 | return io.Discard 215 | } 216 | return os.Stderr 217 | }()) 218 | for chunk := range slices.Chunk(techniques, batchSize) { 219 | pipe := r.conn.Pipeline() 220 | for _, technique := range chunk { 221 | j, err := json.Marshal(technique) 222 | if err != nil { 223 | return xerrors.Errorf("Failed to marshal json. err: %w", err) 224 | } 225 | 226 | _ = pipe.Set(ctx, fmt.Sprintf(ctiIDKeyFormat, technique.TechniqueID), j, 0) 227 | newDeps["cti"][technique.TechniqueID] = map[string]struct{}{} 228 | delete(oldDeps["cti"], technique.TechniqueID) 229 | } 230 | if _, err := pipe.Exec(ctx); err != nil { 231 | return xerrors.Errorf("Failed to exec pipeline. err: %w", err) 232 | } 233 | bar.Add(len(chunk)) 234 | } 235 | bar.Finish() 236 | 237 | log15.Info("Inserting CVE-ID to CTI-ID CveToTechniques...") 238 | bar = pb.StartNew(len(mappings)).SetWriter(func() io.Writer { 239 | if viper.GetBool("log-json") { 240 | return io.Discard 241 | } 242 | return os.Stderr 243 | }()) 244 | for chunk := range slices.Chunk(mappings, batchSize) { 245 | pipe := r.conn.Pipeline() 246 | for _, mapping := range chunk { 247 | cveKey := fmt.Sprintf(cveIDKeyFormat, mapping.CveID) 248 | if _, ok := newDeps["mapping"][mapping.CveID]; !ok { 249 | newDeps["mapping"][mapping.CveID] = map[string]struct{}{} 250 | } 251 | 252 | for _, ctiID := range mapping.TechniqueIDs { 253 | _ = pipe.SAdd(ctx, cveKey, ctiID.TechniqueID) 254 | 255 | newDeps["mapping"][mapping.CveID][ctiID.TechniqueID] = struct{}{} 256 | if _, ok := oldDeps["mapping"][mapping.CveID]; ok { 257 | delete(oldDeps["mapping"][mapping.CveID], ctiID.TechniqueID) 258 | if len(oldDeps["mapping"][mapping.CveID]) == 0 { 259 | delete(oldDeps["mapping"], mapping.CveID) 260 | } 261 | } 262 | } 263 | } 264 | if _, err := pipe.Exec(ctx); err != nil { 265 | return xerrors.Errorf("Failed to exec pipeline. err: %w", err) 266 | } 267 | bar.Add(len(chunk)) 268 | } 269 | bar.Finish() 270 | 271 | log15.Info("Inserting Attackers...") 272 | bar = pb.StartNew(len(attackers)).SetWriter(func() io.Writer { 273 | if viper.GetBool("log-json") { 274 | return io.Discard 275 | } 276 | return os.Stderr 277 | }()) 278 | for chunk := range slices.Chunk(attackers, batchSize) { 279 | pipe := r.conn.Pipeline() 280 | for _, attacker := range chunk { 281 | j, err := json.Marshal(attacker) 282 | if err != nil { 283 | return xerrors.Errorf("Failed to marshal json. err: %w", err) 284 | } 285 | 286 | _ = pipe.Set(ctx, fmt.Sprintf(ctiIDKeyFormat, attacker.AttackerID), j, 0) 287 | newDeps["cti"][attacker.AttackerID] = map[string]struct{}{} 288 | delete(oldDeps["cti"], attacker.AttackerID) 289 | 290 | atkKey := fmt.Sprintf(atkIDKeyFormat, attacker.AttackerID) 291 | if _, ok := newDeps["mapping"][attacker.AttackerID]; !ok { 292 | newDeps["mapping"][attacker.AttackerID] = map[string]struct{}{} 293 | if len(oldDeps["mapping"][attacker.AttackerID]) == 0 { 294 | delete(oldDeps["mapping"], attacker.AttackerID) 295 | } 296 | } 297 | 298 | for _, technique := range attacker.TechniquesUsed { 299 | _ = pipe.SAdd(ctx, atkKey, technique.TechniqueID) 300 | 301 | newDeps["mapping"][attacker.AttackerID][technique.TechniqueID] = struct{}{} 302 | if _, ok := oldDeps["mapping"][attacker.AttackerID]; ok { 303 | delete(oldDeps["mapping"][attacker.AttackerID], technique.TechniqueID) 304 | if len(oldDeps["mapping"][attacker.AttackerID]) == 0 { 305 | delete(oldDeps["mapping"], attacker.AttackerID) 306 | } 307 | } 308 | } 309 | } 310 | if _, err := pipe.Exec(ctx); err != nil { 311 | return xerrors.Errorf("Failed to exec pipeline. err: %w", err) 312 | } 313 | bar.Add(len(chunk)) 314 | } 315 | 316 | pipe := r.conn.Pipeline() 317 | for ctiID := range oldDeps["cti"] { 318 | _ = pipe.Del(ctx, fmt.Sprintf(ctiIDKeyFormat, ctiID)) 319 | } 320 | for id, techniqueIDs := range oldDeps["mapping"] { 321 | if strings.HasPrefix(id, "CVE") { 322 | _ = pipe.SRem(ctx, fmt.Sprintf(cveIDKeyFormat, id), slices.Collect(maps.Keys(techniqueIDs))) 323 | } else { 324 | _ = pipe.SRem(ctx, fmt.Sprintf(atkIDKeyFormat, id), slices.Collect(maps.Keys(techniqueIDs))) 325 | } 326 | } 327 | newDepsJSON, err := json.Marshal(newDeps) 328 | if err != nil { 329 | return xerrors.Errorf("Failed to Marshal JSON. err: %w", err) 330 | } 331 | _ = pipe.Set(ctx, depKey, string(newDepsJSON), 0) 332 | if _, err := pipe.Exec(ctx); err != nil { 333 | return xerrors.Errorf("Failed to exec pipeline. err: %w", err) 334 | } 335 | 336 | return nil 337 | } 338 | 339 | // GetCtiByCtiID : 340 | func (r *RedisDriver) GetCtiByCtiID(ctiID string) (models.CTI, error) { 341 | ctx := context.Background() 342 | 343 | techniqueIDs, attackerIDs, err := classCtiIDs([]string{ctiID}) 344 | if err != nil { 345 | return models.CTI{}, xerrors.Errorf("Failed to classCtiIDs. err: %w", err) 346 | } 347 | 348 | cti := models.CTI{} 349 | if len(techniqueIDs) > 0 { 350 | cti.Type = models.TechniqueType 351 | 352 | str, err := r.conn.Get(ctx, fmt.Sprintf(ctiIDKeyFormat, techniqueIDs[0])).Result() 353 | if err != nil { 354 | return models.CTI{}, xerrors.Errorf("Failed to Get. key: %s, err: %s", fmt.Sprintf(ctiIDKeyFormat, techniqueIDs[0]), err) 355 | } 356 | if err := json.Unmarshal([]byte(str), &cti.Technique); err != nil { 357 | return models.CTI{}, xerrors.Errorf("Failed to Unmarshal JSON. err: %w", err) 358 | } 359 | } else { 360 | cti.Type = models.AttackerType 361 | 362 | str, err := r.conn.Get(ctx, fmt.Sprintf(ctiIDKeyFormat, attackerIDs[0])).Result() 363 | if err != nil { 364 | return models.CTI{}, xerrors.Errorf("Failed to Get. key: %s, err: %s", fmt.Sprintf(ctiIDKeyFormat, attackerIDs[0]), err) 365 | } 366 | if err := json.Unmarshal([]byte(str), &cti.Attacker); err != nil { 367 | return models.CTI{}, xerrors.Errorf("Failed to Unmarshal JSON. err: %w", err) 368 | } 369 | } 370 | 371 | return cti, nil 372 | } 373 | 374 | // GetCtisByMultiCtiID : 375 | func (r *RedisDriver) GetCtisByMultiCtiID(ctiIDs []string) ([]models.CTI, error) { 376 | ctx := context.Background() 377 | 378 | techniqueIDs, attackerIDs, err := classCtiIDs(ctiIDs) 379 | if err != nil { 380 | return nil, xerrors.Errorf("Failed to classCtiIDs. err: %w", err) 381 | } 382 | 383 | ctis := []models.CTI{} 384 | 385 | pipe := r.conn.Pipeline() 386 | for _, techniqueID := range techniqueIDs { 387 | _ = pipe.Get(ctx, fmt.Sprintf(ctiIDKeyFormat, techniqueID)) 388 | } 389 | cmders, err := pipe.Exec(ctx) 390 | if err != nil && !errors.Is(err, redis.Nil) { 391 | return nil, xerrors.Errorf("Failed to exec pipeline. techniqueIDs: %q, err: %w", techniqueIDs, err) 392 | } 393 | 394 | for _, cmder := range cmders { 395 | res, err := cmder.(*redis.StringCmd).Result() 396 | if err != nil { 397 | return nil, xerrors.Errorf("Failed to Get. err: %w", err) 398 | } 399 | 400 | var technique models.Technique 401 | if err := json.Unmarshal([]byte(res), &technique); err != nil { 402 | return nil, xerrors.Errorf("Failed to unmarshal json. err: %w", err) 403 | } 404 | ctis = append(ctis, models.CTI{ 405 | Type: models.TechniqueType, 406 | Technique: &technique, 407 | }) 408 | } 409 | 410 | pipe = r.conn.Pipeline() 411 | for _, attackerID := range attackerIDs { 412 | _ = pipe.Get(ctx, fmt.Sprintf(ctiIDKeyFormat, attackerID)) 413 | } 414 | cmders, err = pipe.Exec(ctx) 415 | if err != nil && !errors.Is(err, redis.Nil) { 416 | return nil, xerrors.Errorf("Failed to exec pipeline. attackerIDs: %q, err: %w", attackerIDs, err) 417 | } 418 | 419 | for _, cmder := range cmders { 420 | res, err := cmder.(*redis.StringCmd).Result() 421 | if err != nil { 422 | return nil, xerrors.Errorf("Failed to Get. err: %w", err) 423 | } 424 | 425 | var attacker models.Attacker 426 | if err := json.Unmarshal([]byte(res), &attacker); err != nil { 427 | return nil, xerrors.Errorf("Failed to unmarshal json. err: %w", err) 428 | } 429 | ctis = append(ctis, models.CTI{ 430 | Type: models.AttackerType, 431 | Attacker: &attacker, 432 | }) 433 | } 434 | 435 | return ctis, nil 436 | } 437 | 438 | // GetTechniqueIDsByCveID : 439 | func (r *RedisDriver) GetTechniqueIDsByCveID(cveID string) ([]string, error) { 440 | techniqueIDs, err := r.conn.SMembers(context.Background(), fmt.Sprintf(cveIDKeyFormat, cveID)).Result() 441 | if err != nil { 442 | return nil, xerrors.Errorf("Failed to SMembers. key: %s, err: %w", fmt.Sprintf(cveIDKeyFormat, cveID), err) 443 | } 444 | return techniqueIDs, nil 445 | } 446 | 447 | // GetTechniqueIDsByMultiCveID : 448 | func (r *RedisDriver) GetTechniqueIDsByMultiCveID(cveIDs []string) (map[string][]string, error) { 449 | ctx := context.Background() 450 | 451 | m := map[string]*redis.StringSliceCmd{} 452 | pipe := r.conn.Pipeline() 453 | for _, cveID := range cveIDs { 454 | m[cveID] = pipe.SMembers(ctx, fmt.Sprintf(cveIDKeyFormat, cveID)) 455 | } 456 | if _, err := pipe.Exec(ctx); err != nil { 457 | return nil, xerrors.Errorf("Failed to exec pipeline. err: %w", err) 458 | } 459 | 460 | techniqueIDs := map[string][]string{} 461 | for cveID, cmd := range m { 462 | ids, err := cmd.Result() 463 | if err != nil { 464 | return nil, xerrors.Errorf("Failed to SMembers. key: %s, err: %w", fmt.Sprintf(cveIDKeyFormat, cveID), err) 465 | } 466 | techniqueIDs[cveID] = ids 467 | } 468 | return techniqueIDs, nil 469 | } 470 | 471 | // GetAttackerIDsByTechniqueIDs : 472 | func (r *RedisDriver) GetAttackerIDsByTechniqueIDs(techniqueIDs []string) ([]string, error) { 473 | ctx := context.Background() 474 | 475 | atkKeys := []string{} 476 | 477 | dbsize, err := r.conn.DBSize(ctx).Result() 478 | if err != nil { 479 | return nil, xerrors.Errorf("Failed to DBSize. err: %w", err) 480 | } 481 | 482 | var cursor uint64 483 | for { 484 | var keys []string 485 | var err error 486 | keys, cursor, err = r.conn.Scan(ctx, cursor, fmt.Sprintf(atkIDKeyFormat, "*"), dbsize/5).Result() 487 | if err != nil { 488 | return nil, xerrors.Errorf("Failed to Scan. err: %w", err) 489 | } 490 | 491 | atkKeys = append(atkKeys, keys...) 492 | 493 | if cursor == 0 { 494 | break 495 | } 496 | } 497 | 498 | m := map[string]*redis.StringSliceCmd{} 499 | pipe := r.conn.Pipeline() 500 | for _, atkKey := range atkKeys { 501 | m[strings.TrimPrefix(atkKey, "CTI#ATK#")] = pipe.SMembers(ctx, atkKey) 502 | } 503 | if _, err := pipe.Exec(ctx); err != nil { 504 | return nil, xerrors.Errorf("Failed to exec pipeline. err: %w", err) 505 | } 506 | 507 | attackerIDs := []string{} 508 | for atkID, cmd := range m { 509 | ids, err := cmd.Result() 510 | if err != nil { 511 | return nil, xerrors.Errorf("Failed to SMembers. key: %s, err: %w", fmt.Sprintf(atkIDKeyFormat, atkID), err) 512 | } 513 | 514 | attackerUsedTechniques := map[string]struct{}{} 515 | for _, id := range ids { 516 | attackerUsedTechniques[id] = struct{}{} 517 | } 518 | 519 | for _, techniqueID := range techniqueIDs { 520 | delete(attackerUsedTechniques, techniqueID) 521 | if len(attackerUsedTechniques) == 0 { 522 | break 523 | } 524 | } 525 | if len(attackerUsedTechniques) == 0 { 526 | attackerIDs = append(attackerIDs, atkID) 527 | } 528 | } 529 | 530 | return attackerIDs, nil 531 | } 532 | -------------------------------------------------------------------------------- /fetcher/attack/attack.go: -------------------------------------------------------------------------------- 1 | package attack 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/inconshreveable/log15" 9 | "golang.org/x/xerrors" 10 | 11 | "github.com/vulsio/go-cti/models" 12 | "github.com/vulsio/go-cti/utils" 13 | ) 14 | 15 | const attackURL = "https://raw.githubusercontent.com/mitre/cti/master/enterprise-attack/enterprise-attack.json" 16 | 17 | // Fetch MITRE ATT&CK data 18 | func Fetch() ([]models.Technique, []models.Attacker, error) { 19 | log15.Info("Fetching MITRE ATT&CK...") 20 | 21 | res, err := utils.FetchURL(attackURL) 22 | if err != nil { 23 | return nil, nil, xerrors.Errorf("Failed to fetch MITRE ATT&CK JSON. err: %w", err) 24 | } 25 | techniques, attackers, err := parse(res) 26 | if err != nil { 27 | return nil, nil, xerrors.Errorf("Failed to parse MITRE ATT&CK Cyber Threat Intelligence. err: %w", err) 28 | } 29 | return techniques, attackers, nil 30 | } 31 | 32 | func parse(res []byte) ([]models.Technique, []models.Attacker, error) { 33 | var r root 34 | if err := json.Unmarshal(res, &r); err != nil { 35 | return nil, nil, xerrors.Errorf("Failed to unmarshal json. err: %w", err) 36 | } 37 | 38 | attackPatterns, attackers, others, relationships, err := parseEachObject(r.Objects) 39 | if err != nil { 40 | return nil, nil, xerrors.Errorf("Failed to parseEachObject. err: %w", err) 41 | } 42 | 43 | techniques := []models.Technique{} 44 | for id, attackPattern := range attackPatterns { 45 | if attackPattern.deprecated { 46 | continue 47 | } 48 | technique, err := fillTechnique(attackPattern, relationships[id], attackPatterns, attackers, others) 49 | if err != nil { 50 | return nil, nil, xerrors.Errorf("Failed to fillTechnique. err: %w", err) 51 | } 52 | techniques = append(techniques, technique) 53 | } 54 | 55 | techniquesUsed := map[string][]techniqueUsed{} 56 | for _, rels := range relationships { 57 | for _, rel := range rels { 58 | if !strings.HasPrefix(rel.targetRef, "attack-pattern--") || !strings.HasPrefix(rel.sourceRef, "intrusion-set--") && !strings.HasPrefix(rel.sourceRef, "malware--") && !strings.HasPrefix(rel.sourceRef, "tool--") && !strings.HasPrefix(rel.sourceRef, "campaign--") { 59 | continue 60 | } 61 | technique, ok := attackPatterns[rel.targetRef] 62 | if !ok { 63 | return nil, nil, xerrors.Errorf("Failed to get Technique. rel.id: %s, rel.targetRef: %s", rel.id, rel.targetRef) 64 | } 65 | techniquesUsed[rel.sourceRef] = append(techniquesUsed[rel.sourceRef], techniqueUsed{ 66 | id: technique.id, 67 | name: technique.name, 68 | use: rel.description, 69 | references: rel.references, 70 | }) 71 | } 72 | } 73 | 74 | groupsUsed := map[string][]groupUsed{} 75 | for _, rels := range relationships { 76 | for _, rel := range rels { 77 | if !strings.HasPrefix(rel.targetRef, "malware--") && !strings.HasPrefix(rel.targetRef, "tool--") || !strings.HasPrefix(rel.sourceRef, "intrusion-set--") { 78 | continue 79 | } 80 | software, ok := attackers[rel.targetRef] 81 | if !ok { 82 | return nil, nil, xerrors.Errorf("Failed to get Attacker Software. rel.id: %s, rel.targetRef: %s", rel.id, rel.targetRef) 83 | } 84 | groupsUsed[rel.sourceRef] = append(groupsUsed[rel.sourceRef], groupUsed{ 85 | name: software.name, 86 | description: rel.description, 87 | references: rel.references, 88 | }) 89 | } 90 | } 91 | 92 | attackerInfos := []models.Attacker{} 93 | for id, attacker := range attackers { 94 | if attacker.deprecated { 95 | continue 96 | } 97 | attackerInfo, err := fillAttacker(attacker, relationships[id], attackers, techniquesUsed[id], groupsUsed[id]) 98 | if err != nil { 99 | return nil, nil, xerrors.Errorf("Failed to fillAttacker. err: %w", err) 100 | } 101 | attackerInfos = append(attackerInfos, attackerInfo) 102 | } 103 | 104 | return techniques, attackerInfos, nil 105 | } 106 | 107 | func parseEachObject(root []ctiObject) (map[string]attackPattern, map[string]attacker, map[string]otherInfo, map[string][]relationship, error) { 108 | attackPatterns := map[string]attackPattern{} 109 | attackers := map[string]attacker{} 110 | others := map[string]otherInfo{} 111 | dataSources := map[string]string{} 112 | dataComponents := map[string]dataComponent{} 113 | relationships := map[string][]relationship{} 114 | for _, obj := range root { 115 | switch obj.Type { 116 | case "attack-pattern": 117 | attackPatterns[obj.ID] = parseAttackPattern(obj) 118 | case "course-of-action": 119 | others[obj.ID] = otherInfo{ 120 | objType: obj.Type, 121 | name: getObjectName(obj.Name, obj.ExternalReferences), 122 | description: obj.Description, 123 | deprecated: obj.Revoked || obj.XMitreDeprecated, 124 | } 125 | case "intrusion-set", "malware", "tool", "campaign": 126 | attackers[obj.ID] = parseAttacker(obj) 127 | case "x-mitre-tactic": 128 | others[obj.XMitreShortname] = otherInfo{ 129 | objType: obj.Type, 130 | name: getObjectName(obj.Name, obj.ExternalReferences), 131 | description: obj.Description, 132 | } 133 | case "x-mitre-data-source": 134 | dataSources[obj.ID] = getObjectName(obj.Name, obj.ExternalReferences) 135 | case "x-mitre-data-component": 136 | dataComponents[obj.ID] = dataComponent{ 137 | name: obj.Name, 138 | description: obj.Description, 139 | dataSourceRef: obj.XMitreDataSourceRef, 140 | } 141 | case "relationship": 142 | relationships[obj.TargetRef] = append(relationships[obj.TargetRef], relationship{ 143 | id: obj.ID, 144 | description: obj.Description, 145 | relationshipType: obj.RelationshipType, 146 | sourceRef: obj.SourceRef, 147 | targetRef: obj.TargetRef, 148 | references: obj.ExternalReferences, 149 | }) 150 | } 151 | } 152 | 153 | for id, component := range dataComponents { 154 | ds, ok := dataSources[component.dataSourceRef] 155 | if !ok { 156 | return nil, nil, nil, nil, xerrors.Errorf("Failed to get data source name. id: %s, err: broken relationships", id) 157 | } 158 | others[id] = otherInfo{ 159 | objType: "x-mitre-data-component", 160 | name: fmt.Sprintf("%s: %s", ds, component.name), 161 | description: component.description, 162 | } 163 | } 164 | 165 | return attackPatterns, attackers, others, relationships, nil 166 | } 167 | 168 | func parseAttackPattern(obj ctiObject) attackPattern { 169 | r := attackPattern{ 170 | name: obj.Name, 171 | description: obj.Description, 172 | permissionRequired: obj.XMitrePermissionsRequired, 173 | effectivePermissions: obj.XMitreEffectivePermissions, 174 | platforms: obj.XMitrePlatforms, 175 | impactType: obj.XMitreImpactType, 176 | networkRequirements: obj.XMitreNetworkRequirements, 177 | remoteSupport: obj.XMitreRemoteSupport, 178 | defenseByPassed: obj.XMitreDefenseBypassed, 179 | detection: obj.XMitreDetection, 180 | created: obj.Created, 181 | modified: obj.Modified, 182 | deprecated: obj.Revoked || obj.XMitreDeprecated, 183 | } 184 | 185 | for _, ref := range obj.ExternalReferences { 186 | switch ref.SourceName { 187 | case "mitre-attack": 188 | r.id = ref.ExternalID 189 | r.name = fmt.Sprintf("%s: %s", ref.ExternalID, obj.Name) 190 | case "capec": 191 | r.capecIDs = append(r.capecIDs, ref.ExternalID) 192 | default: 193 | r.references = append(r.references, ref) 194 | } 195 | } 196 | 197 | for _, phase := range obj.KillChainPhases { 198 | r.killChainPhases = append(r.killChainPhases, phase.PhaseName) 199 | } 200 | 201 | return r 202 | } 203 | 204 | func parseAttacker(obj ctiObject) attacker { 205 | r := attacker{ 206 | objType: obj.Type, 207 | name: obj.Name, 208 | description: obj.Description, 209 | platforms: obj.XMitrePlatforms, 210 | created: obj.Created, 211 | modified: obj.Modified, 212 | deprecated: obj.Revoked || obj.XMitreDeprecated, 213 | } 214 | 215 | for _, ref := range obj.ExternalReferences { 216 | switch ref.SourceName { 217 | case "mitre-attack": 218 | r.id = ref.ExternalID 219 | r.name = fmt.Sprintf("%s: %s", ref.ExternalID, obj.Name) 220 | default: 221 | r.references = append(r.references, ref) 222 | } 223 | } 224 | 225 | switch obj.Type { 226 | case "intrusion-set", "campaign": 227 | r.aliases = append(r.aliases, obj.Aliases...) 228 | case "malware", "tool": 229 | r.aliases = append(r.aliases, obj.XMitreAliases...) 230 | } 231 | 232 | return r 233 | } 234 | 235 | func getObjectName(objName string, refs []reference) string { 236 | for _, ref := range refs { 237 | if ref.SourceName == "mitre-attack" { 238 | return fmt.Sprintf("%s: %s", ref.ExternalID, objName) 239 | } 240 | } 241 | return objName 242 | } 243 | 244 | func fillTechnique(attackPattern attackPattern, relationships []relationship, attackPatterns map[string]attackPattern, attackers map[string]attacker, others map[string]otherInfo) (models.Technique, error) { 245 | technique := models.Technique{ 246 | TechniqueID: attackPattern.id, 247 | Type: models.MitreAttackType, 248 | Name: attackPattern.name, 249 | Description: attackPattern.description, 250 | References: []models.TechniqueReference{}, 251 | Mitigations: []models.Mitigation{}, 252 | MitreAttack: &models.MitreAttack{ 253 | CapecIDs: []models.CapecID{}, 254 | Detection: attackPattern.detection, 255 | KillChainPhases: []models.KillChainPhase{}, 256 | DataSources: []models.DataSource{}, 257 | Procedures: []models.Procedure{}, 258 | Platforms: []models.TechniquePlatform{}, 259 | PermissionsRequired: []models.PermissionRequired{}, 260 | EffectivePermissions: []models.EffectivePermission{}, 261 | DefenseBypassed: []models.DefenseBypassed{}, 262 | ImpactType: []models.ImpactType{}, 263 | NetworkRequirements: attackPattern.networkRequirements, 264 | RemoteSupport: attackPattern.remoteSupport, 265 | SubTechniques: []models.SubTechnique{}, 266 | }, 267 | Created: attackPattern.created, 268 | Modified: attackPattern.modified, 269 | } 270 | 271 | for _, capecID := range attackPattern.capecIDs { 272 | technique.MitreAttack.CapecIDs = append(technique.MitreAttack.CapecIDs, models.CapecID{ 273 | CapecID: capecID, 274 | }) 275 | } 276 | 277 | references := map[string]models.TechniqueReference{} 278 | for _, ref := range attackPattern.references { 279 | references[ref.SourceName] = models.TechniqueReference{ 280 | Reference: models.Reference{ 281 | SourceName: ref.SourceName, 282 | Description: ref.Description, 283 | URL: ref.URL, 284 | }, 285 | } 286 | } 287 | 288 | for _, rel := range relationships { 289 | if rel.relationshipType == "revoked-by" { 290 | continue 291 | } 292 | 293 | for _, ref := range rel.references { 294 | references[ref.SourceName] = models.TechniqueReference{ 295 | Reference: models.Reference{ 296 | SourceName: ref.SourceName, 297 | Description: ref.Description, 298 | URL: ref.URL, 299 | }, 300 | } 301 | } 302 | 303 | switch objType := strings.Split(rel.sourceRef, "--")[0]; objType { 304 | case "attack-pattern": 305 | info, ok := attackPatterns[rel.sourceRef] 306 | if !ok { 307 | return models.Technique{}, xerrors.Errorf("Failed to get attack-pattern. relationship id: %s, err: broken relationships. does not exists source ref: %s", rel.id, rel.sourceRef) 308 | } 309 | if info.deprecated { 310 | continue 311 | } 312 | technique.MitreAttack.SubTechniques = append(technique.MitreAttack.SubTechniques, models.SubTechnique{ 313 | Name: info.name, 314 | }) 315 | case "course-of-action": 316 | info, ok := others[rel.sourceRef] 317 | if !ok { 318 | return models.Technique{}, xerrors.Errorf("Failed to get course-of-action. relationship id: %s, err: broken relationships. does not exists source ref: %s", rel.id, rel.sourceRef) 319 | } 320 | if info.deprecated { 321 | continue 322 | } 323 | technique.Mitigations = append(technique.Mitigations, models.Mitigation{ 324 | Name: info.name, 325 | Description: info.description, 326 | }) 327 | case "intrusion-set", "malware", "tool", "campaign": 328 | info, ok := attackers[rel.sourceRef] 329 | if !ok { 330 | return models.Technique{}, xerrors.Errorf("Failed to get attacker. relationship id: %s, err: broken relationships. does not exists source ref: %s", rel.id, rel.sourceRef) 331 | } 332 | if info.deprecated { 333 | continue 334 | } 335 | technique.MitreAttack.Procedures = append(technique.MitreAttack.Procedures, models.Procedure{ 336 | Name: info.name, 337 | Description: info.description, 338 | }) 339 | case "x-mitre-data-component": 340 | info, ok := others[rel.sourceRef] 341 | if !ok { 342 | return models.Technique{}, xerrors.Errorf("Failed to get data-component. relationship id: %s, err: broken relationships. does not exists source ref: %s", rel.id, rel.sourceRef) 343 | } 344 | if info.deprecated { 345 | continue 346 | } 347 | technique.MitreAttack.DataSources = append(technique.MitreAttack.DataSources, models.DataSource{ 348 | Name: info.name, 349 | Description: info.description, 350 | }) 351 | } 352 | } 353 | 354 | for _, phase := range attackPattern.killChainPhases { 355 | info, ok := others[phase] 356 | if !ok { 357 | return models.Technique{}, xerrors.Errorf("Failed to get kill chain phase name. phase_name(x_mitre_shortname): %s, err: broken relationships", phase) 358 | } 359 | technique.MitreAttack.KillChainPhases = append(technique.MitreAttack.KillChainPhases, models.KillChainPhase{Tactic: info.name}) 360 | } 361 | 362 | for _, platform := range attackPattern.platforms { 363 | technique.MitreAttack.Platforms = append(technique.MitreAttack.Platforms, models.TechniquePlatform{ 364 | Platform: platform, 365 | }) 366 | } 367 | 368 | for _, permission := range attackPattern.permissionRequired { 369 | technique.MitreAttack.PermissionsRequired = append(technique.MitreAttack.PermissionsRequired, models.PermissionRequired{ 370 | Permission: permission, 371 | }) 372 | } 373 | 374 | for _, permission := range attackPattern.effectivePermissions { 375 | technique.MitreAttack.EffectivePermissions = append(technique.MitreAttack.EffectivePermissions, models.EffectivePermission{ 376 | Permission: permission, 377 | }) 378 | } 379 | 380 | for _, defense := range attackPattern.defenseByPassed { 381 | technique.MitreAttack.DefenseBypassed = append(technique.MitreAttack.DefenseBypassed, models.DefenseBypassed{ 382 | Defense: defense, 383 | }) 384 | } 385 | 386 | for _, impactType := range attackPattern.impactType { 387 | technique.MitreAttack.ImpactType = append(technique.MitreAttack.ImpactType, models.ImpactType{ 388 | Type: impactType, 389 | }) 390 | } 391 | 392 | for _, ref := range references { 393 | technique.References = append(technique.References, ref) 394 | } 395 | 396 | return technique, nil 397 | } 398 | 399 | func fillAttacker(attacker attacker, relationships []relationship, attackers map[string]attacker, techniquesUsed []techniqueUsed, groupsUsed []groupUsed) (models.Attacker, error) { 400 | attackerInfo := models.Attacker{ 401 | AttackerID: attacker.id, 402 | Name: attacker.name, 403 | Description: attacker.description, 404 | TechniquesUsed: []models.TechniqueUsed{}, 405 | References: []models.AttackerReference{}, 406 | Created: attacker.created, 407 | Modified: attacker.modified, 408 | } 409 | 410 | references := map[string]models.AttackerReference{} 411 | switch attacker.objType { 412 | case "intrusion-set": 413 | attackerInfo.Type = models.GroupType 414 | attackerInfo.Group = &models.AttackerGroup{ 415 | AssociatedGroups: []models.AssociatedGroup{}, 416 | SoftwaresUsed: []models.SoftwareUsed{}, 417 | } 418 | for _, alias := range attacker.aliases { 419 | if fmt.Sprintf("%s: %s", attackerInfo.AttackerID, alias) != attackerInfo.Name { 420 | attackerInfo.Group.AssociatedGroups = append(attackerInfo.Group.AssociatedGroups, models.AssociatedGroup{ 421 | Name: alias, 422 | }) 423 | } 424 | } 425 | for _, ref := range attacker.references { 426 | found := false 427 | for i := range attackerInfo.Group.AssociatedGroups { 428 | if fmt.Sprintf("%s: %s", attackerInfo.AttackerID, ref.SourceName) == attackerInfo.Name { 429 | found = true 430 | break 431 | } 432 | if ref.SourceName == attackerInfo.Group.AssociatedGroups[i].Name { 433 | attackerInfo.Group.AssociatedGroups[i].Description = ref.Description 434 | found = true 435 | break 436 | } 437 | } 438 | if !found { 439 | references[ref.SourceName] = models.AttackerReference{ 440 | Reference: models.Reference{ 441 | SourceName: ref.SourceName, 442 | Description: ref.Description, 443 | URL: ref.URL, 444 | }, 445 | } 446 | } 447 | } 448 | for _, groupUsed := range groupsUsed { 449 | attackerInfo.Group.SoftwaresUsed = append(attackerInfo.Group.SoftwaresUsed, models.SoftwareUsed{ 450 | Name: groupUsed.name, 451 | Description: groupUsed.description, 452 | }) 453 | for _, ref := range groupUsed.references { 454 | references[ref.SourceName] = models.AttackerReference{ 455 | Reference: models.Reference{ 456 | SourceName: ref.SourceName, 457 | Description: ref.Description, 458 | URL: ref.URL, 459 | }, 460 | } 461 | } 462 | } 463 | case "malware": 464 | attackerInfo.Type = models.SoftwareType 465 | attackerInfo.Software = &models.AttackerSoftware{ 466 | Type: models.MalwareType, 467 | AssociatedSoftwares: []models.AssociatedSoftware{}, 468 | Platforms: []models.SoftwarePlatform{}, 469 | GroupsUsed: []models.GroupUsed{}, 470 | } 471 | for _, alias := range attacker.aliases { 472 | if fmt.Sprintf("%s: %s", attackerInfo.AttackerID, alias) != attackerInfo.Name { 473 | attackerInfo.Software.AssociatedSoftwares = append(attackerInfo.Software.AssociatedSoftwares, models.AssociatedSoftware{ 474 | Name: alias, 475 | }) 476 | } 477 | } 478 | for _, platform := range attacker.platforms { 479 | attackerInfo.Software.Platforms = append(attackerInfo.Software.Platforms, models.SoftwarePlatform{ 480 | Platform: platform, 481 | }) 482 | } 483 | for _, ref := range attacker.references { 484 | found := false 485 | for i := range attackerInfo.Software.AssociatedSoftwares { 486 | if fmt.Sprintf("%s: %s", attackerInfo.AttackerID, ref.SourceName) == attackerInfo.Name { 487 | found = true 488 | break 489 | } 490 | if ref.SourceName == attackerInfo.Software.AssociatedSoftwares[i].Name { 491 | attackerInfo.Software.AssociatedSoftwares[i].Description = ref.Description 492 | found = true 493 | break 494 | } 495 | } 496 | if !found { 497 | references[ref.SourceName] = models.AttackerReference{ 498 | Reference: models.Reference{ 499 | SourceName: ref.SourceName, 500 | Description: ref.Description, 501 | URL: ref.URL, 502 | }, 503 | } 504 | } 505 | } 506 | case "tool": 507 | attackerInfo.Type = models.SoftwareType 508 | attackerInfo.Software = &models.AttackerSoftware{ 509 | Type: models.ToolType, 510 | AssociatedSoftwares: []models.AssociatedSoftware{}, 511 | Platforms: []models.SoftwarePlatform{}, 512 | GroupsUsed: []models.GroupUsed{}, 513 | } 514 | for _, alias := range attacker.aliases { 515 | if fmt.Sprintf("%s: %s", attackerInfo.AttackerID, alias) != attackerInfo.Name { 516 | attackerInfo.Software.AssociatedSoftwares = append(attackerInfo.Software.AssociatedSoftwares, models.AssociatedSoftware{ 517 | Name: alias, 518 | }) 519 | } 520 | } 521 | for _, platform := range attacker.platforms { 522 | attackerInfo.Software.Platforms = append(attackerInfo.Software.Platforms, models.SoftwarePlatform{ 523 | Platform: platform, 524 | }) 525 | } 526 | for _, ref := range attacker.references { 527 | found := false 528 | for i := range attackerInfo.Software.AssociatedSoftwares { 529 | if fmt.Sprintf("%s: %s", attackerInfo.AttackerID, ref.SourceName) == attackerInfo.Name { 530 | found = true 531 | break 532 | } 533 | if ref.SourceName == attackerInfo.Software.AssociatedSoftwares[i].Name { 534 | attackerInfo.Software.AssociatedSoftwares[i].Description = ref.Description 535 | found = true 536 | break 537 | } 538 | } 539 | if !found { 540 | references[ref.SourceName] = models.AttackerReference{ 541 | Reference: models.Reference{ 542 | SourceName: ref.SourceName, 543 | Description: ref.Description, 544 | URL: ref.URL, 545 | }, 546 | } 547 | } 548 | } 549 | case "campaign": 550 | attackerInfo.Type = models.CampaignType 551 | // attackerInfo.Campaign = &models.AttackerCampaign{ 552 | // Softwares: []models.AttackerCampaignSoftware{}, 553 | // Groups: []models.AttackerCampaignGroup{}, 554 | // } 555 | } 556 | 557 | for _, techniqueUsed := range techniquesUsed { 558 | attackerInfo.TechniquesUsed = append(attackerInfo.TechniquesUsed, models.TechniqueUsed{ 559 | TechniqueID: techniqueUsed.id, 560 | Name: techniqueUsed.name, 561 | Use: techniqueUsed.use, 562 | }) 563 | 564 | for _, ref := range techniqueUsed.references { 565 | references[ref.SourceName] = models.AttackerReference{ 566 | Reference: models.Reference{ 567 | SourceName: ref.SourceName, 568 | Description: ref.Description, 569 | URL: ref.URL, 570 | }, 571 | } 572 | } 573 | } 574 | 575 | for _, rel := range relationships { 576 | if rel.relationshipType == "revoked-by" { 577 | continue 578 | } 579 | 580 | for _, ref := range rel.references { 581 | references[ref.SourceName] = models.AttackerReference{ 582 | Reference: models.Reference{ 583 | SourceName: ref.SourceName, 584 | Description: ref.Description, 585 | URL: ref.URL, 586 | }, 587 | } 588 | } 589 | 590 | attackerUsed, ok := attackers[rel.sourceRef] 591 | if !ok { 592 | return models.Attacker{}, xerrors.Errorf("Failed to get attacker used. rel.id: %s, rel.sourceRef: %s", rel.id, rel.sourceRef) 593 | } 594 | if attackerUsed.deprecated { 595 | continue 596 | } 597 | 598 | if attackerUsed.objType == "intrusion-set" { 599 | attackerInfo.Software.GroupsUsed = append(attackerInfo.Software.GroupsUsed, models.GroupUsed{ 600 | Name: attackerUsed.name, 601 | Description: rel.description, 602 | }) 603 | } 604 | } 605 | 606 | for _, ref := range references { 607 | attackerInfo.References = append(attackerInfo.References, ref) 608 | } 609 | 610 | return attackerInfo, nil 611 | } 612 | -------------------------------------------------------------------------------- /fetcher/attack/types.go: -------------------------------------------------------------------------------- 1 | package attack 2 | 3 | import "time" 4 | 5 | type root struct { 6 | Type string `json:"type"` 7 | Objects []ctiObject `json:"objects"` 8 | ID string `json:"id"` 9 | SpecVersion string `json:"spec_version"` 10 | } 11 | 12 | type ctiObject struct { 13 | Created time.Time `json:"created"` 14 | Modified time.Time `json:"modified,omitempty"` 15 | ID string `json:"id"` 16 | Type string `json:"type"` 17 | Name string `json:"name,omitempty"` 18 | Description string `json:"description,omitempty"` 19 | CreatedByRef string `json:"created_by_ref,omitempty"` 20 | ObjectMarkingRefs []string `json:"object_marking_refs,omitempty"` 21 | KillChainPhases []struct { 22 | KillChainName string `json:"kill_chain_name"` 23 | PhaseName string `json:"phase_name"` 24 | } `json:"kill_chain_phases,omitempty"` 25 | ExternalReferences []reference `json:"external_references,omitempty"` 26 | Revoked bool `json:"revoked,omitempty"` 27 | TargetRef string `json:"target_ref,omitempty"` 28 | SourceRef string `json:"source_ref,omitempty"` 29 | TacticRefs []string `json:"tactic_refs,omitempty"` 30 | RelationshipType string `json:"relationship_type,omitempty"` 31 | IdentityClass string `json:"identity_class,omitempty"` 32 | Aliases []string `json:"aliases,omitempty"` 33 | Labels []string `json:"labels,omitempty"` 34 | Definition struct { 35 | Statement string `json:"statement"` 36 | } `json:"definition,omitempty"` 37 | DefinitionType string `json:"definition_type,omitempty"` 38 | XMitreShortname string `json:"x_mitre_shortname,omitempty"` 39 | XMitreDetection string `json:"x_mitre_detection,omitempty"` 40 | XMitreDataSources []string `json:"x_mitre_data_sources,omitempty"` 41 | XMitreVersion string `json:"x_mitre_version,omitempty"` 42 | XMitrePermissionsRequired []string `json:"x_mitre_permissions_required,omitempty"` 43 | XMitrePlatforms []string `json:"x_mitre_platforms,omitempty"` 44 | XMitreIsSubtechnique bool `json:"x_mitre_is_subtechnique,omitempty"` 45 | // XMitreContributors []string `json:"x_mitre_contributors,omitempty"` 46 | XMitreSystemRequirements []string `json:"x_mitre_system_requirements,omitempty"` 47 | XMitreDefenseBypassed []string `json:"x_mitre_defense_bypassed,omitempty"` 48 | XMitreEffectivePermissions []string `json:"x_mitre_effective_permissions,omitempty"` 49 | XMitreImpactType []string `json:"x_mitre_impact_type,omitempty"` 50 | XMitreNetworkRequirements bool `json:"x_mitre_network_requirements,omitempty"` 51 | XMitreRemoteSupport bool `json:"x_mitre_remote_support,omitempty"` 52 | XMitreDeprecated bool `json:"x_mitre_deprecated,omitempty"` 53 | // XMitreOldAttackID string `json:"x_mitre_old_attack_id,omitempty"` 54 | XMitreAliases []string `json:"x_mitre_aliases,omitempty"` 55 | // XMitreCollectionLayers []string `json:"x_mitre_collection_layers,omitempty"` 56 | XMitreDataSourceRef string `json:"x_mitre_data_source_ref,omitempty"` 57 | } 58 | type attackPattern struct { 59 | id string 60 | name string 61 | description string 62 | dataSources string 63 | permissionRequired []string 64 | effectivePermissions []string 65 | platforms []string 66 | impactType []string 67 | networkRequirements bool 68 | remoteSupport bool 69 | defenseByPassed []string 70 | killChainPhases []string 71 | detection string 72 | references []reference 73 | capecIDs []string 74 | created time.Time 75 | modified time.Time 76 | deprecated bool 77 | } 78 | 79 | type attacker struct { 80 | objType string 81 | id string 82 | name string 83 | description string 84 | platforms []string 85 | aliases []string 86 | references []reference 87 | created time.Time 88 | modified time.Time 89 | deprecated bool 90 | } 91 | 92 | type reference struct { 93 | SourceName string `json:"source_name"` 94 | ExternalID string `json:"external_id,omitempty"` 95 | URL string `json:"url"` 96 | Description string `json:"description,omitempty"` 97 | } 98 | 99 | type otherInfo struct { 100 | objType string 101 | name string 102 | description string 103 | deprecated bool 104 | } 105 | 106 | type relationship struct { 107 | id string 108 | description string 109 | relationshipType string 110 | sourceRef string 111 | targetRef string 112 | references []reference 113 | } 114 | type dataComponent struct { 115 | name string 116 | description string 117 | dataSourceRef string 118 | } 119 | 120 | type techniqueUsed struct { 121 | id string 122 | name string 123 | use string 124 | references []reference 125 | } 126 | 127 | type groupUsed struct { 128 | name string 129 | description string 130 | references []reference 131 | } 132 | -------------------------------------------------------------------------------- /fetcher/capec/capec.go: -------------------------------------------------------------------------------- 1 | package capec 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "slices" 7 | 8 | "github.com/inconshreveable/log15" 9 | "golang.org/x/xerrors" 10 | 11 | "github.com/vulsio/go-cti/models" 12 | "github.com/vulsio/go-cti/utils" 13 | ) 14 | 15 | const capecURL = "https://raw.githubusercontent.com/mitre/cti/master/capec/2.1/stix-capec.json" 16 | 17 | // Fetch CAPEC data 18 | func Fetch() ([]models.Technique, error) { 19 | log15.Info("Fetching CAPEC...") 20 | 21 | res, err := utils.FetchURL(capecURL) 22 | if err != nil { 23 | return nil, xerrors.Errorf("Failed to fetch CAPEC JSON. err: %w", err) 24 | } 25 | techniques, err := parse(res) 26 | if err != nil { 27 | return nil, xerrors.Errorf("Failed to parse CAPEC Cyber Threat Intelligence. err: %w", err) 28 | } 29 | return techniques, nil 30 | } 31 | 32 | func parse(res []byte) ([]models.Technique, error) { 33 | var r root 34 | if err := json.Unmarshal(res, &r); err != nil { 35 | return nil, xerrors.Errorf("Failed to unmarshal json. err: %w", err) 36 | } 37 | 38 | attackPatterns := map[string]attackPattern{} 39 | additionalInfos := map[string]additionalInfoObject{} 40 | relationships := map[string][]relationshipObject{} 41 | for _, obj := range r.Objects { 42 | if obj.XCapecStatus == "Deprecated" { 43 | continue 44 | } 45 | 46 | switch obj.Type { 47 | case "attack-pattern": 48 | attackPatterns[obj.ID] = parseCAPECAttackPattern(obj) 49 | case "course-of-action": 50 | additionalInfos[obj.ID] = additionalInfoObject{ 51 | objType: obj.Type, 52 | name: obj.Description, 53 | description: fmt.Sprintf("%s: %s", obj.Name, obj.Description), 54 | } 55 | case "relationship": 56 | relationships[obj.TargetRef] = append(relationships[obj.TargetRef], relationshipObject{ 57 | id: obj.ID, 58 | relationshipType: obj.RelationshipType, 59 | sourceRef: obj.SourceRef, 60 | targetRef: obj.TargetRef, 61 | }) 62 | } 63 | } 64 | 65 | techniques := []models.Technique{} 66 | for id, attackPattern := range attackPatterns { 67 | technique := models.Technique{ 68 | TechniqueID: attackPattern.id, 69 | Type: models.CAPECType, 70 | Name: attackPattern.name, 71 | Description: attackPattern.description, 72 | References: []models.TechniqueReference{}, 73 | Mitigations: []models.Mitigation{}, 74 | Capec: &models.Capec{ 75 | AttackIDs: []models.AttackID{}, 76 | Status: attackPattern.status, 77 | ExtendedDescription: attackPattern.extendedDescription, 78 | TypicalSeverity: attackPattern.typicalSeverity, 79 | LikelihoodOfAttack: attackPattern.likelihoodOfAttack, 80 | Relationships: []models.Relationship{}, 81 | Domains: []models.Domain{}, 82 | AlternateTerms: []models.AlternateTerm{}, 83 | ExampleInstances: []models.ExampleInstance{}, 84 | Prerequisites: []models.Prerequisite{}, 85 | ResourcesRequired: []models.ResourceRequired{}, 86 | SkillsRequired: []models.SkillRequired{}, 87 | Abstraction: attackPattern.abstraction, 88 | ExecutionFlow: attackPattern.executionFlow, 89 | Consequences: []models.Consequence{}, 90 | RelatedWeaknesses: []models.RelatedWeakness{}, 91 | }, 92 | Created: attackPattern.created, 93 | Modified: attackPattern.modified, 94 | } 95 | 96 | for _, attackID := range attackPattern.attackIDs { 97 | technique.Capec.AttackIDs = append(technique.Capec.AttackIDs, models.AttackID{ 98 | AttackID: attackID, 99 | }) 100 | } 101 | 102 | for _, ref := range attackPattern.references { 103 | technique.References = append(technique.References, models.TechniqueReference{ 104 | Reference: models.Reference{ 105 | SourceName: ref.SourceName, 106 | Description: ref.Description, 107 | URL: ref.URL, 108 | }, 109 | }) 110 | } 111 | 112 | slices.Sort(attackPattern.domains) 113 | for _, domain := range slices.Compact(attackPattern.domains) { 114 | technique.Capec.Domains = append(technique.Capec.Domains, models.Domain{ 115 | Domain: domain, 116 | }) 117 | } 118 | 119 | for _, term := range attackPattern.alternateTerms { 120 | technique.Capec.AlternateTerms = append(technique.Capec.AlternateTerms, 121 | models.AlternateTerm{ 122 | Term: term, 123 | }) 124 | } 125 | 126 | for _, exampleInstance := range attackPattern.exampleInstances { 127 | technique.Capec.ExampleInstances = append(technique.Capec.ExampleInstances, models.ExampleInstance{ 128 | Instance: exampleInstance, 129 | }) 130 | } 131 | 132 | for _, prerequisite := range attackPattern.prerequisites { 133 | technique.Capec.Prerequisites = append(technique.Capec.Prerequisites, models.Prerequisite{ 134 | Prerequisite: prerequisite, 135 | }) 136 | } 137 | 138 | for _, resource := range attackPattern.resourcesRequired { 139 | technique.Capec.ResourcesRequired = append(technique.Capec.ResourcesRequired, models.ResourceRequired{ 140 | Resource: resource, 141 | }) 142 | } 143 | 144 | for _, nature := range []string{"ChildOf", "ParentOf", "CanFollow", "CanPrecede", "PeerOf"} { 145 | var refs []string 146 | switch nature { 147 | case "ChildOf": 148 | refs = attackPattern.childOfRefs 149 | case "ParentOf": 150 | refs = attackPattern.parentOfRefs 151 | case "CanFollow": 152 | refs = attackPattern.canFollowRefs 153 | case "CanPrecede": 154 | refs = attackPattern.canPrecedeRefs 155 | case "PeerOf": 156 | refs = attackPattern.peerOfRefs 157 | } 158 | 159 | rels, err := expandFromRefIDToName(refs, nature, attackPatterns) 160 | if err != nil { 161 | return nil, xerrors.Errorf("Failed to expand %s references. id: %s, err: %w", nature, id, err) 162 | } 163 | technique.Capec.Relationships = append(technique.Capec.Relationships, rels...) 164 | } 165 | 166 | for _, skill := range attackPattern.skillRequired { 167 | technique.Capec.SkillsRequired = append(technique.Capec.SkillsRequired, models.SkillRequired{ 168 | Skill: skill, 169 | }) 170 | } 171 | 172 | for _, consequence := range attackPattern.consequences { 173 | technique.Capec.Consequences = append(technique.Capec.Consequences, models.Consequence{ 174 | Consequence: consequence, 175 | }) 176 | } 177 | 178 | for _, cweID := range attackPattern.relatedWeaknesses { 179 | technique.Capec.RelatedWeaknesses = append(technique.Capec.RelatedWeaknesses, models.RelatedWeakness{ 180 | CweID: cweID, 181 | }) 182 | } 183 | 184 | for _, rel := range relationships[id] { 185 | info, ok := additionalInfos[rel.sourceRef] 186 | if !ok { 187 | return nil, xerrors.Errorf("Failed to get additionalInfo. id: %s, err: broken relationships", rel.id) 188 | } 189 | technique.Mitigations = append(technique.Mitigations, models.Mitigation{ 190 | Name: info.name, 191 | Description: info.description, 192 | }) 193 | 194 | } 195 | 196 | techniques = append(techniques, technique) 197 | } 198 | 199 | return techniques, nil 200 | } 201 | 202 | func parseCAPECAttackPattern(obj ctiObject) attackPattern { 203 | slices.Sort(obj.XCapecDomains) 204 | 205 | r := attackPattern{ 206 | status: obj.XCapecStatus, 207 | abstraction: obj.XCapecAbstraction, 208 | likelihoodOfAttack: obj.XCapecLikelihoodOfAttack, 209 | typicalSeverity: obj.XCapecTypicalSeverity, 210 | description: obj.Description, 211 | extendedDescription: obj.XCapecExtendedDescription, 212 | alternateTerms: obj.XCapecAlternateTerms, 213 | executionFlow: obj.XCapecExecutionFlow, 214 | exampleInstances: obj.XCapecExampleInstances, 215 | domains: obj.XCapecDomains, 216 | prerequisites: obj.XCapecPrerequisites, 217 | resourcesRequired: obj.XCapecResourcesRequired, 218 | parentOfRefs: obj.XCapecParentOfRefs, 219 | childOfRefs: obj.XCapecChildOfRefs, 220 | canFollowRefs: obj.XCapecCanFollowRefs, 221 | canPrecedeRefs: obj.XCapecCanPrecedeRefs, 222 | peerOfRefs: obj.XCapecPeerOfRefs, 223 | created: obj.Created, 224 | modified: obj.Modified, 225 | } 226 | 227 | name := obj.Name 228 | for _, ref := range obj.ExternalReferences { 229 | switch ref.SourceName { 230 | case "capec": 231 | r.id = ref.ExternalID 232 | name = fmt.Sprintf("%s: %s", ref.ExternalID, obj.Name) 233 | case "ATTACK": 234 | r.attackIDs = append(r.attackIDs, ref.ExternalID) 235 | case "cwe": 236 | r.relatedWeaknesses = append(r.relatedWeaknesses, ref.ExternalID) 237 | default: 238 | r.references = append(r.references, ref) 239 | } 240 | } 241 | r.name = name 242 | 243 | for scope, impacts := range obj.XCapecConsequences { 244 | for _, impact := range impacts { 245 | r.consequences = append(r.consequences, fmt.Sprintf("%s: %s", scope, impact)) 246 | } 247 | } 248 | 249 | for level, description := range obj.XCapecSkillsRequired { 250 | r.skillRequired = append(r.skillRequired, fmt.Sprintf("%s: %s", level, description)) 251 | } 252 | 253 | return r 254 | } 255 | 256 | func expandFromRefIDToName(references []string, nature string, attackPatterns map[string]attackPattern) ([]models.Relationship, error) { 257 | rels := []models.Relationship{} 258 | for _, refID := range references { 259 | relAttackPattern, ok := attackPatterns[refID] 260 | if !ok { 261 | return nil, xerrors.Errorf("Failed to get relational attack pattern. missing id: %s, err: broken relationships", refID) 262 | } 263 | rels = append(rels, models.Relationship{ 264 | Nature: nature, 265 | Relation: fmt.Sprintf("%s: %s", relAttackPattern.abstraction, relAttackPattern.name), 266 | }) 267 | } 268 | return rels, nil 269 | } 270 | -------------------------------------------------------------------------------- /fetcher/capec/capec_test.go: -------------------------------------------------------------------------------- 1 | package capec 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | "time" 7 | 8 | "github.com/google/go-cmp/cmp" 9 | "github.com/google/go-cmp/cmp/cmpopts" 10 | 11 | "github.com/vulsio/go-cti/models" 12 | ) 13 | 14 | func TestParse(t *testing.T) { 15 | tests := []struct { 16 | in string 17 | expected []models.Technique 18 | wantErr bool 19 | }{ 20 | { 21 | in: "testdata/stix-capec.json", 22 | expected: []models.Technique{ 23 | { 24 | TechniqueID: "CAPEC-1", 25 | Type: models.CAPECType, 26 | Name: "CAPEC-1: Accessing Functionality Not Properly Constrained by ACLs", 27 | Description: "In applications, particularly web applications, access to functionality is mitigated by an authorization framework. This framework maps Access Control Lists (ACLs) to elements of the application's functionality; particularly URL's for web apps. In the case that the administrator failed to specify an ACL for a particular element, an attacker may be able to access it with impunity. An attacker with the ability to access functionality not properly constrained by ACLs can obtain sensitive information and possibly compromise the entire application. Such an attacker can access resources that must be available only to users at a higher privilege level, can access management sections of the application, or can run queries for data that they otherwise not supposed to.", 28 | References: []models.TechniqueReference{}, 29 | Mitigations: []models.Mitigation{ 30 | { 31 | Name: "\n In a J2EE setting, administrators can associate a role that is impossible for the authenticator to grant users, such as \"NoAccess\", with all Servlets to which access is guarded by a limited number of servlets visible to, and accessible by, the user.\n Having done so, any direct access to those protected Servlets will be prohibited by the web container.\n In a more general setting, the administrator must mark every resource besides the ones supposed to be exposed to the user as accessible by a role impossible for the user to assume. The default security setting must be to deny access and then grant access only to those resources intended by business logic.\n ", 32 | Description: "coa-1-0: \n In a J2EE setting, administrators can associate a role that is impossible for the authenticator to grant users, such as \"NoAccess\", with all Servlets to which access is guarded by a limited number of servlets visible to, and accessible by, the user.\n Having done so, any direct access to those protected Servlets will be prohibited by the web container.\n In a more general setting, the administrator must mark every resource besides the ones supposed to be exposed to the user as accessible by a role impossible for the user to assume. The default security setting must be to deny access and then grant access only to those resources intended by business logic.\n ", 33 | }, 34 | }, 35 | MitreAttack: nil, 36 | Capec: &models.Capec{ 37 | AttackIDs: []models.AttackID{ 38 | { 39 | AttackID: "T1574.010", 40 | }, 41 | }, 42 | Status: "Draft", 43 | TypicalSeverity: "High", 44 | LikelihoodOfAttack: "High", 45 | Relationships: []models.Relationship{ 46 | { 47 | Nature: "ChildOf", 48 | Relation: "Meta: CAPEC-122: Privilege Abuse", 49 | }, 50 | { 51 | Nature: "ParentOf", 52 | Relation: "Detailed: CAPEC-58: Restful Privilege Elevation", 53 | }, 54 | { 55 | Nature: "CanFollow", 56 | Relation: "Standard: CAPEC-test: mock for test", 57 | }, 58 | { 59 | Nature: "CanPrecede", 60 | Relation: "Standard: CAPEC-17: Using Malicious Files", 61 | }, 62 | { 63 | Nature: "PeerOf", 64 | Relation: "Standard: CAPEC-test: mock for test", 65 | }, 66 | }, 67 | Domains: []models.Domain{{Domain: "Hardware"}, {Domain: "Software"}}, 68 | AlternateTerms: []models.AlternateTerm{{Term: "term1"}, {Term: "term2"}}, 69 | ExampleInstances: []models.ExampleInstance{{Instance: "\n Implementing the Model-View-Controller (MVC) within Java EE's Servlet paradigm using a \"Single front controller\" pattern that demands that brokered HTTP requests be authenticated before hand-offs to other Action Servlets.\n If no security-constraint is placed on those Action Servlets, such that positively no one can access them, the front controller can be subverted.\n "}}, 70 | Prerequisites: []models.Prerequisite{ 71 | {Prerequisite: "The application must be navigable in a manner that associates elements (subsections) of the application with ACLs."}, 72 | {Prerequisite: "The various resources, or individual URLs, must be somehow discoverable by the attacker"}, 73 | {Prerequisite: "The administrator must have forgotten to associate an ACL or has associated an inappropriately permissive ACL with a particular navigable resource."}, 74 | }, 75 | ResourcesRequired: []models.ResourceRequired{{Resource: "None: No specialized resources are required to execute this type of attack."}}, 76 | SkillsRequired: []models.SkillRequired{ 77 | {Skill: "Low: In order to discover unrestricted resources, the attacker does not need special tools or skills. They only have to observe the resources or access mechanisms invoked as each action is performed and then try and access those access mechanisms directly."}, 78 | }, 79 | Abstraction: "Standard", 80 | ExecutionFlow: "

Execution Flow

Explore

  1. Survey: The attacker surveys the target application, possibly as a valid and authenticated user

    2. Techniques
    Spidering web sites for all available links
    Brute force guessing of resource names
    Brute force guessing of user names / credentials
    Brute force guessing of function names / actions

  2. Identify Functionality: At each step, the attacker notes the resource or functionality access mechanism invoked upon performing specific actions

    3. Techniques
    Use the web inventory of all forms and inputs and apply attack data to those inputs.
    Use a packet sniffer to capture and record network traffic
    Execute the software in a debugger and record API calls into the operating system or important libraries. This might occur in an environment other than a production environment, in order to find weaknesses that can be exploited in a production environment.

Experiment

  1. Iterate over access capabilities: Possibly as a valid user, the attacker then tries to access each of the noted access mechanisms directly in order to perform functions not constrained by the ACLs.

    2. Techniques
    Fuzzing of API parameters (URL parameters, OS API parameters, protocol parameters)
", 81 | Consequences: []models.Consequence{ 82 | {Consequence: "Access_Control: Gain Privileges"}, 83 | {Consequence: "Authorization: Gain Privileges"}, 84 | {Consequence: "Confidentiality: Gain Privileges"}, 85 | }, 86 | RelatedWeaknesses: []models.RelatedWeakness{ 87 | {CweID: "CWE-276"}, 88 | {CweID: "CWE-285"}, 89 | {CweID: "CWE-434"}, 90 | {CweID: "CWE-693"}, 91 | {CweID: "CWE-732"}, 92 | {CweID: "CWE-1193"}, 93 | {CweID: "CWE-1220"}, 94 | {CweID: "CWE-1297"}, 95 | {CweID: "CWE-1311"}, 96 | {CweID: "CWE-1314"}, 97 | {CweID: "CWE-1315"}, 98 | {CweID: "CWE-1318"}, 99 | {CweID: "CWE-1320"}, 100 | {CweID: "CWE-1321"}, 101 | {CweID: "CWE-1327"}, 102 | }, 103 | }, 104 | Created: time.Date(2014, time.June, 23, 0, 0, 0, 0, time.UTC), 105 | Modified: time.Date(2021, time.October, 21, 0, 0, 0, 0, time.UTC), 106 | }, 107 | { 108 | TechniqueID: "CAPEC-122", 109 | Type: models.CAPECType, 110 | Name: "CAPEC-122: Privilege Abuse", 111 | Description: "An adversary is able to exploit features of the target that should be reserved for privileged users or administrators but are exposed to use by lower or non-privileged accounts. Access to sensitive information and functionality must be controlled to ensure that only authorized users are able to access these resources.", 112 | References: []models.TechniqueReference{}, 113 | Mitigations: []models.Mitigation{}, 114 | MitreAttack: nil, 115 | Capec: &models.Capec{ 116 | AttackIDs: []models.AttackID{}, 117 | Status: "Draft", 118 | ExtendedDescription: "\n If access control mechanisms are absent or misconfigured, a user may be able to access resources that are intended only for higher level users. An adversary may be able to exploit this to utilize a less trusted account to gain information and perform activities reserved for more trusted accounts.\n This attack differs from privilege escalation and other privilege stealing attacks in that the adversary never actually escalates their privileges but instead is able to use a lesser degree of privilege to access resources that should be (but are not) reserved for higher privilege accounts. Likewise, the adversary does not exploit trust or subvert systems - all control functionality is working as configured but the configuration does not adequately protect sensitive resources at an appropriate level.\n ", 119 | TypicalSeverity: "Medium", 120 | LikelihoodOfAttack: "High", 121 | Relationships: []models.Relationship{}, 122 | Domains: []models.Domain{{Domain: "Hardware"}, {Domain: "Software"}}, 123 | AlternateTerms: []models.AlternateTerm{}, 124 | ExampleInstances: []models.ExampleInstance{{Instance: "\n Improperly configured account privileges allowed unauthorized users on a hospital's network to access the medical records for over 3,000 patients. Thus compromising data integrity and confidentiality in addition to HIPAA violations.\n "}}, 125 | Prerequisites: []models.Prerequisite{ 126 | {Prerequisite: "The target must have misconfigured their access control mechanisms such that sensitive information, which should only be accessible to more trusted users, remains accessible to less trusted users."}, 127 | {Prerequisite: "The adversary must have access to the target, albeit with an account that is less privileged than would be appropriate for the targeted resources."}, 128 | }, 129 | ResourcesRequired: []models.ResourceRequired{{Resource: "None: No specialized resources are required to execute this type of attack. The ability to access the target is required."}}, 130 | SkillsRequired: []models.SkillRequired{ 131 | {Skill: "Low: Adversary can leverage privileged features they already have access to without additional effort or skill. Adversary is only required to have access to an account with improper privileges."}, 132 | }, 133 | Abstraction: "Meta", 134 | Consequences: []models.Consequence{ 135 | {Consequence: "Access_Control: Bypass Protection Mechanism"}, 136 | {Consequence: "Authorization: Bypass Protection Mechanism"}, 137 | {Consequence: "Authorization: Execute Unauthorized Commands (Run Arbitrary Code)"}, 138 | {Consequence: "Authorization: Gain Privileges"}, 139 | {Consequence: "Confidentiality: Read Data"}, 140 | {Consequence: "Integrity: Modify Data"}, 141 | }, 142 | RelatedWeaknesses: []models.RelatedWeakness{ 143 | {CweID: "CWE-269"}, 144 | {CweID: "CWE-732"}, 145 | {CweID: "CWE-1317"}, 146 | }, 147 | }, 148 | Created: time.Date(2014, time.June, 23, 0, 0, 0, 0, time.UTC), 149 | Modified: time.Date(2022, time.February, 22, 0, 0, 0, 0, time.UTC), 150 | }, 151 | { 152 | TechniqueID: "CAPEC-17", 153 | Type: models.CAPECType, 154 | Name: "CAPEC-17: Using Malicious Files", 155 | Description: "An attack of this type exploits a system's configuration that allows an adversary to either directly access an executable file, for example through shell access; or in a possible worst case allows an adversary to upload a file and then execute it. Web servers, ftp servers, and message oriented middleware systems which have many integration points are particularly vulnerable, because both the programmers and the administrators must be in synch regarding the interfaces and the correct privileges for each interface.", 156 | References: []models.TechniqueReference{ 157 | { 158 | Reference: models.Reference{ 159 | SourceName: "reference_from_CAPEC", 160 | Description: "G. Hoglund, G. McGraw, Exploiting Software: How to Break Code, 2004--02, Addison-Wesley", 161 | }, 162 | }, 163 | }, 164 | Mitigations: []models.Mitigation{}, 165 | MitreAttack: nil, 166 | Capec: &models.Capec{ 167 | AttackIDs: []models.AttackID{ 168 | {AttackID: "T1574.010"}, 169 | }, 170 | Status: "Draft", 171 | TypicalSeverity: "Very High", 172 | LikelihoodOfAttack: "High", 173 | Relationships: []models.Relationship{}, 174 | Domains: []models.Domain{{Domain: "Hardware"}, {Domain: "Software"}}, 175 | AlternateTerms: []models.AlternateTerm{}, 176 | ExampleInstances: []models.ExampleInstance{{Instance: "\n Consider a directory on a web server with the following permissions\n drwxrwxrwx 5 admin public 170 Nov 17 01:08 webroot\n This could allow an attacker to both execute and upload and execute programs' on the web server. This one vulnerability can be exploited by a threat to probe the system and identify additional vulnerabilities to exploit.\n "}}, 177 | Prerequisites: []models.Prerequisite{{Prerequisite: "System's configuration must allow an attacker to directly access executable files or upload files to execute. This means that any access control system that is supposed to mediate communications between the subject and the object is set incorrectly or assumes a benign environment."}}, 178 | ResourcesRequired: []models.ResourceRequired{{Resource: "Ability to communicate synchronously or asynchronously with server that publishes an over-privileged directory, program, or interface. Optionally, ability to capture output directly through synchronous communication or other method such as FTP."}}, 179 | SkillsRequired: []models.SkillRequired{ 180 | {Skill: "Low: To identify and execute against an over-privileged system interface"}, 181 | }, 182 | Abstraction: "Standard", 183 | ExecutionFlow: "

Execution Flow

Explore

  1. Determine File/Directory Configuration: The adversary looks for misconfigured files or directories on a system that might give executable access to an overly broad group of users.

    2. Techniques
    Through shell access to a system, use the command \"ls -l\" to view permissions for files and directories.

Experiment

  1. Upload Malicious Files: If the adversary discovers a directory that has executable permissions, they will attempt to upload a malicious file to execute.

    2. Techniques
    Upload a malicious file through a misconfigured FTP server.

Exploit

  1. Execute Malicious File: The adversary either executes the uploaded malicious file, or executes an existing file that has been misconfigured to allow executable access to the adversary.

", 184 | Consequences: []models.Consequence{ 185 | {Consequence: "Access_Control: Gain Privileges"}, 186 | {Consequence: "Authorization: Gain Privileges"}, 187 | {Consequence: "Availability: Execute Unauthorized Commands (Run Arbitrary Code)"}, 188 | {Consequence: "Confidentiality: Execute Unauthorized Commands (Run Arbitrary Code)"}, 189 | {Consequence: "Confidentiality: Gain Privileges"}, 190 | {Consequence: "Confidentiality: Read Data"}, 191 | {Consequence: "Integrity: Execute Unauthorized Commands (Run Arbitrary Code)"}, 192 | {Consequence: "Integrity: Modify Data"}, 193 | }, 194 | RelatedWeaknesses: []models.RelatedWeakness{ 195 | {CweID: "CWE-732"}, 196 | {CweID: "CWE-285"}, 197 | {CweID: "CWE-272"}, 198 | {CweID: "CWE-59"}, 199 | {CweID: "CWE-282"}, 200 | {CweID: "CWE-270"}, 201 | {CweID: "CWE-693"}, 202 | }, 203 | }, 204 | Created: time.Date(2014, time.June, 23, 0, 0, 0, 0, time.UTC), 205 | Modified: time.Date(2022, time.February, 22, 0, 0, 0, 0, time.UTC), 206 | }, 207 | { 208 | TechniqueID: "CAPEC-58", 209 | Type: models.CAPECType, 210 | Name: "CAPEC-58: Restful Privilege Elevation", 211 | Description: "Rest uses standard HTTP (Get, Put, Delete) style permissions methods, but these are not necessarily correlated generally with back end programs. Strict interpretation of HTTP get methods means that these HTTP Get services should not be used to delete information on the server, but there is no access control mechanism to back up this logic. This means that unless the services are properly ACL'd and the application's service implementation are following these guidelines then an HTTP request can easily execute a delete or update on the server side. The attacker identifies a HTTP Get URL such as http://victimsite/updateOrder, which calls out to a program to update orders on a database or other resource. The URL is not idempotent so the request can be submitted multiple times by the attacker, additionally, the attacker may be able to exploit the URL published as a Get method that actually performs updates (instead of merely retrieving data). This may result in malicious or inadvertent altering of data on the server.", 212 | References: []models.TechniqueReference{ 213 | { 214 | Reference: models.Reference{ 215 | SourceName: "reference_from_CAPEC", 216 | Description: "Mark O'Neill, Security for REST Web Services, Vprde;", 217 | URL: "http://www.vordel.com/downloads/rsa_conf_2006.pdf", 218 | }, 219 | }, 220 | }, 221 | Mitigations: []models.Mitigation{}, 222 | MitreAttack: nil, 223 | Capec: &models.Capec{ 224 | AttackIDs: []models.AttackID{}, 225 | Status: "Draft", 226 | TypicalSeverity: "High", 227 | LikelihoodOfAttack: "High", 228 | Relationships: []models.Relationship{}, 229 | Domains: []models.Domain{{Domain: "Hardware"}, {Domain: "Software"}}, 230 | AlternateTerms: []models.AlternateTerm{}, 231 | ExampleInstances: []models.ExampleInstance{{Instance: "The HTTP Get method is designed to retrieve resources and not to alter the state of the application or resources on the server side. However, developers can easily code programs that accept a HTTP Get request that do in fact create, update or delete data on the server. Both Flickr (http://www.flickr.com/services/api/flickr.photosets.delete.html) and del.icio.us (http://del.icio.us/api/posts/delete) have implemented delete operations using standard HTTP Get requests. These HTTP Get methods do delete data on the server side, despite being called from Get which is not supposed to alter state."}}, 232 | Prerequisites: []models.Prerequisite{{Prerequisite: "The attacker needs to be able to identify HTTP Get URLs. The Get methods must be set to call applications that perform operations other than get such as update and delete."}}, 233 | ResourcesRequired: []models.ResourceRequired{}, 234 | SkillsRequired: []models.SkillRequired{ 235 | {Skill: "Low: It is relatively straightforward to identify an HTTP Get method that changes state on the server side and executes against an over-privileged system interface"}, 236 | }, 237 | Abstraction: "Detailed", 238 | Consequences: []models.Consequence{ 239 | {Consequence: "Access_Control: Gain Privileges"}, 240 | {Consequence: "Authorization: Gain Privileges"}, 241 | {Consequence: "Confidentiality: Gain Privileges"}, 242 | {Consequence: "Integrity: Modify Data"}, 243 | }, 244 | RelatedWeaknesses: []models.RelatedWeakness{ 245 | {CweID: "CWE-267"}, 246 | {CweID: "CWE-269"}, 247 | }, 248 | }, 249 | Created: time.Date(2014, time.June, 23, 0, 0, 0, 0, time.UTC), 250 | Modified: time.Date(2021, time.June, 24, 0, 0, 0, 0, time.UTC), 251 | }, 252 | { 253 | TechniqueID: "CAPEC-test", 254 | Type: models.CAPECType, 255 | Name: "CAPEC-test: mock for test", 256 | Description: "", 257 | References: []models.TechniqueReference{}, 258 | Mitigations: []models.Mitigation{}, 259 | MitreAttack: nil, 260 | Capec: &models.Capec{ 261 | AttackIDs: []models.AttackID{}, 262 | Status: "", 263 | ExtendedDescription: "", 264 | TypicalSeverity: "", 265 | LikelihoodOfAttack: "", 266 | Relationships: []models.Relationship{}, 267 | Domains: []models.Domain{}, 268 | AlternateTerms: []models.AlternateTerm{}, 269 | ExampleInstances: []models.ExampleInstance{}, 270 | Prerequisites: []models.Prerequisite{}, 271 | ResourcesRequired: []models.ResourceRequired{}, 272 | SkillsRequired: []models.SkillRequired{}, 273 | Abstraction: "Standard", 274 | Consequences: []models.Consequence{}, 275 | RelatedWeaknesses: []models.RelatedWeakness{}, 276 | }, 277 | Created: time.Date(2014, time.June, 23, 0, 0, 0, 0, time.UTC), 278 | Modified: time.Date(2019, time.September, 30, 0, 0, 0, 0, time.UTC), 279 | }, 280 | }, 281 | }, 282 | } 283 | 284 | for i, tt := range tests { 285 | res, err := os.ReadFile(tt.in) 286 | if err != nil { 287 | t.Fatalf("[%d] Failed to read file. err: %s", i, err) 288 | } 289 | actual, err := parse(res) 290 | if err != nil { 291 | if tt.wantErr { 292 | continue 293 | } 294 | t.Fatalf("[%d] Failed to parse. err: %s", i, err) 295 | } 296 | 297 | opts := []cmp.Option{ 298 | cmpopts.SortSlices(func(i, j models.Technique) bool { 299 | return i.TechniqueID < j.TechniqueID 300 | }), 301 | cmpopts.SortSlices(func(i, j models.Consequence) bool { 302 | return i.Consequence < j.Consequence 303 | }), 304 | } 305 | if diff := cmp.Diff(actual, tt.expected, opts...); diff != "" { 306 | t.Errorf("[%d] parse diff: (-got +want)\n%s", i, diff) 307 | } 308 | } 309 | } 310 | -------------------------------------------------------------------------------- /fetcher/capec/types.go: -------------------------------------------------------------------------------- 1 | package capec 2 | 3 | import "time" 4 | 5 | type root struct { 6 | Type string `json:"type"` 7 | ID string `json:"id"` 8 | Objects []ctiObject `json:"objects"` 9 | } 10 | 11 | type ctiObject struct { 12 | ID string `json:"id"` 13 | Type string `json:"type"` 14 | Name string `json:"name,omitempty"` 15 | Description string `json:"description,omitempty"` 16 | ExternalReferences []reference `json:"external_references,omitempty"` 17 | RelationshipType string `json:"relationship_type,omitempty"` 18 | SourceRef string `json:"source_ref,omitempty"` 19 | TargetRef string `json:"target_ref,omitempty"` 20 | Created time.Time `json:"created"` 21 | Modified time.Time `json:"modified,omitempty"` 22 | 23 | XCapecStatus string `json:"x_capec_status,omitempty"` 24 | XCapecExtendedDescription string `json:"x_capec_extended_description,omitempty"` 25 | XCapecAbstraction string `json:"x_capec_abstraction,omitempty"` 26 | XCapecTypicalSeverity string `json:"x_capec_typical_severity,omitempty"` 27 | XCapecAlternateTerms []string `json:"x_capec_alternate_terms,omitempty"` 28 | XCapecConsequences map[string][]string `json:"x_capec_consequences,omitempty"` 29 | XCapecSkillsRequired map[string]string `json:"x_capec_skills_required,omitempty"` 30 | XCapecDomains []string `json:"x_capec_domains,omitempty"` 31 | XCapecExampleInstances []string `json:"x_capec_example_instances,omitempty"` 32 | XCapecExecutionFlow string `json:"x_capec_execution_flow,omitempty"` 33 | XCapecLikelihoodOfAttack string `json:"x_capec_likelihood_of_attack,omitempty"` 34 | XCapecPrerequisites []string `json:"x_capec_prerequisites,omitempty"` 35 | XCapecResourcesRequired []string `json:"x_capec_resources_required,omitempty"` 36 | XCapecCanPrecedeRefs []string `json:"x_capec_can_precede_refs,omitempty"` 37 | XCapecChildOfRefs []string `json:"x_capec_child_of_refs,omitempty"` 38 | XCapecParentOfRefs []string `json:"x_capec_parent_of_refs,omitempty"` 39 | XCapecPeerOfRefs []string `json:"x_capec_peer_of_refs,omitempty"` 40 | XCapecCanFollowRefs []string `json:"x_capec_can_follow_refs,omitempty"` 41 | } 42 | 43 | type attackPattern struct { 44 | id string 45 | name string 46 | status string 47 | abstraction string 48 | likelihoodOfAttack string 49 | typicalSeverity string 50 | description string 51 | extendedDescription string 52 | alternateTerms []string 53 | executionFlow string 54 | exampleInstances []string 55 | domains []string 56 | consequences []string 57 | prerequisites []string 58 | resourcesRequired []string 59 | skillRequired []string 60 | relatedWeaknesses []string 61 | references []reference 62 | parentOfRefs []string 63 | childOfRefs []string 64 | canFollowRefs []string 65 | canPrecedeRefs []string 66 | peerOfRefs []string 67 | attackIDs []string 68 | created time.Time 69 | modified time.Time 70 | } 71 | 72 | type reference struct { 73 | SourceName string `json:"source_name"` 74 | ExternalID string `json:"external_id,omitempty"` 75 | URL string `json:"url"` 76 | Description string `json:"description,omitempty"` 77 | } 78 | 79 | type additionalInfoObject struct { 80 | objType string 81 | name string 82 | description string 83 | } 84 | 85 | type relationshipObject struct { 86 | id string 87 | description string 88 | relationshipType string 89 | sourceRef string 90 | targetRef string 91 | references []reference 92 | } 93 | -------------------------------------------------------------------------------- /fetcher/cti.go: -------------------------------------------------------------------------------- 1 | package fetcher 2 | 3 | import ( 4 | "golang.org/x/xerrors" 5 | 6 | "github.com/vulsio/go-cti/fetcher/attack" 7 | "github.com/vulsio/go-cti/fetcher/capec" 8 | "github.com/vulsio/go-cti/fetcher/cwe" 9 | "github.com/vulsio/go-cti/fetcher/nvd" 10 | "github.com/vulsio/go-cti/models" 11 | ) 12 | 13 | // FetchCti : 14 | func FetchCti() ([]models.Technique, []models.CveToTechniques, []models.Attacker, error) { 15 | attackTechniques, attackers, err := attack.Fetch() 16 | if err != nil { 17 | return nil, nil, nil, xerrors.Errorf("Failed to fetch MITRE ATT&CK. err: %w", err) 18 | } 19 | 20 | capecTechniques, err := capec.Fetch() 21 | if err != nil { 22 | return nil, nil, nil, xerrors.Errorf("Failed to fetch CAPEC. err: %w", err) 23 | } 24 | techniques := append(attackTechniques, capecTechniques...) 25 | 26 | cweToCapecs, err := cwe.Fetch() 27 | if err != nil { 28 | return nil, nil, nil, xerrors.Errorf("Failed to fetch CWE. err: %w", err) 29 | } 30 | 31 | cveToCwes, err := nvd.Fetch() 32 | if err != nil { 33 | return nil, nil, nil, xerrors.Errorf("Failed to fetch NVD CVE. err: %w", err) 34 | } 35 | 36 | return techniques, buildCveToTechniques(techniques, cweToCapecs, cveToCwes), attackers, nil 37 | } 38 | 39 | func buildCveToTechniques(techniques []models.Technique, cweToCapecs, cveToCwes map[string][]string) []models.CveToTechniques { 40 | capecToCwes := map[string][]string{} 41 | capecToAttacks := map[string][]string{} 42 | for _, technique := range techniques { 43 | if technique.Type != models.CAPECType { 44 | continue 45 | } 46 | 47 | for _, weak := range technique.Capec.RelatedWeaknesses { 48 | capecToCwes[technique.TechniqueID] = append(capecToCwes[technique.TechniqueID], weak.CweID) 49 | } 50 | 51 | for _, attackID := range technique.Capec.AttackIDs { 52 | capecToAttacks[technique.TechniqueID] = append(capecToAttacks[technique.TechniqueID], attackID.AttackID) 53 | } 54 | } 55 | 56 | for capecID, cweIDs := range capecToCwes { 57 | for _, cweID := range cweIDs { 58 | cweToCapecs[cweID] = append(cweToCapecs[cweID], capecID) 59 | } 60 | } 61 | 62 | mappings := []models.CveToTechniques{} 63 | for cveID, cweIDs := range cveToCwes { 64 | uniqTechniqueIDs := map[string]struct{}{} 65 | for _, cweID := range cweIDs { 66 | for _, capecID := range cweToCapecs[cweID] { 67 | uniqTechniqueIDs[capecID] = struct{}{} 68 | for _, attackID := range capecToAttacks[capecID] { 69 | uniqTechniqueIDs[attackID] = struct{}{} 70 | } 71 | } 72 | } 73 | 74 | if len(uniqTechniqueIDs) > 0 { 75 | techniqueIDs := []models.CveToTechniqueID{} 76 | for techniqueID := range uniqTechniqueIDs { 77 | techniqueIDs = append(techniqueIDs, models.CveToTechniqueID{ 78 | TechniqueID: techniqueID, 79 | }) 80 | } 81 | mappings = append(mappings, models.CveToTechniques{ 82 | CveID: cveID, 83 | TechniqueIDs: techniqueIDs, 84 | }) 85 | } 86 | } 87 | return mappings 88 | } 89 | -------------------------------------------------------------------------------- /fetcher/cti_test.go: -------------------------------------------------------------------------------- 1 | package fetcher 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/google/go-cmp/cmp" 7 | "github.com/google/go-cmp/cmp/cmpopts" 8 | 9 | "github.com/vulsio/go-cti/models" 10 | ) 11 | 12 | func TestBuildCveToTechniquess(t *testing.T) { 13 | type args struct { 14 | techniques []models.Technique 15 | cweToCapecs map[string][]string 16 | cveToCwes map[string][]string 17 | } 18 | 19 | tests := []struct { 20 | in args 21 | expected []models.CveToTechniques 22 | }{ 23 | { 24 | in: args{ 25 | techniques: []models.Technique{ 26 | { 27 | TechniqueID: "CAPEC-1", 28 | Type: models.CAPECType, 29 | Capec: &models.Capec{ 30 | AttackIDs: []models.AttackID{ 31 | {AttackID: "T1083"}, 32 | }, 33 | }, 34 | }, 35 | }, 36 | cweToCapecs: map[string][]string{ 37 | "CWE-284": {"CAPEC-1"}, 38 | }, 39 | cveToCwes: map[string][]string{ 40 | "CVE-2020-10627": {"CWE-284"}, 41 | "CVE-2020-0002": {"CWE-787", "CWE-416"}, 42 | }, 43 | }, 44 | expected: []models.CveToTechniques{ 45 | { 46 | CveID: "CVE-2020-10627", 47 | TechniqueIDs: []models.CveToTechniqueID{ 48 | {TechniqueID: "CAPEC-1"}, 49 | {TechniqueID: "T1083"}, 50 | }, 51 | }, 52 | }, 53 | }, 54 | } 55 | 56 | for i, tt := range tests { 57 | opts := []cmp.Option{ 58 | cmpopts.SortSlices(func(i, j models.CveToTechniqueID) bool { 59 | return i.TechniqueID < j.TechniqueID 60 | }), 61 | } 62 | if diff := cmp.Diff(buildCveToTechniques(tt.in.techniques, tt.in.cweToCapecs, tt.in.cveToCwes), tt.expected, opts...); diff != "" { 63 | t.Errorf("[%d] buildCveToTechniques diff: (-got +want)\n%s", i, diff) 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /fetcher/cwe/cwe.go: -------------------------------------------------------------------------------- 1 | package cwe 2 | 3 | import ( 4 | "archive/zip" 5 | "bytes" 6 | "encoding/xml" 7 | "fmt" 8 | "strings" 9 | 10 | "github.com/inconshreveable/log15" 11 | "golang.org/x/xerrors" 12 | 13 | "github.com/vulsio/go-cti/utils" 14 | ) 15 | 16 | const cweURL = "https://cwe.mitre.org/data/xml/cwec_latest.xml.zip" 17 | 18 | // Fetch CWE data 19 | func Fetch() (map[string][]string, error) { 20 | log15.Info("Fetching CWE...") 21 | 22 | res, err := utils.FetchURL(cweURL) 23 | if err != nil { 24 | return nil, xerrors.Errorf("Failed to fetch CWE XML. err: %w", err) 25 | } 26 | mappings, err := parse(res) 27 | if err != nil { 28 | return nil, xerrors.Errorf("Failed to parse CWE XML. err: %w", err) 29 | } 30 | return mappings, nil 31 | } 32 | 33 | func parse(res []byte) (map[string][]string, error) { 34 | reader, err := zip.NewReader(bytes.NewReader(res), int64(len(res))) 35 | if err != nil { 36 | return nil, xerrors.Errorf("Failed to new zip Reader. err: %w", err) 37 | } 38 | 39 | cweIDtoCapecIDs := map[string][]string{} 40 | for _, file := range reader.File { 41 | if !strings.HasPrefix(file.Name, "cwec_v4") { 42 | log15.Warn("Skip CWE parsing since only CWE v4 is supported", "file", file.Name) 43 | continue 44 | } 45 | 46 | r, err := file.Open() 47 | if err != nil { 48 | return nil, xerrors.Errorf("Failed to open file. err: %w", err) 49 | } 50 | defer r.Close() 51 | 52 | var catalog weaknessCatalog 53 | if err := xml.NewDecoder(r).Decode(&catalog); err != nil { 54 | return nil, xerrors.Errorf("Failed to decode xml. err: %w", err) 55 | } 56 | 57 | for _, weakness := range catalog.Weaknesses.Weakness { 58 | cweID := fmt.Sprintf("CWE-%s", weakness.ID) 59 | for _, attackPattern := range weakness.RelatedAttackPatterns.RelatedAttackPattern { 60 | cweIDtoCapecIDs[cweID] = append(cweIDtoCapecIDs[cweID], fmt.Sprintf("CAPEC-%s", attackPattern.CAPECID)) 61 | } 62 | } 63 | } 64 | return cweIDtoCapecIDs, nil 65 | } 66 | -------------------------------------------------------------------------------- /fetcher/cwe/cwe_test.go: -------------------------------------------------------------------------------- 1 | package cwe 2 | 3 | import ( 4 | "maps" 5 | "os" 6 | "slices" 7 | "testing" 8 | ) 9 | 10 | func TestParse(t *testing.T) { 11 | tests := []struct { 12 | in string 13 | expected map[string][]string 14 | }{ 15 | { 16 | in: "testdata/v4.6/cwec.xml.zip", 17 | expected: map[string][]string{ 18 | "CWE-1021": {"CAPEC-103", "CAPEC-181", "CAPEC-222", "CAPEC-504", "CAPEC-506", "CAPEC-654"}, 19 | }, 20 | }, 21 | { 22 | in: "testdata/v4.14/cwec_latest.xml.zip", 23 | expected: map[string][]string{ 24 | "CWE-1021": {"CAPEC-103", "CAPEC-181", "CAPEC-222", "CAPEC-504", "CAPEC-506", "CAPEC-587", "CAPEC-654"}, 25 | }, 26 | }, 27 | { 28 | in: "testdata/v5.0/cwec_latest.xml.zip", 29 | expected: map[string][]string{}, 30 | }, 31 | } 32 | 33 | for i, tt := range tests { 34 | res, err := os.ReadFile(tt.in) 35 | if err != nil { 36 | t.Fatalf("[%d] Failed to read file. err: %s", i, err) 37 | } 38 | actual, err := parse(res) 39 | if err != nil { 40 | t.Fatalf("[%d] Failed to parse. err: %s", i, err) 41 | } 42 | if !maps.EqualFunc(actual, tt.expected, func(v1 []string, v2 []string) bool { 43 | return slices.Equal(v1, v2) 44 | }) { 45 | t.Errorf("[%d] parse expected: %v, actual: %v\n", i, tt.expected, actual) 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /fetcher/cwe/testdata/v4.14/cwec_latest.xml.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vulsio/go-cti/f05a393a9e2faaa5e72a153583bc1d463a5393f0/fetcher/cwe/testdata/v4.14/cwec_latest.xml.zip -------------------------------------------------------------------------------- /fetcher/cwe/testdata/v4.6/cwec.xml.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vulsio/go-cti/f05a393a9e2faaa5e72a153583bc1d463a5393f0/fetcher/cwe/testdata/v4.6/cwec.xml.zip -------------------------------------------------------------------------------- /fetcher/cwe/testdata/v5.0/cwec_latest.xml.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vulsio/go-cti/f05a393a9e2faaa5e72a153583bc1d463a5393f0/fetcher/cwe/testdata/v5.0/cwec_latest.xml.zip -------------------------------------------------------------------------------- /fetcher/cwe/types.go: -------------------------------------------------------------------------------- 1 | package cwe 2 | 3 | type weaknessCatalog struct { 4 | Weaknesses struct { 5 | Weakness []struct { 6 | ID string `xml:"ID,attr"` 7 | RelatedAttackPatterns struct { 8 | RelatedAttackPattern []struct { 9 | CAPECID string `xml:"CAPEC_ID,attr"` 10 | } `xml:"Related_Attack_Pattern"` 11 | } `xml:"Related_Attack_Patterns"` 12 | } `xml:"Weakness"` 13 | } `xml:"Weaknesses"` 14 | } 15 | -------------------------------------------------------------------------------- /fetcher/nvd/nvd.go: -------------------------------------------------------------------------------- 1 | package nvd 2 | 3 | import ( 4 | "archive/tar" 5 | "context" 6 | "encoding/json" 7 | "io" 8 | "io/fs" 9 | "os" 10 | "os/exec" 11 | "path/filepath" 12 | "slices" 13 | "strings" 14 | "time" 15 | 16 | "github.com/inconshreveable/log15" 17 | "github.com/klauspost/compress/zstd" 18 | ocispec "github.com/opencontainers/image-spec/specs-go/v1" 19 | "golang.org/x/xerrors" 20 | "oras.land/oras-go/v2" 21 | "oras.land/oras-go/v2/registry/remote" 22 | ) 23 | 24 | // Fetch NVD CVE data 25 | func Fetch() (map[string][]string, error) { 26 | log15.Info("Fetching NVD CVE...") 27 | 28 | dir, err := os.MkdirTemp("", "go-cti") 29 | if err != nil { 30 | return nil, xerrors.Errorf("Failed to create temp directory. err: %w", err) 31 | } 32 | defer os.RemoveAll(dir) 33 | 34 | if err := fetch(dir); err != nil { 35 | return nil, xerrors.Errorf("Failed to fetch vuls-data-raw-nvd-api-cve. err: %w", err) 36 | } 37 | 38 | return parse(dir) 39 | } 40 | 41 | func fetch(dir string) error { 42 | ctx := context.TODO() 43 | repo, err := remote.NewRepository("ghcr.io/vulsio/vuls-data-db:vuls-data-raw-nvd-api-cve") 44 | if err != nil { 45 | return xerrors.Errorf("Failed to create client for ghcr.io/vulsio/vuls-data-db:vuls-data-raw-nvd-api-cve. err: %w", err) 46 | } 47 | 48 | _, r, err := oras.Fetch(ctx, repo, repo.Reference.Reference, oras.DefaultFetchOptions) 49 | if err != nil { 50 | return xerrors.Errorf("Failed to fetch manifest. err: %w", err) 51 | } 52 | defer r.Close() 53 | 54 | var manifest ocispec.Manifest 55 | if err := json.NewDecoder(r).Decode(&manifest); err != nil { 56 | return xerrors.Errorf("Failed to decode manifest. err: %w", err) 57 | } 58 | 59 | l := func() *ocispec.Descriptor { 60 | for _, l := range manifest.Layers { 61 | if l.MediaType == "application/vnd.vulsio.vuls-data-db.dotgit.layer.v1.tar+zstd" { 62 | return &l 63 | } 64 | } 65 | return nil 66 | }() 67 | if l == nil { 68 | return xerrors.Errorf("Failed to find digest and filename from layers, actual layers: %#v", manifest.Layers) 69 | } 70 | 71 | r, err = repo.Fetch(ctx, *l) 72 | if err != nil { 73 | return xerrors.Errorf("Failed to fetch content. err: %w", err) 74 | } 75 | defer r.Close() 76 | 77 | zr, err := zstd.NewReader(r) 78 | if err != nil { 79 | return xerrors.Errorf("Failed to new zstd reader. err: %w", err) 80 | } 81 | defer zr.Close() 82 | 83 | tr := tar.NewReader(zr) 84 | for { 85 | hdr, err := tr.Next() 86 | if err == io.EOF { 87 | break 88 | } 89 | if err != nil { 90 | return xerrors.Errorf("Failed to next tar reader. err: %w", err) 91 | } 92 | 93 | p := filepath.Join(dir, hdr.Name) 94 | 95 | switch hdr.Typeflag { 96 | case tar.TypeDir: 97 | if err := os.MkdirAll(p, 0755); err != nil { 98 | return xerrors.Errorf("Failed to mkdir %s. err: %w", p, err) 99 | } 100 | case tar.TypeReg: 101 | if err := os.MkdirAll(filepath.Dir(p), 0755); err != nil { 102 | return xerrors.Errorf("Failed to mkdir %s. err: %w", p, err) 103 | } 104 | 105 | if err := func() error { 106 | f, err := os.Create(p) 107 | if err != nil { 108 | return xerrors.Errorf("Failed to create %s. err: %w", p, err) 109 | } 110 | defer f.Close() 111 | 112 | if _, err := io.Copy(f, tr); err != nil { 113 | return xerrors.Errorf("Failed to copy to %s. err: %w", p, err) 114 | } 115 | 116 | return nil 117 | }(); err != nil { 118 | return xerrors.Errorf("Failed to create %s. err: %w", p, err) 119 | } 120 | } 121 | } 122 | 123 | cmd := exec.Command("git", "-C", filepath.Join(dir, "vuls-data-raw-nvd-api-cve"), "restore", ".") 124 | if err := cmd.Run(); err != nil { 125 | return xerrors.Errorf("Failed to exec %q. err: %w", cmd.String(), err) 126 | } 127 | 128 | return nil 129 | } 130 | 131 | func parse(dir string) (map[string][]string, error) { 132 | cveToCwes := make(map[string][]string) 133 | 134 | if err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { 135 | if err != nil { 136 | return err 137 | } 138 | 139 | if d.IsDir() || !strings.HasPrefix(filepath.Base(path), "CVE-") || filepath.Ext(path) != ".json" { 140 | return nil 141 | } 142 | 143 | ss := strings.Split(filepath.Base(path), "-") 144 | if len(ss) != 3 { 145 | return xerrors.Errorf("Failed to parse year. err: invalid ID format. expected: %q, actual: %q", "CVE-yyyy-\\d{4,}.json", filepath.Base(path)) 146 | } 147 | if _, err := time.Parse("2006", ss[1]); err != nil { 148 | return xerrors.Errorf("Failed to parse year. err: invalid ID format. expected: %q, actual: %q", "CVE-yyyy-\\d{4,}.json", filepath.Base(path)) 149 | } 150 | 151 | f, err := os.Open(path) 152 | if err != nil { 153 | return xerrors.Errorf("Failed to open %s. err: %w", path, err) 154 | } 155 | defer f.Close() 156 | 157 | var nvddata nvd 158 | if err := json.NewDecoder(f).Decode(&nvddata); err != nil { 159 | return xerrors.Errorf("Failed to decode JSON. err: %w", err) 160 | } 161 | 162 | for _, w := range nvddata.Weaknesses { 163 | for _, d := range w.Description { 164 | if strings.HasPrefix(d.Value, "CWE-") && !slices.Contains(cveToCwes[nvddata.ID], d.Value) { 165 | cveToCwes[nvddata.ID] = append(cveToCwes[nvddata.ID], d.Value) 166 | } 167 | } 168 | } 169 | 170 | return nil 171 | }); err != nil { 172 | return nil, xerrors.Errorf("Failed to walk %s. err: %w", dir, err) 173 | } 174 | 175 | return cveToCwes, nil 176 | } 177 | -------------------------------------------------------------------------------- /fetcher/nvd/nvd_test.go: -------------------------------------------------------------------------------- 1 | package nvd 2 | 3 | import ( 4 | "maps" 5 | "slices" 6 | "testing" 7 | ) 8 | 9 | func TestParse(t *testing.T) { 10 | tests := []struct { 11 | in string 12 | base map[string][]string 13 | expected map[string][]string 14 | }{ 15 | { 16 | in: "testdata/go-cti00001", 17 | base: map[string][]string{}, 18 | expected: map[string][]string{ 19 | "CVE-2020-0002": {"CWE-787", "CWE-416"}, 20 | }, 21 | }, 22 | } 23 | 24 | for i, tt := range tests { 25 | actual, err := parse(tt.in) 26 | if err != nil { 27 | t.Fatalf("[%d] Failed to parse. err: %s", i, err) 28 | } 29 | if !maps.EqualFunc(actual, tt.expected, func(v1 []string, v2 []string) bool { 30 | return slices.Equal(v1, v2) 31 | }) { 32 | t.Errorf("[%d] parse expected: %v, actual: %v\n", i, tt.expected, actual) 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /fetcher/nvd/testdata/go-cti00001/vuls-data-raw-nvd-api-cve/2020/CVE-2020-0002.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "CVE-2020-0002", 3 | "sourceIdentifier": "security@android.com", 4 | "vulnStatus": "Analyzed", 5 | "published": "2020-01-08T19:15:12.923", 6 | "lastModified": "2022-01-01T20:01:34.303", 7 | "descriptions": [ 8 | { 9 | "lang": "en", 10 | "value": "In ih264d_init_decoder of ih264d_api.c, there is a possible out of bounds write due to a use after free. This could lead to remote code execution with no additional execution privileges needed. User interaction is needed for exploitation Product: Android Versions: Android-8.0, Android-8.1, Android-9, and Android-10 Android ID: A-142602711" 11 | }, 12 | { 13 | "lang": "es", 14 | "value": "En la función ih264d_init_decoder del archivo ih264d_api.c, hay una posible escritura fuera de límites debido a un uso de la memoria previamente liberada. Esto podría conllevar a una ejecución de código remota sin ser necesarios privilegios de ejecución adicionales. Es requerida una interacción del usuario para su explotación Producto: Android, Versiones: Android-8.0, Android-8.1, Android-9 y Android-10, ID de Android: A-142602711." 15 | } 16 | ], 17 | "references": [ 18 | { 19 | "source": "security@android.com", 20 | "tags": [ 21 | "Patch", 22 | "Vendor Advisory" 23 | ], 24 | "url": "https://source.android.com/security/bulletin/2020-01-01" 25 | } 26 | ], 27 | "metrics": { 28 | "cvssMetricV2": [ 29 | { 30 | "source": "nvd@nist.gov", 31 | "type": "Primary", 32 | "cvssData": { 33 | "version": "2.0", 34 | "vectorString": "AV:N/AC:M/Au:N/C:C/I:C/A:C", 35 | "accessVector": "NETWORK", 36 | "accessComplexity": "MEDIUM", 37 | "authentication": "NONE", 38 | "confidentialityImpact": "COMPLETE", 39 | "integrityImpact": "COMPLETE", 40 | "availabilityImpact": "COMPLETE", 41 | "baseScore": 9.3 42 | }, 43 | "baseSeverity": "HIGH", 44 | "exploitabilityScore": 8.6, 45 | "impactScore": 10, 46 | "userInteractionRequired": true 47 | } 48 | ], 49 | "cvssMetricV31": [ 50 | { 51 | "source": "nvd@nist.gov", 52 | "type": "Primary", 53 | "cvssData": { 54 | "version": "3.1", 55 | "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:H/I:H/A:H", 56 | "attackVector": "NETWORK", 57 | "attackComplexity": "LOW", 58 | "privilegesRequired": "NONE", 59 | "userInteraction": "REQUIRED", 60 | "scope": "UNCHANGED", 61 | "confidentialityImpact": "HIGH", 62 | "integrityImpact": "HIGH", 63 | "availabilityImpact": "HIGH", 64 | "baseScore": 8.8, 65 | "baseSeverity": "HIGH" 66 | }, 67 | "exploitabilityScore": 2.8, 68 | "impactScore": 5.9 69 | } 70 | ] 71 | }, 72 | "weaknesses": [ 73 | { 74 | "source": "nvd@nist.gov", 75 | "type": "Primary", 76 | "description": [ 77 | { 78 | "lang": "en", 79 | "value": "CWE-787" 80 | }, 81 | { 82 | "lang": "en", 83 | "value": "CWE-416" 84 | } 85 | ] 86 | } 87 | ], 88 | "configurations": [ 89 | { 90 | "nodes": [ 91 | { 92 | "operator": "OR", 93 | "cpeMatch": [ 94 | { 95 | "vulnerable": true, 96 | "criteria": "cpe:2.3:o:google:android:8.0:*:*:*:*:*:*:*", 97 | "matchCriteriaId": "B578E383-0D77-4AC7-9C81-3F0B8C18E033" 98 | }, 99 | { 100 | "vulnerable": true, 101 | "criteria": "cpe:2.3:o:google:android:8.1:*:*:*:*:*:*:*", 102 | "matchCriteriaId": "B06BE74B-83F4-41A3-8AD3-2E6248F7B0B2" 103 | }, 104 | { 105 | "vulnerable": true, 106 | "criteria": "cpe:2.3:o:google:android:9.0:*:*:*:*:*:*:*", 107 | "matchCriteriaId": "8DFAAD08-36DA-4C95-8200-C29FE5B6B854" 108 | }, 109 | { 110 | "vulnerable": true, 111 | "criteria": "cpe:2.3:o:google:android:10.0:*:*:*:*:*:*:*", 112 | "matchCriteriaId": "D558D965-FA70-4822-A770-419E73BA9ED3" 113 | } 114 | ] 115 | } 116 | ] 117 | } 118 | ] 119 | } 120 | -------------------------------------------------------------------------------- /fetcher/nvd/types.go: -------------------------------------------------------------------------------- 1 | package nvd 2 | 3 | // https://github.com/MaineK00n/vuls-data-update/blob/38e5f8203f3ba90ce565e4a8eb650c17412ea88d/pkg/fetch/nvd/api/cve/types.go#L18 4 | type nvd struct { 5 | ID string `json:"id"` 6 | Weaknesses []struct { 7 | Source string `json:"source"` 8 | Type string `json:"type"` 9 | Description []struct { 10 | Lang string `json:"lang"` 11 | Value string `json:"value"` 12 | } `json:"description"` 13 | } `json:"weaknesses,omitempty"` 14 | } 15 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/vulsio/go-cti 2 | 3 | go 1.24 4 | 5 | require ( 6 | github.com/cheggaaa/pb/v3 v3.1.7 7 | github.com/glebarez/sqlite v1.11.0 8 | github.com/go-redis/redis/v8 v8.11.5 9 | github.com/google/go-cmp v0.7.0 10 | github.com/inconshreveable/log15 v3.0.0-testing.5+incompatible 11 | github.com/klauspost/compress v1.18.0 12 | github.com/labstack/echo/v4 v4.13.3 13 | github.com/mitchellh/go-homedir v1.1.0 14 | github.com/opencontainers/image-spec v1.1.1 15 | github.com/parnurzeal/gorequest v0.2.16 16 | github.com/spf13/cobra v1.9.1 17 | github.com/spf13/viper v1.20.0 18 | golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 19 | gorm.io/driver/mysql v1.5.5 20 | gorm.io/driver/postgres v1.5.7 21 | gorm.io/gorm v1.25.7 22 | oras.land/oras-go/v2 v2.5.0 23 | ) 24 | 25 | require ( 26 | github.com/VividCortex/ewma v1.2.0 // indirect 27 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 28 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 29 | github.com/dustin/go-humanize v1.0.1 // indirect 30 | github.com/elazarl/goproxy v0.0.0-20230731152917-f99041a5c027 // indirect 31 | github.com/fatih/color v1.18.0 // indirect 32 | github.com/fsnotify/fsnotify v1.8.0 // indirect 33 | github.com/glebarez/go-sqlite v1.21.2 // indirect 34 | github.com/go-sql-driver/mysql v1.7.1 // indirect 35 | github.com/go-stack/stack v1.8.1 // indirect 36 | github.com/go-viper/mapstructure/v2 v2.2.1 // indirect 37 | github.com/google/uuid v1.6.0 // indirect 38 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 39 | github.com/jackc/pgpassfile v1.0.0 // indirect 40 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect 41 | github.com/jackc/pgx/v5 v5.5.4 // indirect 42 | github.com/jackc/puddle/v2 v2.2.1 // indirect 43 | github.com/jinzhu/inflection v1.0.0 // indirect 44 | github.com/jinzhu/now v1.1.5 // indirect 45 | github.com/labstack/gommon v0.4.2 // indirect 46 | github.com/mattn/go-colorable v0.1.14 // indirect 47 | github.com/mattn/go-isatty v0.0.20 // indirect 48 | github.com/mattn/go-runewidth v0.0.16 // indirect 49 | github.com/opencontainers/go-digest v1.0.0 // indirect 50 | github.com/pelletier/go-toml/v2 v2.2.3 // indirect 51 | github.com/pkg/errors v0.9.1 // indirect 52 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect 53 | github.com/rivo/uniseg v0.4.7 // indirect 54 | github.com/sagikazarmark/locafero v0.7.0 // indirect 55 | github.com/smartystreets/goconvey v1.8.0 // indirect 56 | github.com/sourcegraph/conc v0.3.0 // indirect 57 | github.com/spf13/afero v1.12.0 // indirect 58 | github.com/spf13/cast v1.7.1 // indirect 59 | github.com/spf13/pflag v1.0.6 // indirect 60 | github.com/subosito/gotenv v1.6.0 // indirect 61 | github.com/valyala/bytebufferpool v1.0.0 // indirect 62 | github.com/valyala/fasttemplate v1.2.2 // indirect 63 | go.uber.org/atomic v1.9.0 // indirect 64 | go.uber.org/multierr v1.9.0 // indirect 65 | golang.org/x/crypto v0.35.0 // indirect 66 | golang.org/x/net v0.36.0 // indirect 67 | golang.org/x/sync v0.11.0 // indirect 68 | golang.org/x/sys v0.30.0 // indirect 69 | golang.org/x/term v0.29.0 // indirect 70 | golang.org/x/text v0.22.0 // indirect 71 | golang.org/x/time v0.8.0 // indirect 72 | gopkg.in/yaml.v3 v3.0.1 // indirect 73 | modernc.org/libc v1.22.5 // indirect 74 | modernc.org/mathutil v1.5.0 // indirect 75 | modernc.org/memory v1.5.0 // indirect 76 | modernc.org/sqlite v1.23.1 // indirect 77 | moul.io/http2curl v1.0.0 // indirect 78 | ) 79 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= 2 | github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= 3 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 4 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 5 | github.com/cheggaaa/pb/v3 v3.1.7 h1:2FsIW307kt7A/rz/ZI2lvPO+v3wKazzE4K/0LtTWsOI= 6 | github.com/cheggaaa/pb/v3 v3.1.7/go.mod h1:/Ji89zfVPeC/u5j8ukD0MBPHt2bzTYp74lQ7KlgFWTQ= 7 | github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= 8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 10 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 12 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 13 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 14 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 15 | github.com/elazarl/goproxy v0.0.0-20230731152917-f99041a5c027 h1:1L0aalTpPz7YlMxETKpmQoWMBkeiuorElZIXoNmgiPE= 16 | github.com/elazarl/goproxy v0.0.0-20230731152917-f99041a5c027/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= 17 | github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= 18 | github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= 19 | github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= 20 | github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= 21 | github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= 22 | github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= 23 | github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= 24 | github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= 25 | github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= 26 | github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= 27 | github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= 28 | github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= 29 | github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= 30 | github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= 31 | github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= 32 | github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= 33 | github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= 34 | github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= 35 | github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= 36 | github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= 37 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 38 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 39 | github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= 40 | github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= 41 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 42 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 43 | github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= 44 | github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= 45 | github.com/inconshreveable/log15 v3.0.0-testing.5+incompatible h1:VryeOTiaZfAzwx8xBcID1KlJCeoWSIpsNbSk+/D2LNk= 46 | github.com/inconshreveable/log15 v3.0.0-testing.5+incompatible/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= 47 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 48 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 49 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= 50 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= 51 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= 52 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= 53 | github.com/jackc/pgx/v5 v5.5.4 h1:Xp2aQS8uXButQdnCMWNmvx6UysWQQC+u1EoizjguY+8= 54 | github.com/jackc/pgx/v5 v5.5.4/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= 55 | github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= 56 | github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= 57 | github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= 58 | github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= 59 | github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= 60 | github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= 61 | github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= 62 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 63 | github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= 64 | github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= 65 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 66 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 67 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 68 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 69 | github.com/labstack/echo/v4 v4.13.3 h1:pwhpCPrTl5qry5HRdM5FwdXnhXSLSY+WE+YQSeCaafY= 70 | github.com/labstack/echo/v4 v4.13.3/go.mod h1:o90YNEeQWjDozo584l7AwhJMHN0bOC4tAfg+Xox9q5g= 71 | github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= 72 | github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= 73 | github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= 74 | github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= 75 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 76 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 77 | github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= 78 | github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 79 | github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= 80 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 81 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= 82 | github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= 83 | github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= 84 | github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= 85 | github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= 86 | github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= 87 | github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= 88 | github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= 89 | github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= 90 | github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= 91 | github.com/parnurzeal/gorequest v0.2.16 h1:T/5x+/4BT+nj+3eSknXmCTnEVGSzFzPGdpqmUVVZXHQ= 92 | github.com/parnurzeal/gorequest v0.2.16/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE= 93 | github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= 94 | github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= 95 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 96 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 97 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 98 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 99 | github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= 100 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= 101 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= 102 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 103 | github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= 104 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 105 | github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= 106 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= 107 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 108 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 109 | github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= 110 | github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= 111 | github.com/smartystreets/assertions v1.13.1 h1:Ef7KhSmjZcK6AVf9YbJdvPYG9avaF0ZxudX+ThRdWfU= 112 | github.com/smartystreets/assertions v1.13.1/go.mod h1:cXr/IwVfSo/RbCSPhoAPv73p3hlSdrBH/b3SdnW/LMY= 113 | github.com/smartystreets/goconvey v1.8.0 h1:Oi49ha/2MURE0WexF052Z0m+BNSGirfjg5RL+JXWq3w= 114 | github.com/smartystreets/goconvey v1.8.0/go.mod h1:EdX8jtrTIj26jmjCOVNMVSIYAtgexqXKHOXW2Dx9JLg= 115 | github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= 116 | github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= 117 | github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= 118 | github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= 119 | github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= 120 | github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= 121 | github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= 122 | github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= 123 | github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= 124 | github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 125 | github.com/spf13/viper v1.20.0 h1:zrxIyR3RQIOsarIrgL8+sAvALXul9jeEPa06Y0Ph6vY= 126 | github.com/spf13/viper v1.20.0/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= 127 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 128 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 129 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 130 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 131 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 132 | github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= 133 | github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= 134 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 135 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 136 | github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= 137 | github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= 138 | go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= 139 | go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 140 | go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= 141 | go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= 142 | golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= 143 | golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= 144 | golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= 145 | golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= 146 | golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= 147 | golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 148 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 149 | golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= 150 | golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 151 | golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= 152 | golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= 153 | golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= 154 | golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= 155 | golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= 156 | golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= 157 | golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= 158 | golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= 159 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 160 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 161 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 162 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 163 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 164 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 165 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 166 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 167 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 168 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 169 | gorm.io/driver/mysql v1.5.5 h1:WxklwX6FozMs1gk9yVadxGfjGiJjrBKPvIIvYZOMyws= 170 | gorm.io/driver/mysql v1.5.5/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= 171 | gorm.io/driver/postgres v1.5.7 h1:8ptbNJTDbEmhdr62uReG5BGkdQyeasu/FZHxI0IMGnM= 172 | gorm.io/driver/postgres v1.5.7/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA= 173 | gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A= 174 | gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= 175 | modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= 176 | modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= 177 | modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= 178 | modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= 179 | modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= 180 | modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= 181 | modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= 182 | modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= 183 | moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8= 184 | moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE= 185 | oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= 186 | oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= 187 | -------------------------------------------------------------------------------- /integration/.gitignore: -------------------------------------------------------------------------------- 1 | go-cti.* 2 | *.sqlite3 3 | diff -------------------------------------------------------------------------------- /integration/README.md: -------------------------------------------------------------------------------- 1 | # Test Script For go-cti 2 | Documentation on testing for developers 3 | 4 | ## Getting Started 5 | ```terminal 6 | $ pip install -r requirements.txt 7 | ``` 8 | 9 | ## Run test 10 | Use `127.0.0.1:1325` and `127.0.0.1:1326` to diff the server mode between the latest tag and your working branch. 11 | 12 | If you have prepared the two addresses yourself, you can use the following Python script. 13 | ```terminal 14 | $ python diff_server_mode.py --help 15 | usage: diff_server_mode.py [-h] [--sample_rate SAMPLE_RATE] [--debug | --no-debug] {cves,multi-cves} 16 | 17 | positional arguments: 18 | {cves,multi-cves} Specify the mode to test. 19 | 20 | optional arguments: 21 | -h, --help show this help message and exit 22 | --sample_rate SAMPLE_RATE 23 | Adjust the rate of data used for testing (len(test_data) * sample_rate) 24 | --debug, --no-debug print debug message 25 | ``` 26 | 27 | [GNUmakefile](../GNUmakefile) has some tasks for testing. 28 | Please run it in the top directory of the go-cti repository. 29 | 30 | - build-integration: create the go-cti binaries needed for testing 31 | - clean-integration: delete the go-cti process, binary, and docker container used in the test 32 | - fetch-rdb: fetch data for RDB for testing 33 | - fetch-redis: fetch data for Redis for testing 34 | - diff-cveid: Run tests for CVE ID in server mode 35 | - diff-server-rdb: take the result difference of server mode using RDB 36 | - diff-server-redis: take the result difference of server mode using Redis 37 | - diff-server-rdb-redis: take the difference in server mode results between RDB and Redis 38 | -------------------------------------------------------------------------------- /integration/diff_server_mode.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | from typing import Tuple 4 | from deepdiff import DeepDiff 5 | import requests 6 | from requests.adapters import HTTPAdapter 7 | from urllib3.util import Retry 8 | from urllib.parse import quote 9 | import pprint 10 | from concurrent.futures import ThreadPoolExecutor 11 | import os 12 | import random 13 | import math 14 | import json 15 | import shutil 16 | import time 17 | import uuid 18 | 19 | 20 | def diff_response(args: Tuple[str, list[str]]): 21 | session = requests.Session() 22 | retries = Retry(total=5, 23 | backoff_factor=1, 24 | status_forcelist=[503, 504]) 25 | session.mount("http://", HTTPAdapter(max_retries=retries)) 26 | 27 | # Endpoint 28 | # GET /cves/:id 29 | # POST /multi-cves 30 | path = "" 31 | if args[0] in ['cves']: 32 | path = f'{args[0]}/{args[1][0]}' 33 | try: 34 | response_old = requests.get( 35 | f'http://127.0.0.1:1325/{path}', timeout=(3.0, 10.0)).json() 36 | response_new = requests.get( 37 | f'http://127.0.0.1:1326/{path}', timeout=(3.0, 10.0)).json() 38 | except requests.ConnectionError as e: 39 | logger.error( 40 | f'Failed to Connection..., err: {e}, {pprint.pformat({"args": args, "path": path}, indent=2)}') 41 | exit(1) 42 | except requests.ReadTimeout as e: 43 | logger.warning( 44 | f'Failed to ReadTimeout..., err: {e}, {pprint.pformat({"args": args, "path": path}, indent=2)}') 45 | except Exception as e: 46 | logger.error( 47 | f'Failed to GET request..., err: {e}, {pprint.pformat({"args": args, "path": path}, indent=2)}') 48 | exit(1) 49 | 50 | diff = DeepDiff(response_old, response_new, ignore_order=True) 51 | if diff != {}: 52 | logger.warning( 53 | f'There is a difference between old and new(or RDB and Redis):\n {pprint.pformat({"args": args, "path": path}, indent=2)}') 54 | 55 | diff_path = f'integration/diff/{args[0]}/{args[1]}' 56 | with open(f'{diff_path}.old', 'w') as w: 57 | w.write(json.dumps(response_old, indent=4)) 58 | with open(f'{diff_path}.new', 'w') as w: 59 | w.write(json.dumps(response_new, indent=4)) 60 | else: 61 | path = args[0] 62 | 63 | k = math.ceil(len(args[1])/5) 64 | for _ in range(5): 65 | payload = {"args": random.sample(args[1], k)} 66 | try: 67 | response_old = session.post( 68 | f'http://127.0.0.1:1325/{path}', data=json.dumps(payload), headers={'content-type': 'application/json'}, timeout=(3.0, 10.0)).json() 69 | response_new = session.post( 70 | f'http://127.0.0.1:1326/{path}', data=json.dumps(payload), headers={'content-type': 'application/json'}, timeout=(3.0, 10.0)).json() 71 | except requests.ConnectionError as e: 72 | logger.error( 73 | f'Failed to Connection..., err: {e}, {pprint.pformat({"args": args, "path": path}, indent=2)}') 74 | exit(1) 75 | except requests.ReadTimeout as e: 76 | logger.warning( 77 | f'Failed to ReadTimeout..., err: {e}, {pprint.pformat({"args": args, "path": path}, indent=2)}') 78 | except Exception as e: 79 | logger.error( 80 | f'Failed to GET request..., err: {e}, {pprint.pformat({"args": args, "path": path}, indent=2)}') 81 | exit(1) 82 | 83 | diff = DeepDiff(response_old, response_new, ignore_order=True) 84 | if diff != {}: 85 | logger.warning( 86 | f'There is a difference between old and new(or RDB and Redis):\n {pprint.pformat({"args": args, "path": path}, indent=2)}') 87 | 88 | title = uuid.uuid4() 89 | diff_path = f'integration/diff/{args[0]}/{title}' 90 | with open(f'{diff_path}.old', 'w') as w: 91 | w.write(json.dumps( 92 | {'args': args, 'response': response_old}, indent=4)) 93 | with open(f'{diff_path}.new', 'w') as w: 94 | w.write(json.dumps( 95 | {'args': args, 'response': response_new}, indent=4)) 96 | 97 | 98 | parser = argparse.ArgumentParser() 99 | parser.add_argument('mode', choices=['cves', 'multi-cves'], 100 | help='Specify the mode to test.') 101 | parser.add_argument("--sample_rate", type=float, default=0.01, 102 | help="Adjust the rate of data used for testing (len(test_data) * sample_rate)") 103 | parser.add_argument( 104 | '--debug', action=argparse.BooleanOptionalAction, help='print debug message') 105 | args = parser.parse_args() 106 | 107 | logger = logging.getLogger(__name__) 108 | stream_handler = logging.StreamHandler() 109 | 110 | if args.debug: 111 | logger.setLevel(logging.DEBUG) 112 | stream_handler.setLevel(logging.DEBUG) 113 | else: 114 | logger.setLevel(logging.INFO) 115 | stream_handler.setLevel(logging.INFO) 116 | 117 | formatter = logging.Formatter( 118 | '%(levelname)s[%(asctime)s] %(message)s', "%m-%d|%H:%M:%S") 119 | stream_handler.setFormatter(formatter) 120 | logger.addHandler(stream_handler) 121 | 122 | logger.info( 123 | f'start server mode test(mode: {args.mode})') 124 | 125 | logger.info('check the communication with the server') 126 | for i in range(5): 127 | try: 128 | if requests.get('http://127.0.0.1:1325/health').status_code == requests.codes.ok and requests.get('http://127.0.0.1:1326/health').status_code == requests.codes.ok: 129 | logger.info('communication with the server has been confirmed') 130 | break 131 | except Exception: 132 | pass 133 | time.sleep(1) 134 | else: 135 | logger.error('Failed to communicate with server') 136 | exit(1) 137 | 138 | list_path = None 139 | if args.mode in ['cves', 'multi-cves']: 140 | list_path = f"integration/cveid.txt" 141 | if not os.path.isfile(list_path): 142 | logger.error(f'Failed to find list path..., list_path: {list_path}') 143 | exit(1) 144 | 145 | diff_path = f'integration/diff/{args.mode}' 146 | if os.path.exists(diff_path): 147 | shutil.rmtree(diff_path) 148 | os.makedirs(diff_path, exist_ok=True) 149 | 150 | with open(list_path) as f: 151 | list = [s.strip() for s in f.readlines()] 152 | list = random.sample(list, math.ceil(len(list) * args.sample_rate)) 153 | if args.mode in ['multi-cves']: 154 | diff_response((args.mode, list)) 155 | else: 156 | with ThreadPoolExecutor() as executor: 157 | ins = ((args.mode, [e]) for e in list) 158 | executor.map(diff_response, ins) 159 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/vulsio/go-cti/commands" 8 | ) 9 | 10 | func main() { 11 | if err := commands.RootCmd.Execute(); err != nil { 12 | fmt.Fprintln(os.Stderr, err) 13 | os.Exit(1) 14 | } 15 | os.Exit(0) 16 | } 17 | -------------------------------------------------------------------------------- /models/models.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "time" 5 | 6 | "gorm.io/gorm" 7 | ) 8 | 9 | // LatestSchemaVersion manages the Schema version used in the latest go-cti. 10 | const LatestSchemaVersion = 1 11 | 12 | // FetchMeta has meta information 13 | type FetchMeta struct { 14 | gorm.Model `json:"-"` 15 | GoCTIRevision string 16 | SchemaVersion uint 17 | LastFetchedAt time.Time 18 | } 19 | 20 | // OutDated checks whether last fetched feed is out dated 21 | func (f FetchMeta) OutDated() bool { 22 | return f.SchemaVersion != LatestSchemaVersion 23 | } 24 | 25 | // CTIType : 26 | type CTIType string 27 | 28 | // TechniqueSourceType : 29 | type TechniqueSourceType string 30 | 31 | // MitreAttackerType : 32 | type MitreAttackerType string 33 | 34 | // AttackSoftwareType : 35 | type AttackSoftwareType string 36 | 37 | var ( 38 | // TechniqueType : 39 | TechniqueType CTIType = "Technique" 40 | // AttackerType : 41 | AttackerType CTIType = "Attacker" 42 | 43 | // MitreAttackType : 44 | MitreAttackType TechniqueSourceType = "MITRE-ATTACK" 45 | // CAPECType : 46 | CAPECType TechniqueSourceType = "CAPEC" 47 | 48 | // GroupType : 49 | GroupType MitreAttackerType = "Group" 50 | // SoftwareType : 51 | SoftwareType MitreAttackerType = "Software" 52 | // CampaignType : 53 | CampaignType MitreAttackerType = "Campaign" 54 | 55 | // MalwareType : 56 | MalwareType AttackSoftwareType = "Malware" 57 | // ToolType : 58 | ToolType AttackSoftwareType = "Tool" 59 | ) 60 | 61 | // CveToTechniques : 62 | type CveToTechniques struct { 63 | ID int64 `json:"-"` 64 | CveID string `gorm:"type:varchar(255);index:idx_mapping_cve_id" json:"cve_id"` 65 | TechniqueIDs []CveToTechniqueID `json:"technique_ids"` 66 | } 67 | 68 | // CveToTechniqueID : 69 | type CveToTechniqueID struct { 70 | ID int64 `json:"-"` 71 | CveToTechniquesID int64 `json:"-"` 72 | TechniqueID string `gorm:"type:varchar(255)" json:"technique_id"` 73 | } 74 | 75 | // Technique : Cyber Threat Intelligence 76 | type Technique struct { 77 | ID int64 `json:"-"` 78 | TechniqueID string `gorm:"type:varchar(255)" json:"technique_id"` 79 | Type TechniqueSourceType `gorm:"type:varchar(255)" json:"type"` 80 | Name string `gorm:"type:varchar(255)" json:"name"` 81 | Description string `gorm:"type:text" json:"description"` 82 | References []TechniqueReference `json:"references"` 83 | Mitigations []Mitigation `json:"mitigations"` 84 | MitreAttack *MitreAttack `json:"mitre_attack"` 85 | Capec *Capec `json:"capec"` 86 | Created time.Time `json:"created"` 87 | Modified time.Time `json:"modified"` 88 | } 89 | 90 | // Reference is Child model of Technique 91 | type Reference struct { 92 | SourceName string `gorm:"type:varchar(255)" json:"source_name"` 93 | Description string `gorm:"type:text" json:"description"` 94 | URL string `gorm:"type:text" json:"url"` 95 | } 96 | 97 | // TechniqueReference is Child model of Technique 98 | type TechniqueReference struct { 99 | ID int64 `json:"-"` 100 | TechniqueID int64 `gorm:"index:idx_technique_reference_technique_id" json:"-"` 101 | Reference `gorm:"embedded"` 102 | } 103 | 104 | // Mitigation is Child model of Technique 105 | type Mitigation struct { 106 | ID int64 `json:"-"` 107 | TechniqueID int64 `gorm:"index:idx_mitigation_technique_id" json:"-"` 108 | Name string `gorm:"type:text" json:"name"` 109 | Description string `gorm:"type:text" json:"description"` 110 | } 111 | 112 | // MitreAttack is Child model of Technique 113 | type MitreAttack struct { 114 | ID int64 `json:"-"` 115 | TechniqueID int64 `gorm:"index:idx_mitre_attack_technique_id" json:"-"` 116 | CapecIDs []CapecID `json:"capec_ids"` 117 | Detection string `gorm:"type:text" json:"detection"` 118 | KillChainPhases []KillChainPhase `json:"kill_chain_phases"` 119 | DataSources []DataSource `json:"data_sources"` 120 | Procedures []Procedure `json:"procedures"` 121 | Platforms []TechniquePlatform `json:"platforms"` 122 | PermissionsRequired []PermissionRequired `json:"permissions_required"` 123 | EffectivePermissions []EffectivePermission `json:"effective_permissions"` 124 | DefenseBypassed []DefenseBypassed `json:"defense_bypassed"` 125 | ImpactType []ImpactType `json:"impact_type"` 126 | NetworkRequirements bool `json:"network_requirements"` 127 | RemoteSupport bool `json:"remote_support"` 128 | SubTechniques []SubTechnique `json:"sub_techniques"` 129 | } 130 | 131 | // CapecID is Child model of MitreAttack 132 | type CapecID struct { 133 | ID int64 `json:"-"` 134 | MitreAttackID int64 `gorm:"index:idx_capec_id_mitre_attack_id" json:"-"` 135 | CapecID string `gorm:"type:varchar(255)" json:"capec_id"` 136 | } 137 | 138 | // KillChainPhase is Child model of MitreAttack 139 | type KillChainPhase struct { 140 | ID int64 `json:"-"` 141 | MitreAttackID int64 `gorm:"index:idx_kill_chain_phase_mitre_attack_id" json:"-"` 142 | Tactic string `gorm:"type:varchar(255)" json:"tactic"` 143 | } 144 | 145 | // DataSource is Child model of MitreAttack 146 | type DataSource struct { 147 | ID int64 `json:"-"` 148 | MitreAttackID int64 `gorm:"index:idx_data_source_mitre_attack_id" json:"-"` 149 | Name string `gorm:"type:varchar(255)" json:"name"` 150 | Description string `gorm:"type:text" json:"description"` 151 | } 152 | 153 | // Procedure is Child model of MitreAttack 154 | type Procedure struct { 155 | ID int64 `json:"-"` 156 | MitreAttackID int64 `gorm:"index:idx_procedure_mitre_attack_id" json:"-"` 157 | Name string `gorm:"type:varchar(255)" json:"name"` 158 | Description string `gorm:"type:text" json:"description"` 159 | } 160 | 161 | // TechniquePlatform is Child model of MitreAttack 162 | type TechniquePlatform struct { 163 | ID int64 `json:"-"` 164 | MitreAttackID int64 `gorm:"index:idx_technique_platform_mitre_attack_id" json:"-"` 165 | Platform string `gorm:"type:varchar(255)" json:"platform"` 166 | } 167 | 168 | // PermissionRequired is Child model of MitreAttack 169 | type PermissionRequired struct { 170 | ID int64 `json:"-"` 171 | MitreAttackID int64 `gorm:"index:idx_permission_required_mitre_attack_id" json:"-"` 172 | Permission string `gorm:"type:varchar(255)" json:"permission"` 173 | } 174 | 175 | // EffectivePermission is Child model of MitreAttack 176 | type EffectivePermission struct { 177 | ID int64 `json:"-"` 178 | MitreAttackID int64 `gorm:"index:idx_effective_permission_mitre_attack_id" json:"-"` 179 | Permission string `gorm:"type:varchar(255)" json:"permission"` 180 | } 181 | 182 | // DefenseBypassed is Child model of MitreAttack 183 | type DefenseBypassed struct { 184 | ID int64 `json:"-"` 185 | MitreAttackID int64 `gorm:"index:idx_defense_bypassed_mitre_attack_id" json:"-"` 186 | Defense string `gorm:"type:varchar(255)" json:"defense"` 187 | } 188 | 189 | // ImpactType is Child model of MitreAttack 190 | type ImpactType struct { 191 | ID int64 `json:"-"` 192 | MitreAttackID int64 `gorm:"index:idx_impact_type_mitre_attack_id" json:"-"` 193 | Type string `gorm:"type:varchar(255)" json:"type"` 194 | } 195 | 196 | // SubTechnique is Child model of MitreAttack 197 | type SubTechnique struct { 198 | ID int64 `json:"-"` 199 | MitreAttackID int64 `gorm:"index:idx_sub_technique_mitre_attack_id" json:"-"` 200 | Name string `gorm:"type:varchar(255)" json:"name"` 201 | } 202 | 203 | // Capec is Child model of Technique 204 | type Capec struct { 205 | ID int64 `json:"-"` 206 | TechniqueID int64 `gorm:"index:idx_capec_technique_id" json:"-"` 207 | AttackIDs []AttackID `json:"attack_ids"` 208 | Status string `gorm:"type:varchar(255)" json:"status"` 209 | ExtendedDescription string `gorm:"type:text" json:"extended_description"` 210 | TypicalSeverity string `gorm:"type:varchar(255)" json:"typical_severity"` 211 | LikelihoodOfAttack string `gorm:"type:varchar(255)" json:"likelihood_of_attack"` 212 | Relationships []Relationship `json:"relationship"` 213 | Domains []Domain `json:"domains"` 214 | AlternateTerms []AlternateTerm `json:"alternate_terms"` 215 | ExampleInstances []ExampleInstance `json:"example_instances"` 216 | Prerequisites []Prerequisite `json:"prerequisites"` 217 | ResourcesRequired []ResourceRequired `json:"resources_required"` 218 | SkillsRequired []SkillRequired `json:"skills_required"` 219 | Abstraction string `gorm:"type:varchar(255)" json:"abstraction"` 220 | ExecutionFlow string `gorm:"type:text" json:"execution_flow"` 221 | Consequences []Consequence `json:"consequences"` 222 | RelatedWeaknesses []RelatedWeakness `json:"related_weaknesses"` 223 | } 224 | 225 | // AttackID is Child model of Capec 226 | type AttackID struct { 227 | ID int64 `json:"-"` 228 | CapecID int64 `gorm:"index:idx_attack_id_capec_id" json:"-"` 229 | AttackID string `gorm:"type:varchar(255)" json:"capec_id"` 230 | } 231 | 232 | // Relationship is Child model of Capec 233 | type Relationship struct { 234 | ID int64 `json:"-"` 235 | CapecID int64 `gorm:"index:idx_relationship_capec_id" json:"-"` 236 | Nature string `gorm:"type:varchar(255)" json:"nature"` 237 | Relation string `gorm:"type:varchar(255)" json:"relation"` 238 | } 239 | 240 | // Domain is Child model of Capec 241 | type Domain struct { 242 | ID int64 `json:"-"` 243 | CapecID int64 `gorm:"index:idx_domain_capec_id" json:"-"` 244 | Domain string `gorm:"type:varchar(255)" json:"domain"` 245 | } 246 | 247 | // AlternateTerm is Child model of Capec 248 | type AlternateTerm struct { 249 | ID int64 `json:"-"` 250 | CapecID int64 `gorm:"index:idx_alternate_term_capec_id" json:"-"` 251 | Term string `gorm:"type:varchar(255)" json:"term"` 252 | } 253 | 254 | // ExampleInstance is Child model of Capec 255 | type ExampleInstance struct { 256 | ID int64 `json:"-"` 257 | CapecID int64 `gorm:"index:idx_example_instance_capec_id" json:"-"` 258 | Instance string `gorm:"type:text" json:"instance"` 259 | } 260 | 261 | // Prerequisite is Child model of Capec 262 | type Prerequisite struct { 263 | ID int64 `json:"-"` 264 | CapecID int64 `gorm:"index:idx_prerequisite_capec_id" json:"-"` 265 | Prerequisite string `gorm:"type:text" json:"prerequisite"` 266 | } 267 | 268 | // ResourceRequired is Child model of Capec 269 | type ResourceRequired struct { 270 | ID int64 `json:"-"` 271 | CapecID int64 `gorm:"index:idx_resource_required_capec_id" json:"-"` 272 | Resource string `gorm:"type:text" json:"prerequisite"` 273 | } 274 | 275 | // SkillRequired is Child model of Capec 276 | type SkillRequired struct { 277 | ID int64 `json:"-"` 278 | CapecID int64 `gorm:"index:idx_skill_required_capec_id" json:"-"` 279 | Skill string `gorm:"type:text" json:"skill"` 280 | } 281 | 282 | // Consequence is Child model of Capec 283 | type Consequence struct { 284 | ID int64 `json:"-"` 285 | CapecID int64 `gorm:"index:idx_consequence_capec_id" json:"-"` 286 | Consequence string `gorm:"type:text" json:"consequence"` 287 | } 288 | 289 | // RelatedWeakness is Child model of Capec 290 | type RelatedWeakness struct { 291 | ID int64 `json:"-"` 292 | CapecID int64 `gorm:"index:idx_related_weakness_capec_id" json:"-"` 293 | CweID string `gorm:"type:varchar(255)" json:"cwe_id"` 294 | } 295 | 296 | // Attacker : MITRE ATT&CK Group and Software 297 | type Attacker struct { 298 | ID int64 `json:"-"` 299 | AttackerID string `gorm:"type:varchar(255)" json:"attacker_id"` 300 | Type MitreAttackerType `gorm:"type:varchar(255)" json:"type"` 301 | Name string `gorm:"type:varchar(255)" json:"name"` 302 | Description string `gorm:"type:text" json:"description"` 303 | TechniquesUsed []TechniqueUsed `json:"techniques_used"` 304 | References []AttackerReference `json:"references"` 305 | Group *AttackerGroup `json:"group"` 306 | Software *AttackerSoftware `json:"software"` 307 | // Campaign *AttackerCampaign `json:"campaign"` 308 | Created time.Time `json:"created"` 309 | Modified time.Time `json:"modified"` 310 | } 311 | 312 | // TechniqueUsed is Child model of Attacker 313 | type TechniqueUsed struct { 314 | ID int64 `json:"-"` 315 | AttackerID int64 `gorm:"index:idx_technique_used_attacker_id" json:"-"` 316 | TechniqueID string `gorm:"type:varchar(255)" json:"technique_id"` 317 | Name string `gorm:"type:varchar(255)" json:"name"` 318 | Use string `gorm:"type:text" json:"use"` 319 | } 320 | 321 | // AttackerReference is Child model of Attacker 322 | type AttackerReference struct { 323 | ID int64 `json:"-"` 324 | AttackerID int64 `gorm:"index:idx_attacker_reference_attacker_id" json:"-"` 325 | Reference `gorm:"embedded"` 326 | } 327 | 328 | // AttackerGroup is Child model of Attacker 329 | type AttackerGroup struct { 330 | ID int64 `json:"-"` 331 | AttackerID int64 `gorm:"index:idx_attacker_group_attacker_id" json:"-"` 332 | AssociatedGroups []AssociatedGroup `json:"associated_group"` 333 | SoftwaresUsed []SoftwareUsed `json:"softwares_used"` 334 | } 335 | 336 | // AssociatedGroup is Child models of Group 337 | type AssociatedGroup struct { 338 | ID int64 `json:"-"` 339 | AttackerGroupID int64 `gorm:"index:idx_associated_group_attacker_group_id" json:"-"` 340 | Name string `gorm:"type:varchar(255)" json:"name"` 341 | Description string `gorm:"type:text" json:"description"` 342 | } 343 | 344 | // SoftwareUsed is Child models of Group 345 | type SoftwareUsed struct { 346 | ID int64 `json:"-"` 347 | AttackerGroupID int64 `gorm:"index:idx_software_used_attacker_group_id" json:"-"` 348 | Name string `gorm:"type:varchar(255)" json:"name"` 349 | Description string `gorm:"type:text" json:"description"` 350 | } 351 | 352 | // AttackerSoftware is Child model of Attacker 353 | type AttackerSoftware struct { 354 | ID int64 `json:"-"` 355 | AttackerID int64 `gorm:"index:idx_attacker_software_attacker_id" json:"-"` 356 | Type AttackSoftwareType `gorm:"type:varchar(255)" json:"type"` 357 | AssociatedSoftwares []AssociatedSoftware `json:"associated_softwares"` 358 | Platforms []SoftwarePlatform `json:"platforms"` 359 | GroupsUsed []GroupUsed `json:"groups_used"` 360 | } 361 | 362 | // AssociatedSoftware is Child models of Software 363 | type AssociatedSoftware struct { 364 | ID int64 `json:"-"` 365 | AttackerSoftwareID int64 `gorm:"index:idx_associated_software_attacker_software_id" json:"-"` 366 | Name string `gorm:"type:varchar(255)" json:"name"` 367 | Description string `gorm:"type:text" json:"description"` 368 | } 369 | 370 | // SoftwarePlatform is Child models of Software 371 | type SoftwarePlatform struct { 372 | ID int64 `json:"-"` 373 | AttackerSoftwareID int64 `gorm:"index:idx_software_platform_attacker_software_id" json:"-"` 374 | Platform string `gorm:"type:varchar(255)" json:"platform"` 375 | } 376 | 377 | // GroupUsed is Child models of Software 378 | type GroupUsed struct { 379 | ID int64 `json:"-"` 380 | AttackerSoftwareID int64 `gorm:"index:idx_group_used_attacker_software_id" json:"-"` 381 | Name string `gorm:"type:varchar(255)" json:"name"` 382 | Description string `gorm:"type:text" json:"description"` 383 | } 384 | 385 | // type AttackerCampaign struct { 386 | // ID int64 `json:"-"` 387 | // AttackerID int64 `gorm:"index:idx_attacker_campaign_attacker_id" json:"-"` 388 | // Softwares []AttackerCampaignSoftware `json:"softwares"` 389 | // Groups []AttackerCampaignGroup `json:"groups"` 390 | // } 391 | 392 | // type AttackerCampaignSoftware struct { 393 | // ID int64 `json:"-"` 394 | // AttackerCampaignID int64 `gorm:"index:idx_attacker_campaign_software_attacker_campaign_id" json:"-"` 395 | // Name string `gorm:"type:varchar(255)" json:"name"` 396 | // Description string `gorm:"type:text" json:"description"` 397 | // } 398 | 399 | // type AttackerCampaignGroup struct { 400 | // ID int64 `json:"-"` 401 | // AttackerCampaignID int64 `gorm:"index:idx_attacker_campaign_group_attacker_campaign_id" json:"-"` 402 | // Name string `gorm:"type:varchar(255)" json:"name"` 403 | // Description string `gorm:"type:text" json:"description"` 404 | // } 405 | 406 | // CTI for response 407 | type CTI struct { 408 | Type CTIType `json:"type"` 409 | Technique *Technique `json:"technique,omitempty"` 410 | Attacker *Attacker `json:"attacker,omitempty"` 411 | } 412 | -------------------------------------------------------------------------------- /models/models_test.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func Test_FetchMeta(t *testing.T) { 8 | var tests = []struct { 9 | in FetchMeta 10 | outdated bool 11 | }{ 12 | { 13 | in: FetchMeta{ 14 | SchemaVersion: 0, 15 | }, 16 | outdated: true, 17 | }, 18 | { 19 | in: FetchMeta{ 20 | SchemaVersion: LatestSchemaVersion, 21 | }, 22 | outdated: false, 23 | }, 24 | } 25 | 26 | for i, tt := range tests { 27 | if aout := tt.in.OutDated(); tt.outdated != aout { 28 | t.Errorf("[%d] outdated expected: %#v\n actual: %#v\n", i, tt.outdated, aout) 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/inconshreveable/log15" 10 | "github.com/labstack/echo/v4" 11 | "github.com/labstack/echo/v4/middleware" 12 | "github.com/spf13/viper" 13 | "golang.org/x/xerrors" 14 | 15 | "github.com/vulsio/go-cti/db" 16 | ) 17 | 18 | // Start : 19 | func Start(logToFile bool, logDir string, driver db.DB) error { 20 | e := echo.New() 21 | e.Debug = viper.GetBool("debug") 22 | 23 | // Middleware 24 | e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{Output: os.Stderr})) 25 | e.Use(middleware.Recover()) 26 | 27 | // setup access logger 28 | if logToFile { 29 | logPath := filepath.Join(logDir, "access.log") 30 | f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) 31 | if err != nil { 32 | return xerrors.Errorf("Failed to open a log file: %s", err) 33 | } 34 | defer f.Close() 35 | e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{Output: f})) 36 | } 37 | 38 | // Routes 39 | e.GET("/health", health()) 40 | e.GET("/ctis/:cti", getCtiByCtiID(driver)) 41 | e.POST("/multi-ctis", getCtiByMultiCtiID(driver)) 42 | e.GET("/cves/:cve", getTechniqueIDsByCveID(driver)) 43 | e.POST("/multi-cves", getTechniqueIDsByMultiCveID(driver)) 44 | e.POST("/attackers", getAttackerIDsByTechniqueIDs(driver)) 45 | 46 | bindURL := fmt.Sprintf("%s:%s", viper.GetString("bind"), viper.GetString("port")) 47 | log15.Info("Listening...", "URL", bindURL) 48 | 49 | return e.Start(bindURL) 50 | } 51 | 52 | func health() echo.HandlerFunc { 53 | return func(context echo.Context) error { 54 | return context.String(http.StatusOK, "") 55 | } 56 | } 57 | 58 | type param struct { 59 | Args []string `json:"args"` 60 | } 61 | 62 | func getCtiByCtiID(driver db.DB) echo.HandlerFunc { 63 | return func(context echo.Context) (err error) { 64 | ctiID := context.Param("cti") 65 | log15.Debug("Params", "CTI-ID", ctiID) 66 | 67 | cti, err := driver.GetCtiByCtiID(ctiID) 68 | if err != nil { 69 | return xerrors.Errorf("Failed to get CTI by CTI-ID. err: %w", err) 70 | } 71 | return context.JSON(http.StatusOK, cti) 72 | } 73 | } 74 | 75 | func getCtiByMultiCtiID(driver db.DB) echo.HandlerFunc { 76 | return func(context echo.Context) (err error) { 77 | ctiIDs := param{} 78 | if err := context.Bind(&ctiIDs); err != nil { 79 | return err 80 | } 81 | log15.Debug("Params", "CTI-IDs", ctiIDs.Args) 82 | 83 | ctis, err := driver.GetCtisByMultiCtiID(ctiIDs.Args) 84 | if err != nil { 85 | return xerrors.Errorf("Failed to get CTIs by CTI-IDs. err: %w", err) 86 | } 87 | return context.JSON(http.StatusOK, ctis) 88 | } 89 | } 90 | 91 | func getTechniqueIDsByCveID(driver db.DB) echo.HandlerFunc { 92 | return func(context echo.Context) (err error) { 93 | cve := context.Param("cve") 94 | log15.Debug("Params", "CVE-ID", cve) 95 | 96 | ids, err := driver.GetTechniqueIDsByCveID(cve) 97 | if err != nil { 98 | return xerrors.Errorf("Failed to get TechniqueIDs by CVE-ID. err: %w", err) 99 | } 100 | return context.JSON(http.StatusOK, ids) 101 | } 102 | } 103 | 104 | func getTechniqueIDsByMultiCveID(driver db.DB) echo.HandlerFunc { 105 | return func(context echo.Context) (err error) { 106 | cveIDs := param{} 107 | if err := context.Bind(&cveIDs); err != nil { 108 | return err 109 | } 110 | log15.Debug("Params", "CVE-IDs", cveIDs.Args) 111 | 112 | vulns, err := driver.GetTechniqueIDsByMultiCveID(cveIDs.Args) 113 | if err != nil { 114 | return xerrors.Errorf("Failed to get TechniqueIDs by CVE-IDs. err: %w", err) 115 | } 116 | return context.JSON(http.StatusOK, vulns) 117 | } 118 | } 119 | 120 | func getAttackerIDsByTechniqueIDs(driver db.DB) echo.HandlerFunc { 121 | return func(context echo.Context) (err error) { 122 | techIDs := param{} 123 | if err := context.Bind(&techIDs); err != nil { 124 | return err 125 | } 126 | log15.Debug("Params", "TechniqueIDs", techIDs.Args) 127 | 128 | attackerIDs, err := driver.GetAttackerIDsByTechniqueIDs(techIDs.Args) 129 | if err != nil { 130 | return xerrors.Errorf("Failed to get AttackerIDs by TechniqueIDs. err: %w", err) 131 | } 132 | return context.JSON(http.StatusOK, attackerIDs) 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "runtime" 7 | 8 | "github.com/inconshreveable/log15" 9 | "github.com/parnurzeal/gorequest" 10 | "github.com/spf13/viper" 11 | "golang.org/x/xerrors" 12 | ) 13 | 14 | // CacheDir : 15 | func CacheDir() string { 16 | tmpDir, err := os.UserCacheDir() 17 | if err != nil { 18 | tmpDir = os.TempDir() 19 | } 20 | return filepath.Join(tmpDir, "go-cti") 21 | } 22 | 23 | // GetDefaultLogDir : 24 | func GetDefaultLogDir() string { 25 | defaultLogDir := "/var/log/go-cti" 26 | if runtime.GOOS == "windows" { 27 | defaultLogDir = filepath.Join(os.Getenv("APPDATA"), "go-cti") 28 | } 29 | return defaultLogDir 30 | } 31 | 32 | // SetLogger : 33 | func SetLogger(logToFile bool, logDir string, debug, logJSON bool) error { 34 | stderrHandler := log15.StderrHandler 35 | logFormat := log15.LogfmtFormat() 36 | if logJSON { 37 | logFormat = log15.JsonFormatEx(false, true) 38 | stderrHandler = log15.StreamHandler(os.Stderr, logFormat) 39 | } 40 | 41 | lvlHandler := log15.LvlFilterHandler(log15.LvlInfo, stderrHandler) 42 | if debug { 43 | lvlHandler = log15.LvlFilterHandler(log15.LvlDebug, stderrHandler) 44 | } 45 | 46 | var handler log15.Handler 47 | if logToFile { 48 | if _, err := os.Stat(logDir); err != nil { 49 | if os.IsNotExist(err) { 50 | if err := os.Mkdir(logDir, 0700); err != nil { 51 | return xerrors.Errorf("Failed to create log directory. err: %w", err) 52 | } 53 | } else { 54 | return xerrors.Errorf("Failed to check log directory. err: %w", err) 55 | } 56 | } 57 | 58 | logPath := filepath.Join(logDir, "go-cti.log") 59 | if _, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644); err != nil { 60 | return xerrors.Errorf("Failed to open a log file. err: %w", err) 61 | } 62 | handler = log15.MultiHandler( 63 | log15.Must.FileHandler(logPath, logFormat), 64 | lvlHandler, 65 | ) 66 | } else { 67 | handler = lvlHandler 68 | } 69 | log15.Root().SetHandler(handler) 70 | return nil 71 | } 72 | 73 | // FetchURL returns HTTP response body 74 | func FetchURL(url string) ([]byte, error) { 75 | httpProxy := viper.GetString("http-proxy") 76 | 77 | resp, body, err := gorequest.New().Proxy(httpProxy).Get(url).Type("text").EndBytes() 78 | if len(err) > 0 || resp == nil || resp.StatusCode != 200 { 79 | return nil, xerrors.Errorf("HTTP error. url: %s, err: %w", url, err) 80 | } 81 | return body, nil 82 | } 83 | --------------------------------------------------------------------------------