├── .github ├── ISSUE_TEMPLATE │ ├── BUG_REPORT.md │ ├── FEATURE_REQUEST.md │ └── SUPPORT_QUESTION.md ├── PULL_REQUEST_TEMPLATE.md ├── dependabot.yml └── workflows │ ├── docker-build.yml │ ├── fetch.yml │ ├── golangci.yml │ ├── goreleaser.yml │ └── test.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── .revive.toml ├── Dockerfile ├── GNUmakefile ├── LICENSE ├── README.md ├── commands ├── fetch-kevuln.go ├── fetch-vulncheck.go ├── fetch.go ├── root.go ├── server.go └── version.go ├── config └── config.go ├── db ├── common_test.go ├── db.go ├── rdb.go ├── rdb_test.go ├── redis.go └── redis_test.go ├── fetcher ├── kevuln │ ├── kevuln.go │ └── types.go └── vulncheck │ ├── types.go │ └── vulncheck.go ├── go.mod ├── go.sum ├── integration ├── .gitignore ├── README.md ├── cveid.txt └── diff_server_mode.py ├── main.go ├── models ├── models.go └── models_test.go ├── server └── server.go └── utils └── utils.go /.github/ISSUE_TEMPLATE/BUG_REPORT.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | labels: bug 4 | about: If something isn't working as expected. 5 | --- 6 | 7 | # What did you do? (required. The issue will be **closed** when not provided.) 8 | 9 | 10 | # What did you expect to happen? 11 | 12 | 13 | # What happened instead? 14 | 15 | * Current Output 16 | 17 | Please re-run the command using ```-debug``` and provide the output below. 18 | 19 | # Steps to reproduce the behaviour 20 | 21 | 22 | # Configuration (**MUST** fill this out): 23 | 24 | * Go version (`go version`): 25 | 26 | * Go environment (`go env`): 27 | 28 | * go-kev environment: 29 | 30 | Hash : ____ 31 | 32 | To check the commit hash of HEAD 33 | $ go-kev version 34 | 35 | or 36 | 37 | $ cd $GOPATH/src/github.com/vulsio/go-kev 38 | $ git rev-parse --short HEAD 39 | 40 | * command: 41 | 42 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | labels: enhancement 4 | about: I have a suggestion (and might want to implement myself)! 5 | --- 6 | 7 | 10 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/SUPPORT_QUESTION.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Support Question 3 | labels: question 4 | about: If you have a question about Vuls. 5 | --- 6 | 7 | 11 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | If this Pull Request is work in progress, Add a prefix of “[WIP]” in the title. 3 | 4 | # What did you implement: 5 | 6 | Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. 7 | 8 | Fixes # (issue) 9 | 10 | ## Type of change 11 | 12 | Please delete options that are not relevant. 13 | 14 | - [ ] Bug fix (non-breaking change which fixes an issue) 15 | - [ ] New feature (non-breaking change which adds functionality) 16 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 17 | - [ ] This change requires a documentation update 18 | 19 | # How Has This Been Tested? 20 | 21 | Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. 22 | 23 | # Checklist: 24 | You don't have to satisfy all of the following. 25 | 26 | - [ ] Write tests 27 | - [ ] Write documentation 28 | - [ ] Check that there aren't other open pull requests for the same issue/feature 29 | - [ ] Format your source code by `make fmt` 30 | - [ ] Pass the test by `make test` 31 | - [ ] Provide verification config / commands 32 | - [ ] Enable "Allow edits from maintainers" for this PR 33 | - [ ] Update the messages below 34 | 35 | ***Is this ready for review?:*** NO 36 | 37 | # Reference 38 | 39 | * https://blog.github.com/2015-01-21-how-to-write-the-perfect-pull-request/ 40 | 41 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "weekly" 12 | target-branch: "main" 13 | - package-ecosystem: "github-actions" # See documentation for possible values 14 | directory: "/" # Location of package manifests 15 | schedule: 16 | interval: "weekly" 17 | target-branch: "main" 18 | -------------------------------------------------------------------------------- /.github/workflows/docker-build.yml: -------------------------------------------------------------------------------- 1 | name: Publish Docker image 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'main' 7 | tags: 8 | - '*' 9 | 10 | jobs: 11 | docker: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Checkout 15 | uses: actions/checkout@v4 16 | 17 | - name: Set up QEMU 18 | uses: docker/setup-qemu-action@v3 19 | 20 | - name: Set up Docker Buildx 21 | uses: docker/setup-buildx-action@v3 22 | 23 | - name: Login to DockerHub 24 | uses: docker/login-action@v3 25 | with: 26 | username: ${{ secrets.DOCKERHUB_USERNAME }} 27 | password: ${{ secrets.DOCKERHUB_TOKEN }} 28 | 29 | - 30 | name: Docker meta 31 | id: meta 32 | uses: docker/metadata-action@v5 33 | with: 34 | images: vuls/go-kev 35 | tags: | 36 | type=ref,event=tag 37 | 38 | - name: Build and push 39 | uses: docker/build-push-action@v6 40 | with: 41 | push: true 42 | context: . 43 | tags: | 44 | vuls/go-kev:latest 45 | ${{ steps.meta.outputs.tags }} 46 | secrets: | 47 | "github_token=${{ secrets.GITHUB_TOKEN }}" 48 | platforms: linux/amd64,linux/arm64 49 | -------------------------------------------------------------------------------- /.github/workflows/fetch.yml: -------------------------------------------------------------------------------- 1 | name: Fetch Test 2 | 3 | on: 4 | pull_request: 5 | schedule: 6 | - cron: '0 0 * * *' 7 | 8 | jobs: 9 | fetch-kevuln: 10 | name: fetch-kevuln 11 | runs-on: ubuntu-latest 12 | services: 13 | mysql: 14 | image: mysql 15 | ports: 16 | - 3306:3306 17 | env: 18 | MYSQL_ROOT_PASSWORD: password 19 | MYSQL_DATABASE: test 20 | options: >- 21 | --health-cmd "mysqladmin ping" 22 | --health-interval 10s 23 | --health-timeout 5s 24 | --health-retries 5 25 | postgres: 26 | image: postgres 27 | ports: 28 | - 5432:5432 29 | env: 30 | POSTGRES_PASSWORD: password 31 | POSTGRES_DB: test 32 | options: >- 33 | --health-cmd pg_isready 34 | --health-interval 10s 35 | --health-timeout 5s 36 | --health-retries 5 37 | redis: 38 | image: redis 39 | ports: 40 | - 6379:6379 41 | options: >- 42 | --health-cmd "redis-cli ping" 43 | --health-interval 10s 44 | --health-timeout 5s 45 | --health-retries 5 46 | steps: 47 | - name: Check out code into the Go module directory 48 | uses: actions/checkout@v4 49 | - name: Set up Go 50 | uses: actions/setup-go@v5 51 | with: 52 | go-version-file: go.mod 53 | - name: build 54 | id: build 55 | run: make build 56 | - name: fetch sqlite3 57 | if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} 58 | run: ./go-kev fetch --dbtype sqlite3 kevuln 59 | - name: fetch mysql 60 | if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} 61 | run: ./go-kev fetch --dbtype mysql --dbpath "root:password@tcp(127.0.0.1:3306)/test?parseTime=true" kevuln 62 | - name: fetch postgres 63 | if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} 64 | run: ./go-kev fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" kevuln 65 | - name: fetch redis 66 | if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} 67 | run: ./go-kev fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" kevuln 68 | 69 | fetch-vulncheck: 70 | name: fetch-vulncheck 71 | runs-on: ubuntu-latest 72 | services: 73 | mysql: 74 | image: mysql 75 | ports: 76 | - 3306:3306 77 | env: 78 | MYSQL_ROOT_PASSWORD: password 79 | MYSQL_DATABASE: test 80 | options: >- 81 | --health-cmd "mysqladmin ping" 82 | --health-interval 10s 83 | --health-timeout 5s 84 | --health-retries 5 85 | postgres: 86 | image: postgres 87 | ports: 88 | - 5432:5432 89 | env: 90 | POSTGRES_PASSWORD: password 91 | POSTGRES_DB: test 92 | options: >- 93 | --health-cmd pg_isready 94 | --health-interval 10s 95 | --health-timeout 5s 96 | --health-retries 5 97 | redis: 98 | image: redis 99 | ports: 100 | - 6379:6379 101 | options: >- 102 | --health-cmd "redis-cli ping" 103 | --health-interval 10s 104 | --health-timeout 5s 105 | --health-retries 5 106 | steps: 107 | - name: Check out code into the Go module directory 108 | uses: actions/checkout@v4 109 | - name: Set up Go 110 | uses: actions/setup-go@v5 111 | with: 112 | go-version-file: go.mod 113 | - name: build 114 | id: build 115 | run: make build 116 | - name: fetch sqlite3 117 | if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} 118 | run: ./go-kev fetch --dbtype sqlite3 vulncheck 119 | - name: fetch mysql 120 | if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} 121 | run: ./go-kev fetch --dbtype mysql --dbpath "root:password@tcp(127.0.0.1:3306)/test?parseTime=true" vulncheck 122 | - name: fetch postgres 123 | if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} 124 | run: ./go-kev fetch --dbtype postgres --dbpath "host=127.0.0.1 user=postgres dbname=test sslmode=disable password=password" vulncheck 125 | - name: fetch redis 126 | if: ${{ steps.build.conclusion == 'success' && ( success() || failure() )}} 127 | run: ./go-kev fetch --dbtype redis --dbpath "redis://127.0.0.1:6379/0" vulncheck 128 | -------------------------------------------------------------------------------- /.github/workflows/golangci.yml: -------------------------------------------------------------------------------- 1 | name: golangci-lint 2 | on: 3 | push: 4 | tags: 5 | - v* 6 | branches: 7 | - main 8 | pull_request: 9 | jobs: 10 | golangci: 11 | name: lint 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Check out code into the Go module directory 15 | uses: actions/checkout@v4 16 | - name: Set up Go 1.x 17 | uses: actions/setup-go@v5 18 | with: 19 | go-version-file: go.mod 20 | - name: golangci-lint 21 | uses: golangci/golangci-lint-action@v7 22 | with: 23 | # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version 24 | version: v2.0.2 25 | 26 | # Optional: working directory, useful for monorepos 27 | # working-directory: somedir 28 | 29 | # Optional: golangci-lint command line arguments. 30 | # args: --issues-exit-code=0 31 | 32 | # Optional: show only new issues if it's a pull request. The default value is `false`. 33 | # only-new-issues: true 34 | -------------------------------------------------------------------------------- /.github/workflows/goreleaser.yml: -------------------------------------------------------------------------------- 1 | name: goreleaser 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | goreleaser: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - 13 | name: Checkout 14 | uses: actions/checkout@v4 15 | - 16 | name: Unshallow 17 | run: git fetch --prune --unshallow 18 | - 19 | name: Set up Go 20 | uses: actions/setup-go@v5 21 | with: 22 | go-version-file: go.mod 23 | - 24 | name: Run GoReleaser 25 | uses: goreleaser/goreleaser-action@v6 26 | with: 27 | distribution: goreleaser 28 | version: latest 29 | args: release --clean 30 | env: 31 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 32 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: [pull_request] 4 | 5 | jobs: 6 | build: 7 | name: Build 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Check out code into the Go module directory 11 | uses: actions/checkout@v4 12 | - name: Set up Go 1.x 13 | uses: actions/setup-go@v5 14 | with: 15 | go-version-file: go.mod 16 | - name: Test 17 | run: make test 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | vendor/ 16 | 17 | *.sqlite3 18 | go-kev 19 | .vscode 20 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | 3 | linters: 4 | default: none 5 | enable: 6 | - errcheck 7 | - govet 8 | - ineffassign 9 | - misspell 10 | - prealloc 11 | - revive 12 | - staticcheck 13 | settings: 14 | revive: # https://golangci-lint.run/usage/linters/#revive 15 | rules: 16 | - name: blank-imports 17 | - name: context-as-argument 18 | - name: context-keys-type 19 | - name: dot-imports 20 | - name: empty-block 21 | - name: error-naming 22 | - name: error-return 23 | - name: error-strings 24 | - name: errorf 25 | - name: exported 26 | - name: if-return 27 | - name: increment-decrement 28 | - name: indent-error-flow 29 | - name: package-comments 30 | disabled: true 31 | - name: range 32 | - name: receiver-naming 33 | - name: redefines-builtin-id 34 | - name: superfluous-else 35 | - name: time-naming 36 | - name: unexported-return 37 | - name: unreachable-code 38 | - name: unused-parameter 39 | - name: var-declaration 40 | - name: var-naming 41 | staticcheck: # https://golangci-lint.run/usage/linters/#staticcheck 42 | checks: 43 | - all 44 | - -ST1000 # at least one file in a package should have a package comment 45 | - -ST1005 # error strings should not be capitalized 46 | exclusions: 47 | rules: 48 | - source: "defer .+\\.Close\\(\\)" 49 | linters: 50 | - errcheck 51 | - source: "defer os.RemoveAll\\(.+\\)" 52 | linters: 53 | - errcheck 54 | 55 | formatters: 56 | enable: 57 | - goimports 58 | 59 | run: 60 | timeout: 10m 61 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: go-kev 2 | release: 3 | github: 4 | owner: vulsio 5 | name: go-kev 6 | env: 7 | - CGO_ENABLED=0 8 | builds: 9 | - id: go-kev 10 | goos: 11 | - linux 12 | - windows 13 | - darwin 14 | goarch: 15 | - amd64 16 | - arm64 17 | main: . 18 | ldflags: -s -w -X github.com/vulsio/go-kev/config.Version={{.Version}} -X github.com/vulsio/go-kev/config.Revision={{.Commit}} 19 | binary: go-kev 20 | archives: 21 | - name_template: '{{ .Binary }}_{{.Version}}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}' 22 | format: tar.gz 23 | files: 24 | - LICENSE 25 | - README* 26 | snapshot: 27 | name_template: SNAPSHOT-{{ .Commit }} 28 | -------------------------------------------------------------------------------- /.revive.toml: -------------------------------------------------------------------------------- 1 | ignoreGeneratedHeader = false 2 | severity = "warning" 3 | confidence = 0.8 4 | errorCode = 0 5 | warningCode = 0 6 | 7 | [rule.blank-imports] 8 | [rule.context-as-argument] 9 | [rule.context-keys-type] 10 | [rule.dot-imports] 11 | [rule.error-return] 12 | [rule.error-strings] 13 | [rule.error-naming] 14 | [rule.exported] 15 | [rule.if-return] 16 | [rule.increment-decrement] 17 | [rule.var-naming] 18 | [rule.var-declaration] 19 | [rule.package-comments] 20 | [rule.range] 21 | [rule.receiver-naming] 22 | [rule.time-naming] 23 | [rule.unexported-return] 24 | [rule.indent-error-flow] 25 | [rule.errorf] 26 | [rule.empty-block] 27 | [rule.superfluous-else] 28 | [rule.unused-parameter] 29 | [rule.unreachable-code] 30 | [rule.redefines-builtin-id] -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:alpine as builder 2 | 3 | RUN apk add --no-cache \ 4 | git \ 5 | make \ 6 | gcc \ 7 | musl-dev 8 | 9 | ENV REPOSITORY github.com/vulsio/go-kev 10 | COPY . $GOPATH/src/$REPOSITORY 11 | RUN cd $GOPATH/src/$REPOSITORY && make install 12 | 13 | 14 | FROM alpine:3.14 15 | 16 | ENV LOGDIR /var/log/go-kev 17 | ENV WORKDIR /go-kev 18 | 19 | RUN apk add --no-cache ca-certificates git \ 20 | && mkdir -p $WORKDIR $LOGDIR 21 | 22 | COPY --from=builder /go/bin/go-kev /usr/local/bin/ 23 | 24 | VOLUME ["$WORKDIR", "$LOGDIR"] 25 | WORKDIR $WORKDIR 26 | ENV PWD $WORKDIR 27 | 28 | ENTRYPOINT ["go-kev"] 29 | CMD ["help"] -------------------------------------------------------------------------------- /GNUmakefile: -------------------------------------------------------------------------------- 1 | .PHONY: \ 2 | all \ 3 | build \ 4 | install \ 5 | vendor \ 6 | lint \ 7 | golangci \ 8 | vet \ 9 | fmt \ 10 | fmtcheck \ 11 | pretest \ 12 | test \ 13 | integration \ 14 | cov \ 15 | clean \ 16 | build-integration \ 17 | clean-integration \ 18 | fetch-rdb \ 19 | fetch-redis \ 20 | diff-cveid \ 21 | diff-package \ 22 | diff-server-rdb \ 23 | diff-server-redis \ 24 | diff-server-rdb-redis 25 | 26 | SRCS = $(shell git ls-files '*.go') 27 | PKGS = $(shell go list ./...) 28 | VERSION := $(shell git describe --tags --abbrev=0) 29 | REVISION := $(shell git rev-parse --short HEAD) 30 | BUILDTIME := $(shell date "+%Y%m%d_%H%M%S") 31 | LDFLAGS := -X 'github.com/vulsio/go-kev/config.Version=$(VERSION)' \ 32 | -X 'github.com/vulsio/go-kev/config.Revision=$(REVISION)' 33 | GO := CGO_ENABLED=0 go 34 | 35 | all: build test 36 | 37 | build: main.go 38 | $(GO) build -ldflags "$(LDFLAGS)" -o go-kev $< 39 | 40 | install: main.go 41 | $(GO) install -ldflags "$(LDFLAGS)" 42 | 43 | lint: 44 | go install github.com/mgechev/revive@latest 45 | revive -config ./.revive.toml -formatter plain $(PKGS) 46 | 47 | golangci: 48 | go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest 49 | golangci-lint run 50 | 51 | vet: 52 | echo $(PKGS) | xargs env $(GO) vet || exit; 53 | 54 | fmt: 55 | gofmt -w $(SRCS) 56 | 57 | fmtcheck: 58 | $(foreach file,$(SRCS),gofmt -d $(file);) 59 | 60 | pretest: lint vet fmtcheck 61 | 62 | test: pretest 63 | $(GO) test -cover -v ./... || exit; 64 | 65 | cov: 66 | @ go get -v github.com/axw/gocov/gocov 67 | @ go get golang.org/x/tools/cmd/cover 68 | gocov test | gocov report 69 | 70 | clean: 71 | $(foreach pkg,$(PKGS),go clean $(pkg) || exit;) 72 | 73 | BRANCH := $(shell git symbolic-ref --short HEAD) 74 | build-integration: 75 | @ git stash save 76 | $(GO) build -ldflags "$(LDFLAGS)" -o integration/go-kev.new 77 | git checkout $(shell git describe --tags --abbrev=0) 78 | @git reset --hard 79 | $(GO) build -ldflags "$(LDFLAGS)" -o integration/go-kev.old 80 | git checkout $(BRANCH) 81 | -@ git stash apply stash@{0} && git stash drop stash@{0} 82 | 83 | clean-integration: 84 | -pkill go-kev.old 85 | -pkill go-kev.new 86 | -rm integration/go-kev.old integration/go-kev.new integration/go-kev.old.sqlite3 integration/go-kev.new.sqlite3 87 | -rm -rf integration/diff 88 | -docker kill redis-old redis-new 89 | -docker rm redis-old redis-new 90 | 91 | fetch-rdb: 92 | integration/go-kev.old fetch kevuln --dbpath=integration/go-kev.old.sqlite3 93 | integration/go-kev.new fetch kevuln --dbpath=integration/go-kev.new.sqlite3 94 | 95 | fetch-redis: 96 | docker run --name redis-old -d -p 127.0.0.1:6379:6379 redis 97 | docker run --name redis-new -d -p 127.0.0.1:6380:6379 redis 98 | 99 | integration/go-kev.old fetch kevuln --dbtype redis --dbpath "redis://127.0.0.1:6379/0" 100 | integration/go-kev.new fetch kevuln --dbtype redis --dbpath "redis://127.0.0.1:6380/0" 101 | 102 | diff-cves: 103 | @ python integration/diff_server_mode.py cves --sample_rate 0.01 104 | @ python integration/diff_server_mode.py multi-cves --sample_rate 0.01 105 | 106 | diff-server-rdb: 107 | integration/go-kev.old server --dbpath=integration/go-kev.old.sqlite3 --port 1325 > /dev/null 2>&1 & 108 | integration/go-kev.new server --dbpath=integration/go-kev.new.sqlite3 --port 1326 > /dev/null 2>&1 & 109 | make diff-cves 110 | pkill go-kev.old 111 | pkill go-kev.new 112 | 113 | diff-server-redis: 114 | integration/go-kev.old server --dbtype redis --dbpath "redis://127.0.0.1:6379/0" --port 1325 > /dev/null 2>&1 & 115 | integration/go-kev.new server --dbtype redis --dbpath "redis://127.0.0.1:6380/0" --port 1326 > /dev/null 2>&1 & 116 | make diff-cves 117 | pkill go-kev.old 118 | pkill go-kev.new 119 | 120 | diff-server-rdb-redis: 121 | integration/go-kev.new server --dbpath=integration/go-kev.new.sqlite3 --port 1325 > /dev/null 2>&1 & 122 | integration/go-kev.new server --dbtype redis --dbpath "redis://127.0.0.1:6380/0" --port 1326 > /dev/null 2>&1 & 123 | make diff-cves 124 | pkill go-kev.new -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 vulsio 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-kev 2 | `go-kev` build a local copy of Known Exploited Vulnerabilities Catalog by CISA. 3 | 4 | # Usage 5 | ```console 6 | $ go-kev help 7 | Go Known Exploited Vulnerabilities 8 | 9 | Usage: 10 | go-kev [command] 11 | 12 | Available Commands: 13 | completion generate the autocompletion script for the specified shell 14 | fetch Fetch the data of vulnerabilities 15 | help Help about any command 16 | server Start go-kev HTTP server 17 | version Show version 18 | 19 | Flags: 20 | --config string config file (default is $HOME/.go-kev.yaml) 21 | --dbpath string /path/to/sqlite3 or SQL connection string 22 | --dbtype string Database type to store data in (sqlite3, mysql, postgres or redis supported) 23 | --debug debug mode (default: false) 24 | --debug-sql SQL debug mode 25 | -h, --help help for go-kev 26 | --http-proxy string http://proxy-url:port (default: empty) 27 | --log-dir string /path/to/log 28 | --log-json output log as JSON 29 | --log-to-file output log to file 30 | --quiet quiet mode (no output) 31 | 32 | Use "go-kev [command] --help" for more information about a command. 33 | ``` 34 | 35 | # Fetch CISA Known Exploited Vulnerabilities 36 | ```console 37 | $ go-kev fetch kevuln 38 | INFO[11-16|04:39:00] Fetching Known Exploited Vulnerabilities 39 | INFO[11-16|04:39:00] Fetching URL=https://www.cisa.gov/sites/default/files/csv/known_exploited_vulnerabilities.csv 40 | INFO[11-16|04:39:00] Insert Known Exploited Vulnerabilities into go-kev. db=sqlite3 41 | INFO[11-16|04:39:00] Inserting Known Exploited Vulnerabilities... 42 | 291 / 291 [------------------------------------------------------------------------------] 100.00% ? p/s 43 | INFO[11-16|04:39:00] CveID Count count=291 44 | ``` 45 | 46 | # Fetch VulnCheck Known Exploited Vulnerabilities (https://vulncheck.com/kev) 47 | Before you use this data from VulnCheck, you MUST read https://docs.vulncheck.com/community/vulncheck-kev/attribution and make sure it's satisfied. 48 | 49 | ```console 50 | $ go-kev fetch vulncheck 51 | INFO[08-23|02:34:55] Fetching VulnCheck Known Exploited Vulnerabilities 52 | INFO[08-23|02:34:56] Insert VulnCheck Known Exploited Vulnerabilities into go-kev. db=sqlite3 53 | INFO[08-23|02:34:56] Inserting VulnCheck Known Exploited Vulnerabilities... 54 | 2832 / 2832 [------------------------------------------------------------------------------] 100.00% 2931 p/s 55 | INFO[08-23|02:34:57] CveID Count count=2832 56 | ``` 57 | 58 | # Server mode 59 | ```console 60 | $ go-kev server 61 | INFO[11-16|04:40:28] Starting HTTP Server... 62 | INFO[11-16|04:40:28] Listening... URL=127.0.0.1:1328 63 | 64 | ____ __ 65 | / __/___/ / ___ 66 | / _// __/ _ \/ _ \ 67 | /___/\__/_//_/\___/ v3.3.10-dev 68 | High performance, minimalist Go web framework 69 | https://echo.labstack.com 70 | ____________________________________O/_______ 71 | O\ 72 | ⇨ http server started on 127.0.0.1:1328 73 | {"time":"2021-11-16T04:40:30.511368993+09:00","id":"","remote_ip":"127.0.0.1","host":"127.0.0.1:1328","method":"GET","uri":"/cves/CVE-2021-27104​","user_agent":"curl/7.68.0","status":200,"error":"","latency":5870905,"latency_human":"5.870905ms","bytes_in":0,"bytes_out":397} 74 | 75 | $ curl http://127.0.0.1:1328/cves/CVE-2021-27104 | jq 76 | { 77 | "cisa": [ 78 | { 79 | "cveID": "CVE-2021-27104", 80 | "vendorProject": "Accellion", 81 | "product": "FTA", 82 | "vulnerabilityName": "Accellion FTA OS Command Injection Vulnerability", 83 | "dateAdded": "2021-11-03T00:00:00Z", 84 | "shortDescription": "Accellion FTA contains an OS command injection vulnerability exploited via a crafted POST request to various admin endpoints.", 85 | "requiredAction": "Apply updates per vendor instructions.", 86 | "dueDate": "2021-11-17T00:00:00Z", 87 | "knownRansomwareCampaignUse": "Known", 88 | "notes": "" 89 | } 90 | ], 91 | "vulncheck": [ 92 | { 93 | "vendorProject": "Accellion", 94 | "product": "FTA", 95 | "shortDescription": "Accellion FTA contains an OS command injection vulnerability exploited via a crafted POST request to various admin endpoints.", 96 | "vulnerabilityName": "Accellion FTA OS Command Injection Vulnerability", 97 | "required_action": "Apply updates per vendor instructions.", 98 | "knownRansomwareCampaignUse": "Known", 99 | "cve": [ 100 | { 101 | "cveID": "CVE-2021-27104" 102 | } 103 | ], 104 | "vulncheck_xdb": [], 105 | "vulncheck_reported_exploitation": [ 106 | { 107 | "url": "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", 108 | "date_added": "2021-11-03T00:00:00Z" 109 | }, 110 | { 111 | "url": "https://unit42.paloaltonetworks.com/clop-ransomware/", 112 | "date_added": "2021-04-13T00:00:00Z" 113 | }, 114 | { 115 | "url": "https://www.trendmicro.com/vinfo/us/security/news/cybercrime-and-digital-threats/ransomware-double-extortion-and-beyond-revil-clop-and-conti", 116 | "date_added": "2021-06-15T00:00:00Z" 117 | }, 118 | { 119 | "url": "https://cybersecurityworks.com/howdymanage/uploads/file/ransomware-_-2022-spotlight-report_compressed.pdf", 120 | "date_added": "2022-01-26T00:00:00Z" 121 | }, 122 | { 123 | "url": "https://www.paloaltonetworks.com/content/dam/pan/en_US/assets/pdf/reports/2022-unit42-ransomware-threat-report-final.pdf", 124 | "date_added": "2022-03-24T00:00:00Z" 125 | }, 126 | { 127 | "url": "https://static.tenable.com/marketing/whitepapers/Whitepaper-Ransomware_Ecosystem.pdf", 128 | "date_added": "2022-06-22T00:00:00Z" 129 | }, 130 | { 131 | "url": "https://www.group-ib.com/resources/research-hub/hi-tech-crime-trends-2022/", 132 | "date_added": "2023-01-17T00:00:00Z" 133 | }, 134 | { 135 | "url": "https://fourcore.io/blogs/clop-ransomware-history-adversary-simulation", 136 | "date_added": "2023-06-03T00:00:00Z" 137 | }, 138 | { 139 | "url": "https://blog.talosintelligence.com/talos-ir-q2-2023-quarterly-recap/", 140 | "date_added": "2023-07-26T00:00:00Z" 141 | }, 142 | { 143 | "url": "https://www.sentinelone.com/resources/watchtower-end-of-year-report-2023/", 144 | "date_added": "2021-11-03T00:00:00Z" 145 | }, 146 | { 147 | "url": "https://www.trustwave.com/en-us/resources/blogs/trustwave-blog/defending-the-energy-sector-against-cyber-threats-insights-from-trustwave-spiderlabs/", 148 | "date_added": "2024-05-15T00:00:00Z" 149 | }, 150 | { 151 | "url": "https://cisa.gov/news-events/cybersecurity-advisories/aa21-055a", 152 | "date_added": "2021-06-17T00:00:00Z" 153 | }, 154 | { 155 | "url": "https://www.cisa.gov/news-events/cybersecurity-advisories/aa21-209a", 156 | "date_added": "2021-08-20T00:00:00Z" 157 | }, 158 | { 159 | "url": "https://cisa.gov/news-events/alerts/2022/04/27/2021-top-routinely-exploited-vulnerabilities", 160 | "date_added": "2022-04-28T00:00:00Z" 161 | }, 162 | { 163 | "url": "https://cisa.gov/news-events/cybersecurity-advisories/aa22-117a", 164 | "date_added": "2022-04-28T00:00:00Z" 165 | }, 166 | { 167 | "url": "https://www.hhs.gov/sites/default/files/threat-profile-june-2023.pdf", 168 | "date_added": "2023-06-13T00:00:00Z" 169 | } 170 | ], 171 | "dueDate": "2021-11-17T00:00:00Z", 172 | "cisa_date_added": "2021-11-03T00:00:00Z", 173 | "date_added": "2021-04-13T00:00:00Z" 174 | } 175 | ] 176 | } 177 | ``` 178 | 179 | # License 180 | MIT 181 | 182 | # Author 183 | [MaineK00n](https://twitter.com/MaineK00n) -------------------------------------------------------------------------------- /commands/fetch-kevuln.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "errors" 5 | "time" 6 | 7 | "github.com/inconshreveable/log15" 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | "golang.org/x/xerrors" 11 | 12 | "github.com/vulsio/go-kev/db" 13 | fetcher "github.com/vulsio/go-kev/fetcher/kevuln" 14 | "github.com/vulsio/go-kev/models" 15 | "github.com/vulsio/go-kev/utils" 16 | ) 17 | 18 | var fetchCatalogCmd = &cobra.Command{ 19 | Use: "kevuln", 20 | Short: "Fetch the data of known exploited vulnerabilities catalog by CISA", 21 | Long: `Fetch the data of known exploited vulnerabilities catalog by CISA`, 22 | RunE: fetchKEVuln, 23 | } 24 | 25 | func init() { 26 | fetchCmd.AddCommand(fetchCatalogCmd) 27 | } 28 | 29 | func fetchKEVuln(_ *cobra.Command, _ []string) (err error) { 30 | if err := utils.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil { 31 | return xerrors.Errorf("Failed to SetLogger. err: %w", err) 32 | } 33 | 34 | driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{}) 35 | if err != nil { 36 | if errors.Is(err, db.ErrDBLocked) { 37 | return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err) 38 | } 39 | return xerrors.Errorf("Failed to open DB. err: %w", err) 40 | } 41 | 42 | fetchMeta, err := driver.GetFetchMeta() 43 | if err != nil { 44 | return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err) 45 | } 46 | if fetchMeta.OutDated() { 47 | return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion}) 48 | } 49 | // If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion. 50 | if err := driver.UpsertFetchMeta(fetchMeta); err != nil { 51 | return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err) 52 | } 53 | 54 | log15.Info("Fetching Known Exploited Vulnerabilities") 55 | var vulns []models.KEVuln 56 | if vulns, err = fetcher.Fetch(); err != nil { 57 | return xerrors.Errorf("Failed to fetch Known Exploited Vulnerabilities. err: %w", err) 58 | } 59 | 60 | log15.Info("Insert Known Exploited Vulnerabilities into go-kev.", "db", driver.Name()) 61 | if err := driver.InsertKEVulns(vulns); err != nil { 62 | return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err) 63 | } 64 | 65 | fetchMeta.LastFetchedAt = time.Now() 66 | if err := driver.UpsertFetchMeta(fetchMeta); err != nil { 67 | return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err) 68 | } 69 | 70 | return nil 71 | } 72 | -------------------------------------------------------------------------------- /commands/fetch-vulncheck.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "errors" 5 | "time" 6 | 7 | "github.com/inconshreveable/log15" 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | "golang.org/x/xerrors" 11 | 12 | "github.com/vulsio/go-kev/db" 13 | fetcher "github.com/vulsio/go-kev/fetcher/vulncheck" 14 | "github.com/vulsio/go-kev/models" 15 | "github.com/vulsio/go-kev/utils" 16 | ) 17 | 18 | var fetchVulnCheckCmd = &cobra.Command{ 19 | Use: "vulncheck ", 20 | Short: "Fetch the data of known exploited vulnerabilities catalog by VulnCheck (https://vulncheck.com/kev)", 21 | Long: `Fetch the data of known exploited vulnerabilities catalog by VulnCheck (https://vulncheck.com/kev)`, 22 | Args: cobra.NoArgs, 23 | RunE: fetchVulnCheck, 24 | } 25 | 26 | func init() { 27 | fetchCmd.AddCommand(fetchVulnCheckCmd) 28 | } 29 | 30 | func fetchVulnCheck(_ *cobra.Command, _ []string) (err error) { 31 | if err := utils.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil { 32 | return xerrors.Errorf("Failed to SetLogger. err: %w", err) 33 | } 34 | 35 | driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{}) 36 | if err != nil { 37 | if errors.Is(err, db.ErrDBLocked) { 38 | return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err) 39 | } 40 | return xerrors.Errorf("Failed to open DB. err: %w", err) 41 | } 42 | 43 | fetchMeta, err := driver.GetFetchMeta() 44 | if err != nil { 45 | return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err) 46 | } 47 | if fetchMeta.OutDated() { 48 | return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion}) 49 | } 50 | // If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion. 51 | if err := driver.UpsertFetchMeta(fetchMeta); err != nil { 52 | return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err) 53 | } 54 | 55 | log15.Info("Fetching VulnCheck Known Exploited Vulnerabilities") 56 | var vulns []models.VulnCheck 57 | if vulns, err = fetcher.Fetch(); err != nil { 58 | return xerrors.Errorf("Failed to fetch VulnCheck Known Exploited Vulnerabilities. err: %w", err) 59 | } 60 | 61 | log15.Info("Insert VulnCheck Known Exploited Vulnerabilities into go-kev.", "db", driver.Name()) 62 | if err := driver.InsertVulnCheck(vulns); err != nil { 63 | return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err) 64 | } 65 | 66 | fetchMeta.LastFetchedAt = time.Now() 67 | if err := driver.UpsertFetchMeta(fetchMeta); err != nil { 68 | return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err) 69 | } 70 | 71 | return nil 72 | } 73 | -------------------------------------------------------------------------------- /commands/fetch.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | "github.com/spf13/viper" 6 | ) 7 | 8 | var fetchCmd = &cobra.Command{ 9 | Use: "fetch", 10 | Short: "Fetch the data of vulnerabilities", 11 | Long: `Fetch the data of vulnerabilities`, 12 | } 13 | 14 | func init() { 15 | RootCmd.AddCommand(fetchCmd) 16 | 17 | fetchCmd.PersistentFlags().Int("batch-size", 50, "The number of batch size to insert.") 18 | _ = viper.BindPFlag("batch-size", fetchCmd.PersistentFlags().Lookup("batch-size")) 19 | } 20 | -------------------------------------------------------------------------------- /commands/root.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/inconshreveable/log15" 9 | homedir "github.com/mitchellh/go-homedir" 10 | "github.com/spf13/cobra" 11 | "github.com/spf13/viper" 12 | 13 | "github.com/vulsio/go-kev/utils" 14 | ) 15 | 16 | var cfgFile string 17 | 18 | // RootCmd represents the base command when called without any subcommands 19 | var RootCmd = &cobra.Command{ 20 | Use: "go-kev", 21 | Short: "Go Known Exploited Vulnerabilities", 22 | Long: "Go Known Exploited Vulnerabilities", 23 | SilenceErrors: true, 24 | SilenceUsage: true, 25 | } 26 | 27 | func init() { 28 | cobra.OnInitialize(initConfig) 29 | 30 | RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.go-kev.yaml)") 31 | 32 | RootCmd.PersistentFlags().Bool("log-to-file", false, "output log to file") 33 | _ = viper.BindPFlag("log-to-file", RootCmd.PersistentFlags().Lookup("log-to-file")) 34 | 35 | RootCmd.PersistentFlags().String("log-dir", "", "/path/to/log") 36 | if err := viper.BindPFlag("log-dir", RootCmd.PersistentFlags().Lookup("log-dir")); err != nil { 37 | panic(err) 38 | } 39 | viper.SetDefault("log-dir", utils.GetDefaultLogDir()) 40 | 41 | RootCmd.PersistentFlags().Bool("log-json", false, "output log as JSON") 42 | if err := viper.BindPFlag("log-json", RootCmd.PersistentFlags().Lookup("log-json")); err != nil { 43 | panic(err) 44 | } 45 | viper.SetDefault("log-json", false) 46 | 47 | RootCmd.PersistentFlags().Bool("quiet", false, "quiet mode (no output)") 48 | if err := viper.BindPFlag("quiet", RootCmd.PersistentFlags().Lookup("quiet")); err != nil { 49 | panic(err) 50 | } 51 | viper.SetDefault("quiet", false) 52 | 53 | RootCmd.PersistentFlags().Bool("debug", false, "debug mode (default: false)") 54 | if err := viper.BindPFlag("debug", RootCmd.PersistentFlags().Lookup("debug")); err != nil { 55 | panic(err) 56 | } 57 | viper.SetDefault("debug", false) 58 | 59 | RootCmd.PersistentFlags().Bool("debug-sql", false, "SQL debug mode") 60 | if err := viper.BindPFlag("debug-sql", RootCmd.PersistentFlags().Lookup("debug-sql")); err != nil { 61 | panic(err) 62 | } 63 | viper.SetDefault("debug-sql", false) 64 | 65 | RootCmd.PersistentFlags().String("dbpath", "", "/path/to/sqlite3 or SQL connection string") 66 | if err := viper.BindPFlag("dbpath", RootCmd.PersistentFlags().Lookup("dbpath")); err != nil { 67 | panic(err) 68 | } 69 | pwd := os.Getenv("PWD") 70 | viper.SetDefault("dbpath", filepath.Join(pwd, "go-kev.sqlite3")) 71 | 72 | RootCmd.PersistentFlags().String("dbtype", "", "Database type to store data in (sqlite3, mysql, postgres or redis supported)") 73 | if err := viper.BindPFlag("dbtype", RootCmd.PersistentFlags().Lookup("dbtype")); err != nil { 74 | panic(err) 75 | } 76 | viper.SetDefault("dbtype", "sqlite3") 77 | 78 | // proxy support 79 | RootCmd.PersistentFlags().String("http-proxy", "", "http://proxy-url:port (default: empty)") 80 | if err := viper.BindPFlag("http-proxy", RootCmd.PersistentFlags().Lookup("http-proxy")); err != nil { 81 | panic(err) 82 | } 83 | viper.SetDefault("http-proxy", "") 84 | } 85 | 86 | // initConfig reads in config file and ENV variables if set. 87 | func initConfig() { 88 | if cfgFile != "" { 89 | viper.SetConfigFile(cfgFile) 90 | } else { 91 | // Find home directory. 92 | home, err := homedir.Dir() 93 | if err != nil { 94 | log15.Error("Failed to find home directory.", "err", err) 95 | os.Exit(1) 96 | } 97 | 98 | // Search config in home directory with name ".go-kev" (without extension). 99 | viper.AddConfigPath(home) 100 | viper.SetConfigName(".go-kev") 101 | } 102 | 103 | viper.AutomaticEnv() // read in environment variables that match 104 | 105 | // If a config file is found, read it in. 106 | if err := viper.ReadInConfig(); err == nil { 107 | fmt.Println("Using config file:", viper.ConfigFileUsed()) 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /commands/server.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/inconshreveable/log15" 7 | "github.com/spf13/cobra" 8 | "github.com/spf13/viper" 9 | "golang.org/x/xerrors" 10 | 11 | "github.com/vulsio/go-kev/db" 12 | "github.com/vulsio/go-kev/models" 13 | "github.com/vulsio/go-kev/server" 14 | "github.com/vulsio/go-kev/utils" 15 | ) 16 | 17 | // serverCmd represents the server command 18 | var serverCmd = &cobra.Command{ 19 | Use: "server", 20 | Short: "Start go-kev HTTP server", 21 | Long: `Start go-kev HTTP server`, 22 | RunE: executeServer, 23 | } 24 | 25 | func init() { 26 | RootCmd.AddCommand(serverCmd) 27 | 28 | serverCmd.PersistentFlags().String("bind", "", "HTTP server bind to IP address") 29 | _ = viper.BindPFlag("bind", serverCmd.PersistentFlags().Lookup("bind")) 30 | viper.SetDefault("bind", "127.0.0.1") 31 | 32 | serverCmd.PersistentFlags().String("port", "", "HTTP server port number") 33 | _ = viper.BindPFlag("port", serverCmd.PersistentFlags().Lookup("port")) 34 | viper.SetDefault("port", "1328") 35 | 36 | } 37 | 38 | func executeServer(_ *cobra.Command, _ []string) (err error) { 39 | if err := utils.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil { 40 | return xerrors.Errorf("Failed to SetLogger. err: %w", err) 41 | } 42 | 43 | driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{}) 44 | if err != nil { 45 | if errors.Is(err, db.ErrDBLocked) { 46 | return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err) 47 | } 48 | return xerrors.Errorf("Failed to open DB. err: %w", err) 49 | } 50 | 51 | fetchMeta, err := driver.GetFetchMeta() 52 | if err != nil { 53 | return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err) 54 | } 55 | if fetchMeta.OutDated() { 56 | return xerrors.Errorf("Failed to start server. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion}) 57 | } 58 | 59 | log15.Info("Starting HTTP Server...") 60 | if err = server.Start(viper.GetBool("log-to-file"), viper.GetString("log-dir"), driver); err != nil { 61 | return xerrors.Errorf("Failed to start server. err: %w", err) 62 | } 63 | 64 | return nil 65 | } 66 | -------------------------------------------------------------------------------- /commands/version.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/spf13/cobra" 7 | 8 | "github.com/vulsio/go-kev/config" 9 | ) 10 | 11 | func init() { 12 | RootCmd.AddCommand(versionCmd) 13 | } 14 | 15 | var versionCmd = &cobra.Command{ 16 | Use: "version", 17 | Short: "Show version", 18 | Long: `Show version`, 19 | Run: func(_ *cobra.Command, _ []string) { 20 | fmt.Printf("go-kev %s %s\n", config.Version, config.Revision) 21 | }, 22 | } 23 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | // Version of go-kev 4 | var Version = "`make build` or `make install` will show the version" 5 | 6 | // Revision of Git 7 | var Revision string 8 | -------------------------------------------------------------------------------- /db/common_test.go: -------------------------------------------------------------------------------- 1 | package db_test 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/spf13/viper" 7 | "golang.org/x/xerrors" 8 | 9 | "github.com/vulsio/go-kev/db" 10 | "github.com/vulsio/go-kev/models" 11 | "github.com/vulsio/go-kev/utils" 12 | ) 13 | 14 | func prepareTestData(driver db.DB) error { 15 | viper.Set("threads", 1) 16 | viper.Set("batch-size", 1) 17 | 18 | var testKEVs = []models.KEVuln{ 19 | { 20 | CveID: "CVE-2021-27104", 21 | VendorProject: "Accellion", 22 | Product: "FTA", 23 | VulnerabilityName: "Accellion FTA OS Command Injection Vulnerability", 24 | DateAdded: time.Date(2021, 11, 3, 0, 0, 0, 0, time.UTC), 25 | ShortDescription: "Accellion FTA contains an OS command injection vulnerability exploited via a crafted POST request to various admin endpoints.", 26 | RequiredAction: "Apply updates per vendor instructions.", 27 | DueDate: time.Date(2021, 11, 17, 0, 0, 0, 0, time.UTC), 28 | KnownRansomwareCampaignUse: "Known", 29 | Notes: "", 30 | }, 31 | } 32 | var testVulnChecks = []models.VulnCheck{ 33 | { 34 | VendorProject: "Accellion", 35 | Product: "FTA", 36 | Description: "Accellion FTA contains an OS command injection vulnerability exploited via a crafted POST request to various admin endpoints.", 37 | Name: "Accellion FTA OS Command Injection Vulnerability", 38 | RequiredAction: "Apply updates per vendor instructions.", 39 | KnownRansomwareCampaignUse: "Known", 40 | CVE: []models.VulnCheckCVE{ 41 | { 42 | CveID: "CVE-2021-27104", 43 | }, 44 | }, 45 | VulnCheckXDB: []models.VulnCheckXDB{}, 46 | VulnCheckReportedExploitation: []models.VulnCheckReportedExploitation{ 47 | { 48 | URL: "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", 49 | DateAdded: time.Date(2021, 11, 3, 0, 0, 0, 0, time.UTC), 50 | }, 51 | { 52 | URL: "https://unit42.paloaltonetworks.com/clop-ransomware/", 53 | DateAdded: time.Date(2021, 4, 13, 0, 0, 0, 0, time.UTC), 54 | }, 55 | { 56 | URL: "https://www.trendmicro.com/vinfo/us/security/news/cybercrime-and-digital-threats/ransomware-double-extortion-and-beyond-revil-clop-and-conti", 57 | DateAdded: time.Date(2021, 6, 3, 0, 0, 0, 0, time.UTC), 58 | }, 59 | { 60 | URL: "https://cybersecurityworks.com/howdymanage/uploads/file/ransomware-_-2022-spotlight-report_compressed.pdf", 61 | DateAdded: time.Date(2022, 1, 26, 0, 0, 0, 0, time.UTC), 62 | }, 63 | { 64 | URL: "https://www.paloaltonetworks.com/content/dam/pan/en_US/assets/pdf/reports/2022-unit42-ransomware-threat-report-final.pdf", 65 | DateAdded: time.Date(2022, 3, 24, 0, 0, 0, 0, time.UTC), 66 | }, 67 | { 68 | URL: "https://static.tenable.com/marketing/whitepapers/Whitepaper-Ransomware_Ecosystem.pdf", 69 | DateAdded: time.Date(2022, 6, 22, 0, 0, 0, 0, time.UTC), 70 | }, 71 | { 72 | URL: "https://www.group-ib.com/resources/research-hub/hi-tech-crime-trends-2022/", 73 | DateAdded: time.Date(2023, 1, 17, 0, 0, 0, 0, time.UTC), 74 | }, 75 | { 76 | URL: "https://fourcore.io/blogs/clop-ransomware-history-adversary-simulation", 77 | DateAdded: time.Date(2023, 6, 3, 0, 0, 0, 0, time.UTC), 78 | }, 79 | { 80 | URL: "https://blog.talosintelligence.com/talos-ir-q2-2023-quarterly-recap/", 81 | DateAdded: time.Date(2023, 7, 26, 0, 0, 0, 0, time.UTC), 82 | }, 83 | { 84 | URL: "https://www.sentinelone.com/resources/watchtower-end-of-year-report-2023/", 85 | DateAdded: time.Date(2021, 11, 3, 0, 0, 0, 0, time.UTC), 86 | }, 87 | { 88 | URL: "https://www.trustwave.com/en-us/resources/blogs/trustwave-blog/defending-the-energy-sector-against-cyber-threats-insights-from-trustwave-spiderlabs/", 89 | DateAdded: time.Date(2024, 5, 15, 0, 0, 0, 0, time.UTC), 90 | }, 91 | { 92 | URL: "https://cisa.gov/news-events/cybersecurity-advisories/aa21-055a", 93 | DateAdded: time.Date(2021, 6, 17, 0, 0, 0, 0, time.UTC), 94 | }, 95 | { 96 | URL: "https://www.cisa.gov/news-events/cybersecurity-advisories/aa21-209a", 97 | DateAdded: time.Date(2021, 8, 20, 0, 0, 0, 0, time.UTC), 98 | }, 99 | { 100 | URL: "https://cisa.gov/news-events/alerts/2022/04/27/2021-top-routinely-exploited-vulnerabilities", 101 | DateAdded: time.Date(2022, 4, 28, 0, 0, 0, 0, time.UTC), 102 | }, 103 | { 104 | URL: "https://cisa.gov/news-events/cybersecurity-advisories/aa22-117a", 105 | DateAdded: time.Date(2022, 4, 28, 0, 0, 0, 0, time.UTC), 106 | }, 107 | { 108 | URL: "https://www.hhs.gov/sites/default/files/threat-profile-june-2023.pdf", 109 | DateAdded: time.Date(2023, 06, 13, 0, 0, 0, 0, time.UTC), 110 | }, 111 | }, 112 | DueDate: utils.ToPtr(time.Date(2021, 11, 17, 0, 0, 0, 0, time.UTC)), 113 | CisaDateAdded: utils.ToPtr(time.Date(2021, 11, 3, 0, 0, 0, 0, time.UTC)), 114 | DateAdded: time.Date(2021, 4, 13, 0, 0, 0, 0, time.UTC), 115 | }, 116 | } 117 | 118 | if err := driver.InsertKEVulns(testKEVs); err != nil { 119 | return xerrors.Errorf("Failed to insert KEVulns. err: %w", err) 120 | } 121 | if err := driver.InsertVulnCheck(testVulnChecks); err != nil { 122 | return xerrors.Errorf("Failed to insert VulnCheck. err: %w", err) 123 | } 124 | return nil 125 | } 126 | -------------------------------------------------------------------------------- /db/db.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "time" 5 | 6 | "golang.org/x/xerrors" 7 | 8 | "github.com/vulsio/go-kev/models" 9 | ) 10 | 11 | // DB : 12 | type DB interface { 13 | Name() string 14 | OpenDB(dbType, dbPath string, debugSQL bool, option Option) error 15 | MigrateDB() error 16 | CloseDB() error 17 | 18 | IsGoKEVModelV1() (bool, error) 19 | GetFetchMeta() (*models.FetchMeta, error) 20 | UpsertFetchMeta(*models.FetchMeta) error 21 | 22 | InsertKEVulns([]models.KEVuln) error 23 | InsertVulnCheck([]models.VulnCheck) error 24 | GetKEVByCveID(string) (Response, error) 25 | GetKEVByMultiCveID([]string) (map[string]Response, error) 26 | } 27 | 28 | // Option : 29 | type Option struct { 30 | RedisTimeout time.Duration 31 | } 32 | 33 | // Response : 34 | type Response struct { 35 | CISA []models.KEVuln `json:"cisa,omitempty"` 36 | VulnCheck []models.VulnCheck `json:"vulncheck,omitempty"` 37 | } 38 | 39 | // NewDB : 40 | func NewDB(dbType string, dbPath string, debugSQL bool, option Option) (driver DB, err error) { 41 | if driver, err = newDB(dbType); err != nil { 42 | return driver, xerrors.Errorf("Failed to new db. err: %w", err) 43 | } 44 | 45 | if err := driver.OpenDB(dbType, dbPath, debugSQL, option); err != nil { 46 | return nil, xerrors.Errorf("Failed to open db. err: %w", err) 47 | } 48 | 49 | isV1, err := driver.IsGoKEVModelV1() 50 | if err != nil { 51 | return nil, xerrors.Errorf("Failed to IsGoKEVModelV1. err: %w", err) 52 | } 53 | if isV1 { 54 | return nil, xerrors.New("Failed to NewDB. Since SchemaVersion is incompatible, delete Database and fetch again.") 55 | } 56 | 57 | if err := driver.MigrateDB(); err != nil { 58 | return driver, xerrors.Errorf("Failed to migrate db. err: %w", err) 59 | } 60 | return driver, nil 61 | } 62 | 63 | func newDB(dbType string) (DB, error) { 64 | switch dbType { 65 | case dialectSqlite3, dialectMysql, dialectPostgreSQL: 66 | return &RDBDriver{name: dbType}, nil 67 | case dialectRedis: 68 | return &RedisDriver{name: dbType}, nil 69 | } 70 | return nil, xerrors.Errorf("Invalid database dialect, %s", dbType) 71 | } 72 | -------------------------------------------------------------------------------- /db/rdb.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "log" 10 | "os" 11 | "slices" 12 | "time" 13 | 14 | "github.com/cheggaaa/pb/v3" 15 | "github.com/glebarez/sqlite" 16 | "github.com/inconshreveable/log15" 17 | "github.com/spf13/viper" 18 | "golang.org/x/xerrors" 19 | "gorm.io/driver/mysql" 20 | "gorm.io/driver/postgres" 21 | "gorm.io/gorm" 22 | "gorm.io/gorm/logger" 23 | 24 | "github.com/vulsio/go-kev/config" 25 | "github.com/vulsio/go-kev/models" 26 | ) 27 | 28 | const ( 29 | dialectSqlite3 = "sqlite3" 30 | dialectMysql = "mysql" 31 | dialectPostgreSQL = "postgres" 32 | ) 33 | 34 | // RDBDriver : 35 | type RDBDriver struct { 36 | name string 37 | conn *gorm.DB 38 | } 39 | 40 | // https://github.com/mattn/go-sqlite3/blob/edc3bb69551dcfff02651f083b21f3366ea2f5ab/error.go#L18-L66 41 | type errNo int 42 | 43 | type sqliteError struct { 44 | Code errNo /* The error code returned by SQLite */ 45 | } 46 | 47 | // result codes from http://www.sqlite.org/c3ref/c_abort.html 48 | var ( 49 | errBusy = errNo(5) /* The database file is locked */ 50 | errLocked = errNo(6) /* A table in the database is locked */ 51 | ) 52 | 53 | // ErrDBLocked : 54 | var ErrDBLocked = xerrors.New("database is locked") 55 | 56 | // Name return db name 57 | func (r *RDBDriver) Name() string { 58 | return r.name 59 | } 60 | 61 | // OpenDB opens Database 62 | func (r *RDBDriver) OpenDB(dbType, dbPath string, debugSQL bool, _ Option) (err error) { 63 | gormConfig := gorm.Config{ 64 | DisableForeignKeyConstraintWhenMigrating: true, 65 | Logger: logger.New( 66 | log.New(os.Stderr, "\r\n", log.LstdFlags), 67 | logger.Config{ 68 | LogLevel: logger.Silent, 69 | }, 70 | ), 71 | } 72 | 73 | if debugSQL { 74 | gormConfig.Logger = logger.New( 75 | log.New(os.Stderr, "\r\n", log.LstdFlags), 76 | logger.Config{ 77 | SlowThreshold: time.Second, 78 | LogLevel: logger.Info, 79 | Colorful: true, 80 | }, 81 | ) 82 | } 83 | 84 | switch r.name { 85 | case dialectSqlite3: 86 | r.conn, err = gorm.Open(sqlite.Open(dbPath), &gormConfig) 87 | if err != nil { 88 | parsedErr, marshalErr := json.Marshal(err) 89 | if marshalErr != nil { 90 | return xerrors.Errorf("Failed to marshal err. err: %w", marshalErr) 91 | } 92 | 93 | var errMsg sqliteError 94 | if unmarshalErr := json.Unmarshal(parsedErr, &errMsg); unmarshalErr != nil { 95 | return xerrors.Errorf("Failed to unmarshal. err: %w", unmarshalErr) 96 | } 97 | 98 | switch errMsg.Code { 99 | case errBusy, errLocked: 100 | return xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, ErrDBLocked) 101 | default: 102 | return xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, err) 103 | } 104 | } 105 | 106 | r.conn.Exec("PRAGMA foreign_keys = ON") 107 | case dialectMysql: 108 | r.conn, err = gorm.Open(mysql.Open(dbPath), &gormConfig) 109 | if err != nil { 110 | return xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, err) 111 | } 112 | case dialectPostgreSQL: 113 | r.conn, err = gorm.Open(postgres.Open(dbPath), &gormConfig) 114 | if err != nil { 115 | return xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dbType, dbPath, err) 116 | } 117 | default: 118 | return xerrors.Errorf("Not Supported DB dialects. r.name: %s", r.name) 119 | } 120 | return nil 121 | } 122 | 123 | // CloseDB close Database 124 | func (r *RDBDriver) CloseDB() (err error) { 125 | if r.conn == nil { 126 | return 127 | } 128 | 129 | var sqlDB *sql.DB 130 | if sqlDB, err = r.conn.DB(); err != nil { 131 | return xerrors.Errorf("Failed to get DB Object. err : %w", err) 132 | } 133 | if err = sqlDB.Close(); err != nil { 134 | return xerrors.Errorf("Failed to close DB. Type: %s. err: %w", r.name, err) 135 | } 136 | return 137 | } 138 | 139 | // MigrateDB migrates Database 140 | func (r *RDBDriver) MigrateDB() error { 141 | if err := r.conn.AutoMigrate( 142 | &models.FetchMeta{}, 143 | 144 | &models.KEVuln{}, 145 | 146 | &models.VulnCheck{}, 147 | &models.VulnCheckCVE{}, 148 | &models.VulnCheckXDB{}, 149 | &models.VulnCheckReportedExploitation{}, 150 | ); err != nil { 151 | switch r.name { 152 | case dialectSqlite3: 153 | if r.name == dialectSqlite3 { 154 | parsedErr, marshalErr := json.Marshal(err) 155 | if marshalErr != nil { 156 | return xerrors.Errorf("Failed to marshal err. err: %w", marshalErr) 157 | } 158 | 159 | var errMsg sqliteError 160 | if unmarshalErr := json.Unmarshal(parsedErr, &errMsg); unmarshalErr != nil { 161 | return xerrors.Errorf("Failed to unmarshal. err: %w", unmarshalErr) 162 | } 163 | 164 | switch errMsg.Code { 165 | case errBusy, errLocked: 166 | return xerrors.Errorf("Failed to migrate. err: %w", ErrDBLocked) 167 | default: 168 | return xerrors.Errorf("Failed to migrate. err: %w", err) 169 | } 170 | } 171 | case dialectMysql, dialectPostgreSQL: 172 | return xerrors.Errorf("Failed to migrate. err: %w", err) 173 | default: 174 | return xerrors.Errorf("Not Supported DB dialects. r.name: %s", r.name) 175 | } 176 | } 177 | 178 | return nil 179 | } 180 | 181 | // IsGoKEVModelV1 determines if the DB was created at the time of go-kev Model v1 182 | func (r *RDBDriver) IsGoKEVModelV1() (bool, error) { 183 | if r.conn.Migrator().HasTable(&models.FetchMeta{}) { 184 | return false, nil 185 | } 186 | 187 | var ( 188 | count int64 189 | err error 190 | ) 191 | switch r.name { 192 | case dialectSqlite3: 193 | err = r.conn.Table("sqlite_master").Where("type = ?", "table").Count(&count).Error 194 | case dialectMysql: 195 | err = r.conn.Table("information_schema.tables").Where("table_schema = ?", r.conn.Migrator().CurrentDatabase()).Count(&count).Error 196 | case dialectPostgreSQL: 197 | err = r.conn.Table("pg_tables").Where("schemaname = ?", "public").Count(&count).Error 198 | } 199 | 200 | if count > 0 { 201 | return true, nil 202 | } 203 | return false, err 204 | } 205 | 206 | // GetFetchMeta get FetchMeta from Database 207 | func (r *RDBDriver) GetFetchMeta() (fetchMeta *models.FetchMeta, err error) { 208 | if err = r.conn.Take(&fetchMeta).Error; err != nil { 209 | if !errors.Is(err, gorm.ErrRecordNotFound) { 210 | return nil, err 211 | } 212 | return &models.FetchMeta{GoKEVRevision: config.Revision, SchemaVersion: models.LatestSchemaVersion, LastFetchedAt: time.Date(1000, time.January, 1, 0, 0, 0, 0, time.UTC)}, nil 213 | } 214 | 215 | return fetchMeta, nil 216 | } 217 | 218 | // UpsertFetchMeta upsert FetchMeta to Database 219 | func (r *RDBDriver) UpsertFetchMeta(fetchMeta *models.FetchMeta) error { 220 | fetchMeta.GoKEVRevision = config.Revision 221 | fetchMeta.SchemaVersion = models.LatestSchemaVersion 222 | return r.conn.Save(fetchMeta).Error 223 | } 224 | 225 | // InsertKEVulns : 226 | func (r *RDBDriver) InsertKEVulns(records []models.KEVuln) (err error) { 227 | log15.Info("Inserting Known Exploited Vulnerabilities...") 228 | return r.deleteAndInsertKEVulns(records) 229 | } 230 | 231 | func (r *RDBDriver) deleteAndInsertKEVulns(records []models.KEVuln) (err error) { 232 | bar := pb.StartNew(len(records)).SetWriter(func() io.Writer { 233 | if viper.GetBool("log-json") { 234 | return io.Discard 235 | } 236 | return os.Stderr 237 | }()) 238 | tx := r.conn.Begin() 239 | defer func() { 240 | if err != nil { 241 | tx.Rollback() 242 | return 243 | } 244 | tx.Commit() 245 | }() 246 | 247 | // Delete all old records 248 | if err := tx.Session(&gorm.Session{AllowGlobalUpdate: true}).Unscoped().Delete(models.KEVuln{}).Error; err != nil { 249 | return xerrors.Errorf("Failed to delete old records. err: %w", err) 250 | } 251 | 252 | batchSize := viper.GetInt("batch-size") 253 | if batchSize < 1 { 254 | return fmt.Errorf("Failed to set batch-size. err: batch-size option is not set properly") 255 | } 256 | 257 | for chunk := range slices.Chunk(records, batchSize) { 258 | if err = tx.Create(chunk).Error; err != nil { 259 | return xerrors.Errorf("Failed to insert. err: %w", err) 260 | } 261 | bar.Add(len(chunk)) 262 | } 263 | bar.Finish() 264 | log15.Info("CveID Count", "count", len(records)) 265 | return nil 266 | } 267 | 268 | // InsertVulnCheck : 269 | func (r *RDBDriver) InsertVulnCheck(records []models.VulnCheck) (err error) { 270 | log15.Info("Inserting VulnCheck Known Exploited Vulnerabilities...") 271 | return r.deleteAndInsertVulnCheck(records) 272 | } 273 | 274 | func (r *RDBDriver) deleteAndInsertVulnCheck(records []models.VulnCheck) (err error) { 275 | bar := pb.StartNew(len(records)).SetWriter(func() io.Writer { 276 | if viper.GetBool("log-json") { 277 | return io.Discard 278 | } 279 | return os.Stderr 280 | }()) 281 | tx := r.conn.Begin() 282 | defer func() { 283 | if err != nil { 284 | tx.Rollback() 285 | return 286 | } 287 | tx.Commit() 288 | }() 289 | 290 | // Delete all old records 291 | for _, table := range []interface{}{models.VulnCheck{}, models.VulnCheckCVE{}, models.VulnCheckXDB{}, models.VulnCheckReportedExploitation{}} { 292 | if err := tx.Session(&gorm.Session{AllowGlobalUpdate: true}).Unscoped().Delete(table).Error; err != nil { 293 | return xerrors.Errorf("Failed to delete old records. err: %w", err) 294 | } 295 | } 296 | 297 | batchSize := viper.GetInt("batch-size") 298 | if batchSize < 1 { 299 | return fmt.Errorf("Failed to set batch-size. err: batch-size option is not set properly") 300 | } 301 | 302 | for chunk := range slices.Chunk(records, batchSize) { 303 | if err = tx.Create(chunk).Error; err != nil { 304 | return xerrors.Errorf("Failed to insert. err: %w", err) 305 | } 306 | bar.Add(len(chunk)) 307 | } 308 | bar.Finish() 309 | log15.Info("CveID Count", "count", len(records)) 310 | return nil 311 | } 312 | 313 | // GetKEVByCveID : 314 | func (r *RDBDriver) GetKEVByCveID(cveID string) (Response, error) { 315 | var res Response 316 | if err := r.conn.Where(&models.KEVuln{CveID: cveID}).Find(&res.CISA).Error; err != nil { 317 | return Response{}, xerrors.Errorf("Failed to get CISA info by CVE-ID. err: %w", err) 318 | } 319 | if err := r.conn. 320 | Joins("JOIN vuln_check_cves ON vuln_check_cves.vuln_check_id = vuln_checks.id AND vuln_check_cves.cve_id = ?", cveID). 321 | Preload("CVE"). 322 | Preload("VulnCheckXDB"). 323 | Preload("VulnCheckReportedExploitation"). 324 | Find(&res.VulnCheck).Error; err != nil { 325 | return Response{}, xerrors.Errorf("Failed to get VulnCheck info by CVE-ID. err: %w", err) 326 | } 327 | return res, nil 328 | } 329 | 330 | // GetKEVByMultiCveID : 331 | func (r *RDBDriver) GetKEVByMultiCveID(cveIDs []string) (map[string]Response, error) { 332 | m := make(map[string]Response) 333 | for _, cveID := range cveIDs { 334 | res, err := r.GetKEVByCveID(cveID) 335 | if err != nil { 336 | return nil, xerrors.Errorf("Failed to get KEV by %s. err: %w", cveID, err) 337 | } 338 | m[cveID] = res 339 | } 340 | return m, nil 341 | } 342 | -------------------------------------------------------------------------------- /db/rdb_test.go: -------------------------------------------------------------------------------- 1 | package db_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/google/go-cmp/cmp" 8 | "github.com/google/go-cmp/cmp/cmpopts" 9 | 10 | "github.com/vulsio/go-kev/db" 11 | "github.com/vulsio/go-kev/models" 12 | "github.com/vulsio/go-kev/utils" 13 | ) 14 | 15 | func TestRDBDriver_GetKEVByMultiCveID(t *testing.T) { 16 | type args struct { 17 | cveIDs []string 18 | } 19 | tests := []struct { 20 | name string 21 | args args 22 | want map[string]db.Response 23 | wantErr bool 24 | }{ 25 | { 26 | name: "happy", 27 | args: args{ 28 | cveIDs: []string{"CVE-2021-27104"}, 29 | }, 30 | want: map[string]db.Response{ 31 | "CVE-2021-27104": { 32 | []models.KEVuln{ 33 | { 34 | CveID: "CVE-2021-27104", 35 | VendorProject: "Accellion", 36 | Product: "FTA", 37 | VulnerabilityName: "Accellion FTA OS Command Injection Vulnerability", 38 | DateAdded: time.Date(2021, 11, 3, 0, 0, 0, 0, time.UTC), 39 | ShortDescription: "Accellion FTA contains an OS command injection vulnerability exploited via a crafted POST request to various admin endpoints.", 40 | RequiredAction: "Apply updates per vendor instructions.", 41 | DueDate: time.Date(2021, 11, 17, 0, 0, 0, 0, time.UTC), 42 | KnownRansomwareCampaignUse: "Known", 43 | Notes: "", 44 | }, 45 | }, 46 | []models.VulnCheck{ 47 | { 48 | VendorProject: "Accellion", 49 | Product: "FTA", 50 | Description: "Accellion FTA contains an OS command injection vulnerability exploited via a crafted POST request to various admin endpoints.", 51 | Name: "Accellion FTA OS Command Injection Vulnerability", 52 | RequiredAction: "Apply updates per vendor instructions.", 53 | KnownRansomwareCampaignUse: "Known", 54 | CVE: []models.VulnCheckCVE{ 55 | { 56 | CveID: "CVE-2021-27104", 57 | }, 58 | }, 59 | VulnCheckXDB: []models.VulnCheckXDB{}, 60 | VulnCheckReportedExploitation: []models.VulnCheckReportedExploitation{ 61 | { 62 | URL: "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", 63 | DateAdded: time.Date(2021, 11, 3, 0, 0, 0, 0, time.UTC), 64 | }, 65 | { 66 | URL: "https://unit42.paloaltonetworks.com/clop-ransomware/", 67 | DateAdded: time.Date(2021, 4, 13, 0, 0, 0, 0, time.UTC), 68 | }, 69 | { 70 | URL: "https://www.trendmicro.com/vinfo/us/security/news/cybercrime-and-digital-threats/ransomware-double-extortion-and-beyond-revil-clop-and-conti", 71 | DateAdded: time.Date(2021, 6, 3, 0, 0, 0, 0, time.UTC), 72 | }, 73 | { 74 | URL: "https://cybersecurityworks.com/howdymanage/uploads/file/ransomware-_-2022-spotlight-report_compressed.pdf", 75 | DateAdded: time.Date(2022, 1, 26, 0, 0, 0, 0, time.UTC), 76 | }, 77 | { 78 | URL: "https://www.paloaltonetworks.com/content/dam/pan/en_US/assets/pdf/reports/2022-unit42-ransomware-threat-report-final.pdf", 79 | DateAdded: time.Date(2022, 3, 24, 0, 0, 0, 0, time.UTC), 80 | }, 81 | { 82 | URL: "https://static.tenable.com/marketing/whitepapers/Whitepaper-Ransomware_Ecosystem.pdf", 83 | DateAdded: time.Date(2022, 6, 22, 0, 0, 0, 0, time.UTC), 84 | }, 85 | { 86 | URL: "https://www.group-ib.com/resources/research-hub/hi-tech-crime-trends-2022/", 87 | DateAdded: time.Date(2023, 1, 17, 0, 0, 0, 0, time.UTC), 88 | }, 89 | { 90 | URL: "https://fourcore.io/blogs/clop-ransomware-history-adversary-simulation", 91 | DateAdded: time.Date(2023, 6, 3, 0, 0, 0, 0, time.UTC), 92 | }, 93 | { 94 | URL: "https://blog.talosintelligence.com/talos-ir-q2-2023-quarterly-recap/", 95 | DateAdded: time.Date(2023, 7, 26, 0, 0, 0, 0, time.UTC), 96 | }, 97 | { 98 | URL: "https://www.sentinelone.com/resources/watchtower-end-of-year-report-2023/", 99 | DateAdded: time.Date(2021, 11, 3, 0, 0, 0, 0, time.UTC), 100 | }, 101 | { 102 | URL: "https://www.trustwave.com/en-us/resources/blogs/trustwave-blog/defending-the-energy-sector-against-cyber-threats-insights-from-trustwave-spiderlabs/", 103 | DateAdded: time.Date(2024, 5, 15, 0, 0, 0, 0, time.UTC), 104 | }, 105 | { 106 | URL: "https://cisa.gov/news-events/cybersecurity-advisories/aa21-055a", 107 | DateAdded: time.Date(2021, 6, 17, 0, 0, 0, 0, time.UTC), 108 | }, 109 | { 110 | URL: "https://www.cisa.gov/news-events/cybersecurity-advisories/aa21-209a", 111 | DateAdded: time.Date(2021, 8, 20, 0, 0, 0, 0, time.UTC), 112 | }, 113 | { 114 | URL: "https://cisa.gov/news-events/alerts/2022/04/27/2021-top-routinely-exploited-vulnerabilities", 115 | DateAdded: time.Date(2022, 4, 28, 0, 0, 0, 0, time.UTC), 116 | }, 117 | { 118 | URL: "https://cisa.gov/news-events/cybersecurity-advisories/aa22-117a", 119 | DateAdded: time.Date(2022, 4, 28, 0, 0, 0, 0, time.UTC), 120 | }, 121 | { 122 | URL: "https://www.hhs.gov/sites/default/files/threat-profile-june-2023.pdf", 123 | DateAdded: time.Date(2023, 06, 13, 0, 0, 0, 0, time.UTC), 124 | }, 125 | }, 126 | DueDate: utils.ToPtr(time.Date(2021, 11, 17, 0, 0, 0, 0, time.UTC)), 127 | CisaDateAdded: utils.ToPtr(time.Date(2021, 11, 3, 0, 0, 0, 0, time.UTC)), 128 | DateAdded: time.Date(2021, 4, 13, 0, 0, 0, 0, time.UTC), 129 | }, 130 | }, 131 | }, 132 | }, 133 | }, 134 | } 135 | 136 | driver, err := db.NewDB("sqlite3", ":memory:", false, db.Option{}) 137 | if err != nil { 138 | t.Fatalf("Failed to new sqlite3 driver. err: %s", err) 139 | } 140 | defer func() { 141 | _ = driver.CloseDB() 142 | }() 143 | 144 | if err := prepareTestData(driver); err != nil { 145 | t.Fatalf("Failed to prepare testdata of KEV. err: %s", err) 146 | } 147 | 148 | for _, tt := range tests { 149 | t.Run(tt.name, func(t *testing.T) { 150 | got, err := driver.GetKEVByMultiCveID(tt.args.cveIDs) 151 | if (err != nil) != tt.wantErr { 152 | t.Errorf("RDBDriver.GetKEVByMultiCveID() error = %v, wantErr %v", err, tt.wantErr) 153 | return 154 | } 155 | if diff := cmp.Diff(got, tt.want, 156 | cmpopts.IgnoreFields(models.KEVuln{}, "ID"), 157 | cmpopts.IgnoreFields(models.VulnCheck{}, "ID"), 158 | cmpopts.IgnoreFields(models.VulnCheckCVE{}, "ID", "VulnCheckID"), 159 | cmpopts.IgnoreFields(models.VulnCheckXDB{}, "ID", "VulnCheckID"), 160 | cmpopts.IgnoreFields(models.VulnCheckReportedExploitation{}, "ID", "VulnCheckID"), 161 | ); diff != "" { 162 | t.Errorf("RDBDriver.GetKEVByMultiCveID(): (-got +want)\n%s", diff) 163 | } 164 | }) 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /db/redis.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "context" 5 | "crypto/md5" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "os" 11 | "slices" 12 | "strconv" 13 | "strings" 14 | "time" 15 | 16 | "github.com/cheggaaa/pb/v3" 17 | "github.com/go-redis/redis/v8" 18 | "github.com/inconshreveable/log15" 19 | "github.com/spf13/viper" 20 | "golang.org/x/xerrors" 21 | 22 | "github.com/vulsio/go-kev/config" 23 | "github.com/vulsio/go-kev/models" 24 | ) 25 | 26 | /** 27 | # Redis Data Structure 28 | - Hash 29 | ┌───┬────────────────┬─────────────────────┬───────────┬──────────────────────────────────────────────────┐ 30 | │NO │ KEY │ FIELD │ VALUE │ PURPOSE │ 31 | └───┴────────────────┴─────────────────────┴───────────┴──────────────────────────────────────────────────┘ 32 | ┌───┬────────────────┬─────────────────────┬───────────┬──────────────────────────────────────────────────┐ 33 | │ 1 │ KEV#CVE#$CVEID │ :MD5SUM │ JSON │ TO GET VULN FROM CVEID │ 34 | ├───┼────────────────┼─────────────────────┼───────────┼──────────────────────────────────────────────────┤ 35 | │ 2 │ KEV#DEP │ CISA/VulnCheck │ JSON │ TO DELETE OUTDATED AND UNNEEDED FIELD AND MEMBER │ 36 | ├───┼────────────────┼─────────────────────┼───────────┼──────────────────────────────────────────────────┤ 37 | │ 3 │ KEV#FETCHMETA │ Revision │ string │ GET Go-KEV Binary Revision │ 38 | ├───┼────────────────┼─────────────────────┼───────────┼──────────────────────────────────────────────────┤ 39 | │ 4 │ KEV#FETCHMETA │ SchemaVersion │ uint │ GET Go-KEV Schema Version │ 40 | ├───┼────────────────┼─────────────────────┼───────────┼──────────────────────────────────────────────────┤ 41 | │ 5 │ KEV#FETCHMETA │ LastFetchedAt │ time.Time │ GET Go-KEV Last Fetched Time │ 42 | └───┴────────────────┴─────────────────────┴───────────┴──────────────────────────────────────────────────┘ 43 | **/ 44 | 45 | const ( 46 | dialectRedis = "redis" 47 | cveIDKeyFormat = "KEV#CVE#%s" 48 | depKey = "KEV#DEP" 49 | fetchMetaKey = "KEV#FETCHMETA" 50 | 51 | kevulnType = "CISA" 52 | vulncheckType = "VulnCheck" 53 | ) 54 | 55 | // RedisDriver is Driver for Redis 56 | type RedisDriver struct { 57 | name string 58 | conn *redis.Client 59 | } 60 | 61 | // Name return db name 62 | func (r *RedisDriver) Name() string { 63 | return r.name 64 | } 65 | 66 | // OpenDB opens Database 67 | func (r *RedisDriver) OpenDB(_, dbPath string, _ bool, option Option) error { 68 | if err := r.connectRedis(dbPath, option); err != nil { 69 | return xerrors.Errorf("Failed to open DB. dbtype: %s, dbpath: %s, err: %w", dialectRedis, dbPath, err) 70 | } 71 | return nil 72 | } 73 | 74 | func (r *RedisDriver) connectRedis(dbPath string, option Option) error { 75 | ctx := context.Background() 76 | var err error 77 | var opt *redis.Options 78 | if opt, err = redis.ParseURL(dbPath); err != nil { 79 | return xerrors.Errorf("Failed to parse url. err: %w", err) 80 | } 81 | if 0 < option.RedisTimeout.Seconds() { 82 | opt.ReadTimeout = option.RedisTimeout 83 | } 84 | r.conn = redis.NewClient(opt) 85 | return r.conn.Ping(ctx).Err() 86 | } 87 | 88 | // CloseDB close Database 89 | func (r *RedisDriver) CloseDB() (err error) { 90 | if r.conn == nil { 91 | return 92 | } 93 | if err = r.conn.Close(); err != nil { 94 | return xerrors.Errorf("Failed to close DB. Type: %s. err: %w", r.name, err) 95 | } 96 | return 97 | } 98 | 99 | // MigrateDB migrates Database 100 | func (r *RedisDriver) MigrateDB() error { 101 | return nil 102 | } 103 | 104 | // IsGoKEVModelV1 determines if the DB was created at the time of go-kev Model v1 105 | func (r *RedisDriver) IsGoKEVModelV1() (bool, error) { 106 | ctx := context.Background() 107 | 108 | exists, err := r.conn.Exists(ctx, fetchMetaKey).Result() 109 | if err != nil { 110 | return false, xerrors.Errorf("Failed to Exists. err: %w", err) 111 | } 112 | if exists == 0 { 113 | keys, _, err := r.conn.Scan(ctx, 0, "KEV#*", 1).Result() 114 | if err != nil { 115 | return false, xerrors.Errorf("Failed to Scan. err: %w", err) 116 | } 117 | if len(keys) == 0 { 118 | return false, nil 119 | } 120 | return true, nil 121 | } 122 | 123 | return false, nil 124 | } 125 | 126 | // GetFetchMeta get FetchMeta from Database 127 | func (r *RedisDriver) GetFetchMeta() (*models.FetchMeta, error) { 128 | ctx := context.Background() 129 | 130 | exists, err := r.conn.Exists(ctx, fetchMetaKey).Result() 131 | if err != nil { 132 | return nil, xerrors.Errorf("Failed to Exists. err: %w", err) 133 | } 134 | if exists == 0 { 135 | return &models.FetchMeta{GoKEVRevision: config.Revision, SchemaVersion: models.LatestSchemaVersion, LastFetchedAt: time.Date(1000, time.January, 1, 0, 0, 0, 0, time.UTC)}, nil 136 | } 137 | 138 | revision, err := r.conn.HGet(ctx, fetchMetaKey, "Revision").Result() 139 | if err != nil { 140 | return nil, xerrors.Errorf("Failed to HGet Revision. err: %w", err) 141 | } 142 | 143 | verstr, err := r.conn.HGet(ctx, fetchMetaKey, "SchemaVersion").Result() 144 | if err != nil { 145 | return nil, xerrors.Errorf("Failed to HGet SchemaVersion. err: %w", err) 146 | } 147 | version, err := strconv.ParseUint(verstr, 10, 8) 148 | if err != nil { 149 | return nil, xerrors.Errorf("Failed to ParseUint. err: %w", err) 150 | } 151 | 152 | datestr, err := r.conn.HGet(ctx, fetchMetaKey, "LastFetchedAt").Result() 153 | if err != nil { 154 | if !errors.Is(err, redis.Nil) { 155 | return nil, xerrors.Errorf("Failed to HGet LastFetchedAt. err: %w", err) 156 | } 157 | datestr = time.Date(1000, time.January, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339) 158 | } 159 | date, err := time.Parse(time.RFC3339, datestr) 160 | if err != nil { 161 | return nil, xerrors.Errorf("Failed to Parse date. err: %w", err) 162 | } 163 | 164 | return &models.FetchMeta{GoKEVRevision: revision, SchemaVersion: uint(version), LastFetchedAt: date}, nil 165 | } 166 | 167 | // UpsertFetchMeta upsert FetchMeta to Database 168 | func (r *RedisDriver) UpsertFetchMeta(fetchMeta *models.FetchMeta) error { 169 | return r.conn.HSet(context.Background(), fetchMetaKey, map[string]interface{}{"Revision": config.Revision, "SchemaVersion": models.LatestSchemaVersion, "LastFetchedAt": fetchMeta.LastFetchedAt}).Err() 170 | } 171 | 172 | // InsertKEVulns : 173 | func (r *RedisDriver) InsertKEVulns(records []models.KEVuln) (err error) { 174 | ctx := context.Background() 175 | batchSize := viper.GetInt("batch-size") 176 | if batchSize < 1 { 177 | return fmt.Errorf("Failed to set batch-size. err: batch-size option is not set properly") 178 | } 179 | 180 | // newDeps, oldDeps: {"CVEID": {"CISA:HashSum(CVEJSON)": {}}} 181 | newDeps := map[string]map[string]struct{}{} 182 | oldDepsStr := "{}" 183 | t, err := r.conn.Type(ctx, depKey).Result() 184 | if err != nil { 185 | return xerrors.Errorf("Failed to Type key: %s. err: %w", depKey, err) 186 | } 187 | switch t { 188 | case "string": 189 | oldDepsStr, err = r.conn.Get(ctx, depKey).Result() 190 | if err != nil { 191 | return xerrors.Errorf("Failed to Get key: %s. err: %w", depKey, err) 192 | } 193 | if _, err := r.conn.Del(ctx, depKey).Result(); err != nil { 194 | return xerrors.Errorf("Failed to Del key: %s. err: %w", depKey, err) 195 | } 196 | case "hash": 197 | oldDepsStr, err = r.conn.HGet(ctx, depKey, kevulnType).Result() 198 | if err != nil { 199 | if !errors.Is(err, redis.Nil) { 200 | return xerrors.Errorf("Failed to Get key: %s, field: %s. err: %w", depKey, kevulnType, err) 201 | } 202 | oldDepsStr = "{}" 203 | } 204 | case "none": 205 | default: 206 | return xerrors.Errorf("unexpected %s key type. expected: %q, actual: %q", depKey, []string{"string", "hash", "none"}, t) 207 | } 208 | var oldDeps map[string]map[string]struct{} 209 | if err := json.Unmarshal([]byte(oldDepsStr), &oldDeps); err != nil { 210 | return xerrors.Errorf("Failed to unmarshal JSON. err: %w", err) 211 | } 212 | 213 | log15.Info("Inserting Known Exploited Vulnerabilities...") 214 | bar := pb.StartNew(len(records)).SetWriter(func() io.Writer { 215 | if viper.GetBool("log-json") { 216 | return io.Discard 217 | } 218 | return os.Stderr 219 | }()) 220 | for chunk := range slices.Chunk(records, batchSize) { 221 | pipe := r.conn.Pipeline() 222 | for _, record := range chunk { 223 | j, err := json.Marshal(record) 224 | if err != nil { 225 | return xerrors.Errorf("Failed to marshal json. err: %w", err) 226 | } 227 | 228 | hash := fmt.Sprintf("%s:%x", kevulnType, md5.Sum(j)) 229 | _ = pipe.HSet(ctx, fmt.Sprintf(cveIDKeyFormat, record.CveID), hash, string(j)) 230 | 231 | if _, ok := newDeps[record.CveID]; !ok { 232 | newDeps[record.CveID] = map[string]struct{}{} 233 | } 234 | if _, ok := newDeps[record.CveID][hash]; !ok { 235 | newDeps[record.CveID][hash] = struct{}{} 236 | } 237 | if _, ok := oldDeps[record.CveID]; ok { 238 | delete(oldDeps[record.CveID], hash) 239 | if len(oldDeps[record.CveID]) == 0 { 240 | delete(oldDeps, record.CveID) 241 | } 242 | } 243 | } 244 | if _, err := pipe.Exec(ctx); err != nil { 245 | return xerrors.Errorf("Failed to exec pipeline. err: %w", err) 246 | } 247 | bar.Add(len(chunk)) 248 | } 249 | bar.Finish() 250 | 251 | pipe := r.conn.Pipeline() 252 | for cveID, hashes := range oldDeps { 253 | for hash := range hashes { 254 | _ = pipe.HDel(ctx, fmt.Sprintf(cveIDKeyFormat, cveID), hash) 255 | } 256 | } 257 | newDepsJSON, err := json.Marshal(newDeps) 258 | if err != nil { 259 | return xerrors.Errorf("Failed to Marshal JSON. err: %w", err) 260 | } 261 | _ = pipe.HSet(ctx, depKey, kevulnType, string(newDepsJSON)) 262 | if _, err := pipe.Exec(ctx); err != nil { 263 | return xerrors.Errorf("Failed to exec pipeline. err: %w", err) 264 | } 265 | 266 | log15.Info("CveID Count", "count", len(records)) 267 | return nil 268 | } 269 | 270 | // InsertVulnCheck : 271 | func (r *RedisDriver) InsertVulnCheck(records []models.VulnCheck) (err error) { 272 | ctx := context.Background() 273 | batchSize := viper.GetInt("batch-size") 274 | if batchSize < 1 { 275 | return fmt.Errorf("Failed to set batch-size. err: batch-size option is not set properly") 276 | } 277 | 278 | // newDeps, oldDeps: {"CVEID": {"VulnCheck:HashSum(CVEJSON)": {}}} 279 | newDeps := map[string]map[string]struct{}{} 280 | oldDepsStr := "{}" 281 | t, err := r.conn.Type(ctx, depKey).Result() 282 | if err != nil { 283 | return xerrors.Errorf("Failed to Type key: %s. err: %w", depKey, err) 284 | } 285 | switch t { 286 | case "string": 287 | depsStr, err := r.conn.Get(ctx, depKey).Result() 288 | if err != nil { 289 | return xerrors.Errorf("Failed to Get key: %s. err: %w", depKey, err) 290 | } 291 | if _, err := r.conn.Del(ctx, depKey).Result(); err != nil { 292 | return xerrors.Errorf("Failed to Del key: %s. err: %w", depKey, err) 293 | } 294 | if _, err := r.conn.HSet(ctx, depKey, kevulnType, depsStr).Result(); err != nil { 295 | return xerrors.Errorf("Failed to HSet key: %s, field: %s. err: %w", depKey, kevulnType, err) 296 | } 297 | case "hash": 298 | oldDepsStr, err = r.conn.HGet(ctx, depKey, vulncheckType).Result() 299 | if err != nil { 300 | if !errors.Is(err, redis.Nil) { 301 | return xerrors.Errorf("Failed to Get key: %s, field: %s. err: %w", depKey, vulncheckType, err) 302 | } 303 | oldDepsStr = "{}" 304 | } 305 | case "none": 306 | default: 307 | return xerrors.Errorf("unexpected %s key type. expected: %q, actual: %q", depKey, []string{"string", "hash", "none"}, t) 308 | } 309 | var oldDeps map[string]map[string]struct{} 310 | if err := json.Unmarshal([]byte(oldDepsStr), &oldDeps); err != nil { 311 | return xerrors.Errorf("Failed to unmarshal JSON. err: %w", err) 312 | } 313 | 314 | log15.Info("Inserting VulnCheck Known Exploited Vulnerabilities...") 315 | bar := pb.StartNew(len(records)).SetWriter(func() io.Writer { 316 | if viper.GetBool("log-json") { 317 | return io.Discard 318 | } 319 | return os.Stderr 320 | }()) 321 | for chunk := range slices.Chunk(records, batchSize) { 322 | pipe := r.conn.Pipeline() 323 | for _, record := range chunk { 324 | j, err := json.Marshal(record) 325 | if err != nil { 326 | return xerrors.Errorf("Failed to marshal json. err: %w", err) 327 | } 328 | 329 | hash := fmt.Sprintf("%s:%x", vulncheckType, md5.Sum(j)) 330 | for _, c := range record.CVE { 331 | _ = pipe.HSet(ctx, fmt.Sprintf(cveIDKeyFormat, c.CveID), hash, string(j)) 332 | 333 | if _, ok := newDeps[c.CveID]; !ok { 334 | newDeps[c.CveID] = map[string]struct{}{} 335 | } 336 | if _, ok := newDeps[c.CveID][hash]; !ok { 337 | newDeps[c.CveID][hash] = struct{}{} 338 | } 339 | if _, ok := oldDeps[c.CveID]; ok { 340 | delete(oldDeps[c.CveID], hash) 341 | if len(oldDeps[c.CveID]) == 0 { 342 | delete(oldDeps, c.CveID) 343 | } 344 | } 345 | } 346 | } 347 | if _, err := pipe.Exec(ctx); err != nil { 348 | return xerrors.Errorf("Failed to exec pipeline. err: %w", err) 349 | } 350 | bar.Add(len(chunk)) 351 | } 352 | bar.Finish() 353 | 354 | pipe := r.conn.Pipeline() 355 | for cveID, hashes := range oldDeps { 356 | for hash := range hashes { 357 | _ = pipe.HDel(ctx, fmt.Sprintf(cveIDKeyFormat, cveID), hash) 358 | } 359 | } 360 | newDepsJSON, err := json.Marshal(newDeps) 361 | if err != nil { 362 | return xerrors.Errorf("Failed to Marshal JSON. err: %w", err) 363 | } 364 | _ = pipe.HSet(ctx, depKey, vulncheckType, string(newDepsJSON)) 365 | if _, err := pipe.Exec(ctx); err != nil { 366 | return xerrors.Errorf("Failed to exec pipeline. err: %w", err) 367 | } 368 | 369 | log15.Info("CveID Count", "count", len(records)) 370 | return nil 371 | } 372 | 373 | // GetKEVByCveID : 374 | func (r *RedisDriver) GetKEVByCveID(cveID string) (Response, error) { 375 | results, err := r.conn.HGetAll(context.Background(), fmt.Sprintf(cveIDKeyFormat, cveID)).Result() 376 | if err != nil { 377 | return Response{}, xerrors.Errorf("Failed to HGetAll key: %s. err: %w", fmt.Sprintf(cveIDKeyFormat, cveID), err) 378 | } 379 | 380 | var res Response 381 | for f, s := range results { 382 | switch { 383 | case strings.HasPrefix(f, kevulnType): 384 | var v models.KEVuln 385 | if err := json.Unmarshal([]byte(s), &v); err != nil { 386 | return Response{}, xerrors.Errorf("Failed to unmarshal json. key: %s, field: %s. err: %w", fmt.Sprintf(cveIDKeyFormat, cveID), f, err) 387 | } 388 | res.CISA = append(res.CISA, v) 389 | case strings.HasPrefix(f, vulncheckType): 390 | var v models.VulnCheck 391 | if err := json.Unmarshal([]byte(s), &v); err != nil { 392 | return Response{}, xerrors.Errorf("Failed to unmarshal json. key: %s, field: %s. err: %w", fmt.Sprintf(cveIDKeyFormat, cveID), f, err) 393 | } 394 | res.VulnCheck = append(res.VulnCheck, v) 395 | default: 396 | if f != fmt.Sprintf("%x", md5.Sum([]byte(s))) { 397 | return Response{}, xerrors.Errorf("unexpected %s field. expected: %q, actual: %q", fmt.Sprintf(cveIDKeyFormat, cveID), []string{fmt.Sprintf("%s:", kevulnType), fmt.Sprintf("%s:", vulncheckType)}, f) 398 | } 399 | var v models.KEVuln 400 | if err := json.Unmarshal([]byte(s), &v); err != nil { 401 | return Response{}, xerrors.Errorf("Failed to unmarshal json. key: %s, field: %s. err: %w", fmt.Sprintf(cveIDKeyFormat, cveID), f, err) 402 | } 403 | res.CISA = append(res.CISA, v) 404 | } 405 | } 406 | return res, nil 407 | } 408 | 409 | // GetKEVByMultiCveID : 410 | func (r *RedisDriver) GetKEVByMultiCveID(cveIDs []string) (map[string]Response, error) { 411 | ctx := context.Background() 412 | 413 | if len(cveIDs) == 0 { 414 | return map[string]Response{}, nil 415 | } 416 | 417 | m := map[string]*redis.StringStringMapCmd{} 418 | pipe := r.conn.Pipeline() 419 | for _, cveID := range cveIDs { 420 | m[cveID] = pipe.HGetAll(ctx, fmt.Sprintf(cveIDKeyFormat, cveID)) 421 | } 422 | if _, err := pipe.Exec(ctx); err != nil { 423 | return nil, xerrors.Errorf("Failed to exec pipeline. err: %w", err) 424 | } 425 | 426 | rm := make(map[string]Response) 427 | for cveID, cmd := range m { 428 | results, err := cmd.Result() 429 | if err != nil { 430 | return nil, xerrors.Errorf("Failed to HGetAll. err: %w", err) 431 | } 432 | 433 | var res Response 434 | for f, s := range results { 435 | switch { 436 | case strings.HasPrefix(f, kevulnType): 437 | var v models.KEVuln 438 | if err := json.Unmarshal([]byte(s), &v); err != nil { 439 | return nil, xerrors.Errorf("Failed to unmarshal json. key: %s, field: %s. err: %w", fmt.Sprintf(cveIDKeyFormat, cveID), f, err) 440 | } 441 | res.CISA = append(res.CISA, v) 442 | case strings.HasPrefix(f, vulncheckType): 443 | var v models.VulnCheck 444 | if err := json.Unmarshal([]byte(s), &v); err != nil { 445 | return nil, xerrors.Errorf("Failed to unmarshal json. key: %s, field: %s. err: %w", fmt.Sprintf(cveIDKeyFormat, cveID), f, err) 446 | } 447 | res.VulnCheck = append(res.VulnCheck, v) 448 | default: 449 | if f != fmt.Sprintf("%x", md5.Sum([]byte(s))) { 450 | return nil, xerrors.Errorf("unexpected %s field. expected: %q, actual: %q", fmt.Sprintf(cveIDKeyFormat, cveID), []string{fmt.Sprintf("%s:", kevulnType), fmt.Sprintf("%s:", vulncheckType)}, f) 451 | } 452 | var v models.KEVuln 453 | if err := json.Unmarshal([]byte(s), &v); err != nil { 454 | return nil, xerrors.Errorf("Failed to unmarshal json. key: %s, field: %s. err: %w", fmt.Sprintf(cveIDKeyFormat, cveID), f, err) 455 | } 456 | res.CISA = append(res.CISA, v) 457 | } 458 | } 459 | rm[cveID] = res 460 | } 461 | return rm, nil 462 | } 463 | -------------------------------------------------------------------------------- /db/redis_test.go: -------------------------------------------------------------------------------- 1 | package db_test 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "testing" 7 | "time" 8 | 9 | "github.com/alicebob/miniredis/v2" 10 | "golang.org/x/xerrors" 11 | 12 | "github.com/vulsio/go-kev/db" 13 | "github.com/vulsio/go-kev/models" 14 | "github.com/vulsio/go-kev/utils" 15 | ) 16 | 17 | func setupRedis() (*miniredis.Miniredis, db.DB, error) { 18 | s, err := miniredis.Run() 19 | if err != nil { 20 | return nil, nil, xerrors.Errorf("Failed to run miniredis: %w", err) 21 | } 22 | driver, err := db.NewDB("redis", fmt.Sprintf("redis://%s", s.Addr()), false, db.Option{}) 23 | if err != nil { 24 | return nil, nil, xerrors.Errorf("Failed to new db: %w", err) 25 | } 26 | return s, driver, nil 27 | } 28 | 29 | func teardownRedis(s *miniredis.Miniredis, driver db.DB) { 30 | s.Close() 31 | _ = driver.CloseDB() 32 | } 33 | 34 | func TestRedisDriver_GetKEVByMultiCveID(t *testing.T) { 35 | t.Parallel() 36 | s, driver, err := setupRedis() 37 | if err != nil { 38 | t.Fatalf("Failed to prepare redis: %s", err) 39 | } 40 | defer teardownRedis(s, driver) 41 | if err := prepareTestData(driver); err != nil { 42 | t.Fatalf("Failed to prepare testdata of KEV: %s", err) 43 | } 44 | 45 | testdata := []struct { 46 | name string 47 | cveIDs []string 48 | expected map[string]db.Response 49 | }{ 50 | { 51 | name: "single cveID", 52 | cveIDs: []string{"CVE-2021-27104"}, 53 | expected: map[string]db.Response{ 54 | "CVE-2021-27104": { 55 | []models.KEVuln{ 56 | { 57 | CveID: "CVE-2021-27104", 58 | VendorProject: "Accellion", 59 | Product: "FTA", 60 | VulnerabilityName: "Accellion FTA OS Command Injection Vulnerability", 61 | DateAdded: time.Date(2021, 11, 3, 0, 0, 0, 0, time.UTC), 62 | ShortDescription: "Accellion FTA contains an OS command injection vulnerability exploited via a crafted POST request to various admin endpoints.", 63 | RequiredAction: "Apply updates per vendor instructions.", 64 | DueDate: time.Date(2021, 11, 17, 0, 0, 0, 0, time.UTC), 65 | KnownRansomwareCampaignUse: "Known", 66 | Notes: "", 67 | }, 68 | }, 69 | []models.VulnCheck{ 70 | { 71 | VendorProject: "Accellion", 72 | Product: "FTA", 73 | Description: "Accellion FTA contains an OS command injection vulnerability exploited via a crafted POST request to various admin endpoints.", 74 | Name: "Accellion FTA OS Command Injection Vulnerability", 75 | RequiredAction: "Apply updates per vendor instructions.", 76 | KnownRansomwareCampaignUse: "Known", 77 | CVE: []models.VulnCheckCVE{ 78 | { 79 | CveID: "CVE-2021-27104", 80 | }, 81 | }, 82 | VulnCheckXDB: []models.VulnCheckXDB{}, 83 | VulnCheckReportedExploitation: []models.VulnCheckReportedExploitation{ 84 | { 85 | URL: "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json", 86 | DateAdded: time.Date(2021, 11, 3, 0, 0, 0, 0, time.UTC), 87 | }, 88 | { 89 | URL: "https://unit42.paloaltonetworks.com/clop-ransomware/", 90 | DateAdded: time.Date(2021, 4, 13, 0, 0, 0, 0, time.UTC), 91 | }, 92 | { 93 | URL: "https://www.trendmicro.com/vinfo/us/security/news/cybercrime-and-digital-threats/ransomware-double-extortion-and-beyond-revil-clop-and-conti", 94 | DateAdded: time.Date(2021, 6, 3, 0, 0, 0, 0, time.UTC), 95 | }, 96 | { 97 | URL: "https://cybersecurityworks.com/howdymanage/uploads/file/ransomware-_-2022-spotlight-report_compressed.pdf", 98 | DateAdded: time.Date(2022, 1, 26, 0, 0, 0, 0, time.UTC), 99 | }, 100 | { 101 | URL: "https://www.paloaltonetworks.com/content/dam/pan/en_US/assets/pdf/reports/2022-unit42-ransomware-threat-report-final.pdf", 102 | DateAdded: time.Date(2022, 3, 24, 0, 0, 0, 0, time.UTC), 103 | }, 104 | { 105 | URL: "https://static.tenable.com/marketing/whitepapers/Whitepaper-Ransomware_Ecosystem.pdf", 106 | DateAdded: time.Date(2022, 6, 22, 0, 0, 0, 0, time.UTC), 107 | }, 108 | { 109 | URL: "https://www.group-ib.com/resources/research-hub/hi-tech-crime-trends-2022/", 110 | DateAdded: time.Date(2023, 1, 17, 0, 0, 0, 0, time.UTC), 111 | }, 112 | { 113 | URL: "https://fourcore.io/blogs/clop-ransomware-history-adversary-simulation", 114 | DateAdded: time.Date(2023, 6, 3, 0, 0, 0, 0, time.UTC), 115 | }, 116 | { 117 | URL: "https://blog.talosintelligence.com/talos-ir-q2-2023-quarterly-recap/", 118 | DateAdded: time.Date(2023, 7, 26, 0, 0, 0, 0, time.UTC), 119 | }, 120 | { 121 | URL: "https://www.sentinelone.com/resources/watchtower-end-of-year-report-2023/", 122 | DateAdded: time.Date(2021, 11, 3, 0, 0, 0, 0, time.UTC), 123 | }, 124 | { 125 | URL: "https://www.trustwave.com/en-us/resources/blogs/trustwave-blog/defending-the-energy-sector-against-cyber-threats-insights-from-trustwave-spiderlabs/", 126 | DateAdded: time.Date(2024, 5, 15, 0, 0, 0, 0, time.UTC), 127 | }, 128 | { 129 | URL: "https://cisa.gov/news-events/cybersecurity-advisories/aa21-055a", 130 | DateAdded: time.Date(2021, 6, 17, 0, 0, 0, 0, time.UTC), 131 | }, 132 | { 133 | URL: "https://www.cisa.gov/news-events/cybersecurity-advisories/aa21-209a", 134 | DateAdded: time.Date(2021, 8, 20, 0, 0, 0, 0, time.UTC), 135 | }, 136 | { 137 | URL: "https://cisa.gov/news-events/alerts/2022/04/27/2021-top-routinely-exploited-vulnerabilities", 138 | DateAdded: time.Date(2022, 4, 28, 0, 0, 0, 0, time.UTC), 139 | }, 140 | { 141 | URL: "https://cisa.gov/news-events/cybersecurity-advisories/aa22-117a", 142 | DateAdded: time.Date(2022, 4, 28, 0, 0, 0, 0, time.UTC), 143 | }, 144 | { 145 | URL: "https://www.hhs.gov/sites/default/files/threat-profile-june-2023.pdf", 146 | DateAdded: time.Date(2023, 06, 13, 0, 0, 0, 0, time.UTC), 147 | }, 148 | }, 149 | DueDate: utils.ToPtr(time.Date(2021, 11, 17, 0, 0, 0, 0, time.UTC)), 150 | CisaDateAdded: utils.ToPtr(time.Date(2021, 11, 3, 0, 0, 0, 0, time.UTC)), 151 | DateAdded: time.Date(2021, 4, 13, 0, 0, 0, 0, time.UTC), 152 | }, 153 | }, 154 | }, 155 | }, 156 | }, 157 | } 158 | 159 | for _, tt := range testdata { 160 | got, err := driver.GetKEVByMultiCveID(tt.cveIDs) 161 | if err != nil { 162 | t.Errorf("RedisDriver.GetKEVByMultiCveID() error = %v", err) 163 | return 164 | } 165 | if !reflect.DeepEqual(got, tt.expected) { 166 | t.Errorf("got: %v, expected: %v", got, tt.expected) 167 | } 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /fetcher/kevuln/kevuln.go: -------------------------------------------------------------------------------- 1 | package kevuln 2 | 3 | import ( 4 | "encoding/json" 5 | "time" 6 | 7 | "github.com/inconshreveable/log15" 8 | "golang.org/x/xerrors" 9 | 10 | "github.com/vulsio/go-kev/models" 11 | "github.com/vulsio/go-kev/utils" 12 | ) 13 | 14 | // Fetch : 15 | func Fetch() ([]models.KEVuln, error) { 16 | url := "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" 17 | log15.Info("Fetching", "URL", url) 18 | vulnJSON, err := utils.FetchURL(url) 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | catalog := catalog{} 24 | if err := json.Unmarshal(vulnJSON, &catalog); err != nil { 25 | return nil, xerrors.Errorf("failed to decode CISA Known Exploited Vulnerabilities JSON: %w", err) 26 | } 27 | 28 | vs := make([]models.KEVuln, 0, len(catalog.Vulnerabilities)) 29 | for _, v := range catalog.Vulnerabilities { 30 | vs = append(vs, models.KEVuln{ 31 | CveID: v.CveID, 32 | VendorProject: v.VendorProject, 33 | Product: v.Product, 34 | VulnerabilityName: v.VulnerabilityName, 35 | DateAdded: parsedOrDefaultTime("2006-01-02", v.DateAdded), 36 | ShortDescription: v.ShortDescription, 37 | RequiredAction: v.RequiredAction, 38 | DueDate: parsedOrDefaultTime("2006-01-02", v.DueDate), 39 | KnownRansomwareCampaignUse: v.KnownRansomwareCampaignUse, 40 | Notes: v.Notes, 41 | }) 42 | } 43 | 44 | return vs, nil 45 | } 46 | 47 | func parsedOrDefaultTime(layout string, value string) time.Time { 48 | defaultTime := time.Date(1000, time.January, 1, 0, 0, 0, 0, time.UTC) 49 | if value == "" { 50 | return defaultTime 51 | } 52 | 53 | if t, err := time.Parse(layout, value); err == nil { 54 | return t 55 | } 56 | log15.Warn("Failed to parse string", "timeformat", layout, "target string", value) 57 | return defaultTime 58 | } 59 | -------------------------------------------------------------------------------- /fetcher/kevuln/types.go: -------------------------------------------------------------------------------- 1 | package kevuln 2 | 3 | import "time" 4 | 5 | type catalog struct { 6 | Title string `json:"title"` 7 | CatalogVersion string `json:"catalogVersion"` 8 | DateReleased time.Time `json:"dateReleased"` 9 | Count int `json:"count"` 10 | Vulnerabilities []struct { 11 | CveID string `json:"cveID"` 12 | VendorProject string `json:"vendorProject"` 13 | Product string `json:"product"` 14 | VulnerabilityName string `json:"vulnerabilityName"` 15 | DateAdded string `json:"dateAdded"` 16 | ShortDescription string `json:"shortDescription"` 17 | RequiredAction string `json:"requiredAction"` 18 | DueDate string `json:"dueDate"` 19 | KnownRansomwareCampaignUse string `json:"knownRansomwareCampaignUse"` 20 | Notes string `json:"notes"` 21 | } `json:"vulnerabilities"` 22 | } 23 | -------------------------------------------------------------------------------- /fetcher/vulncheck/types.go: -------------------------------------------------------------------------------- 1 | package vulncheck 2 | 3 | import "time" 4 | 5 | // https://docs.vulncheck.com/community/vulncheck-kev/schema 6 | type vulncheck struct { 7 | VendorProject string `json:"vendorProject"` 8 | Product string `json:"product"` 9 | Description string `json:"shortDescription"` 10 | Name string `json:"vulnerabilityName"` 11 | RequiredAction string `json:"required_action"` 12 | KnownRansomwareCampaignUse string `json:"knownRansomwareCampaignUse"` 13 | 14 | CVE []string `json:"cve"` 15 | 16 | VulnCheckXDB []xdb `json:"vulncheck_xdb"` 17 | VulnCheckReportedExploitation []reportedExploit `json:"vulncheck_reported_exploitation"` 18 | 19 | DueDate *time.Time `json:"dueDate,omitempty"` 20 | CisaDateAdded *time.Time `json:"cisa_date_added,omitempty"` 21 | DateAdded time.Time `json:"date_added"` 22 | } 23 | 24 | type reportedExploit struct { 25 | URL string `json:"url"` 26 | DateAdded time.Time `json:"date_added"` 27 | } 28 | 29 | type xdb struct { 30 | XDBID string `json:"xdb_id"` 31 | XDBURL string `json:"xdb_url"` 32 | DateAdded time.Time `json:"date_added"` 33 | ExploitType string `json:"exploit_type"` 34 | CloneSSHURL string `json:"clone_ssh_url"` 35 | } 36 | -------------------------------------------------------------------------------- /fetcher/vulncheck/vulncheck.go: -------------------------------------------------------------------------------- 1 | package vulncheck 2 | 3 | import ( 4 | "archive/tar" 5 | "context" 6 | "encoding/json" 7 | "io" 8 | "io/fs" 9 | "os" 10 | "os/exec" 11 | "path/filepath" 12 | "time" 13 | 14 | "github.com/klauspost/compress/zstd" 15 | ocispec "github.com/opencontainers/image-spec/specs-go/v1" 16 | "golang.org/x/xerrors" 17 | "oras.land/oras-go/v2" 18 | "oras.land/oras-go/v2/registry/remote" 19 | 20 | "github.com/vulsio/go-kev/models" 21 | ) 22 | 23 | // Fetch : 24 | func Fetch() ([]models.VulnCheck, error) { 25 | dir, err := os.MkdirTemp("", "go-kev") 26 | if err != nil { 27 | return nil, xerrors.Errorf("Failed to create temp directory. err: %w", err) 28 | } 29 | defer os.RemoveAll(dir) 30 | 31 | if err := fetch(dir); err != nil { 32 | return nil, xerrors.Errorf("Failed to fetch vuls-data-raw-vulncheck-kev. err: %w", err) 33 | } 34 | 35 | var vs []models.VulnCheck 36 | 37 | if err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { 38 | if err != nil { 39 | return err 40 | } 41 | 42 | if d.IsDir() || filepath.Ext(path) != ".json" { 43 | return nil 44 | } 45 | 46 | f, err := os.Open(path) 47 | if err != nil { 48 | return xerrors.Errorf("Failed to open %s. err: %w", path, err) 49 | } 50 | defer f.Close() 51 | 52 | var v vulncheck 53 | if err := json.NewDecoder(f).Decode(&v); err != nil { 54 | return xerrors.Errorf("Failed to decode %s", path) 55 | } 56 | 57 | vs = append(vs, models.VulnCheck{ 58 | VendorProject: v.VendorProject, 59 | Product: v.Product, 60 | Description: v.Description, 61 | Name: v.Name, 62 | RequiredAction: v.RequiredAction, 63 | KnownRansomwareCampaignUse: v.KnownRansomwareCampaignUse, 64 | 65 | CVE: func() []models.VulnCheckCVE { 66 | cs := make([]models.VulnCheckCVE, 0, len(v.CVE)) 67 | for _, c := range v.CVE { 68 | cs = append(cs, models.VulnCheckCVE{ 69 | CveID: c, 70 | }) 71 | } 72 | return cs 73 | }(), 74 | 75 | VulnCheckXDB: func() []models.VulnCheckXDB { 76 | xs := make([]models.VulnCheckXDB, 0, len(v.VulnCheckXDB)) 77 | for _, x := range v.VulnCheckXDB { 78 | xs = append(xs, models.VulnCheckXDB{ 79 | XDBID: x.XDBID, 80 | XDBURL: x.XDBURL, 81 | DateAdded: func() time.Time { 82 | if x.DateAdded.Equal(time.Time{}) { 83 | return time.Date(1000, time.January, 1, 0, 0, 0, 0, time.UTC) 84 | } 85 | return x.DateAdded 86 | }(), 87 | ExploitType: x.ExploitType, 88 | CloneSSHURL: x.CloneSSHURL, 89 | }) 90 | } 91 | return xs 92 | }(), 93 | VulnCheckReportedExploitation: func() []models.VulnCheckReportedExploitation { 94 | es := make([]models.VulnCheckReportedExploitation, 0, len(v.VulnCheckReportedExploitation)) 95 | for _, e := range v.VulnCheckReportedExploitation { 96 | es = append(es, models.VulnCheckReportedExploitation{ 97 | URL: e.URL, 98 | DateAdded: func() time.Time { 99 | if e.DateAdded.Equal(time.Time{}) { 100 | return time.Date(1000, time.January, 1, 0, 0, 0, 0, time.UTC) 101 | } 102 | return e.DateAdded 103 | }(), 104 | }) 105 | } 106 | return es 107 | }(), 108 | 109 | DueDate: func() *time.Time { 110 | if v.DueDate == nil || (*v.DueDate).Equal(time.Time{}) { 111 | return nil 112 | } 113 | return v.DueDate 114 | }(), 115 | CisaDateAdded: func() *time.Time { 116 | if v.CisaDateAdded == nil || (*v.CisaDateAdded).Equal(time.Time{}) { 117 | return nil 118 | } 119 | return v.CisaDateAdded 120 | }(), 121 | DateAdded: func() time.Time { 122 | if v.DateAdded.Equal(time.Time{}) { 123 | return time.Date(1000, time.January, 1, 0, 0, 0, 0, time.UTC) 124 | } 125 | return v.DateAdded 126 | }(), 127 | }) 128 | 129 | return nil 130 | }); err != nil { 131 | return nil, xerrors.Errorf("Failed to walk %s. err: %w", dir, err) 132 | } 133 | 134 | return vs, nil 135 | } 136 | 137 | func fetch(dir string) error { 138 | ctx := context.TODO() 139 | repo, err := remote.NewRepository("ghcr.io/vulsio/vuls-data-db:vuls-data-raw-vulncheck-kev") 140 | if err != nil { 141 | return xerrors.Errorf("Failed to create client for ghcr.io/vulsio/vuls-data-db:vuls-data-raw-vulncheck-kev. err: %w", err) 142 | } 143 | 144 | _, r, err := oras.Fetch(ctx, repo, repo.Reference.Reference, oras.DefaultFetchOptions) 145 | if err != nil { 146 | return xerrors.Errorf("Failed to fetch manifest. err: %w", err) 147 | } 148 | defer r.Close() 149 | 150 | var manifest ocispec.Manifest 151 | if err := json.NewDecoder(r).Decode(&manifest); err != nil { 152 | return xerrors.Errorf("Failed to decode manifest. err: %w", err) 153 | } 154 | 155 | l := func() *ocispec.Descriptor { 156 | for _, l := range manifest.Layers { 157 | if l.MediaType == "application/vnd.vulsio.vuls-data-db.dotgit.layer.v1.tar+zstd" { 158 | return &l 159 | } 160 | } 161 | return nil 162 | }() 163 | if l == nil { 164 | return xerrors.Errorf("Failed to find digest and filename from layers, actual layers: %#v", manifest.Layers) 165 | } 166 | 167 | r, err = repo.Fetch(ctx, *l) 168 | if err != nil { 169 | return xerrors.Errorf("Failed to fetch content. err: %w", err) 170 | } 171 | defer r.Close() 172 | 173 | zr, err := zstd.NewReader(r) 174 | if err != nil { 175 | return xerrors.Errorf("Failed to new zstd reader. err: %w", err) 176 | } 177 | defer zr.Close() 178 | 179 | tr := tar.NewReader(zr) 180 | for { 181 | hdr, err := tr.Next() 182 | if err == io.EOF { 183 | break 184 | } 185 | if err != nil { 186 | return xerrors.Errorf("Failed to next tar reader. err: %w", err) 187 | } 188 | 189 | p := filepath.Join(dir, hdr.Name) 190 | 191 | switch hdr.Typeflag { 192 | case tar.TypeDir: 193 | if err := os.MkdirAll(p, 0755); err != nil { 194 | return xerrors.Errorf("Failed to mkdir %s. err: %w", p, err) 195 | } 196 | case tar.TypeReg: 197 | if err := os.MkdirAll(filepath.Dir(p), 0755); err != nil { 198 | return xerrors.Errorf("Failed to mkdir %s. err: %w", p, err) 199 | } 200 | 201 | if err := func() error { 202 | f, err := os.Create(p) 203 | if err != nil { 204 | return xerrors.Errorf("Failed to create %s. err: %w", p, err) 205 | } 206 | defer f.Close() 207 | 208 | if _, err := io.Copy(f, tr); err != nil { 209 | return xerrors.Errorf("Failed to copy to %s. err: %w", p, err) 210 | } 211 | 212 | return nil 213 | }(); err != nil { 214 | return xerrors.Errorf("Failed to create %s. err: %w", p, err) 215 | } 216 | } 217 | } 218 | 219 | cmd := exec.Command("git", "-C", filepath.Join(dir, "vuls-data-raw-vulncheck-kev"), "restore", ".") 220 | if err := cmd.Run(); err != nil { 221 | return xerrors.Errorf("Failed to exec %q. err: %w", cmd.String(), err) 222 | } 223 | 224 | return nil 225 | } 226 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/vulsio/go-kev 2 | 3 | go 1.24 4 | 5 | require ( 6 | github.com/alicebob/miniredis/v2 v2.34.0 7 | github.com/cheggaaa/pb/v3 v3.1.7 8 | github.com/glebarez/sqlite v1.11.0 9 | github.com/go-redis/redis/v8 v8.11.5 10 | github.com/google/go-cmp v0.7.0 11 | github.com/inconshreveable/log15 v3.0.0-testing.5+incompatible 12 | github.com/klauspost/compress v1.18.0 13 | github.com/labstack/echo/v4 v4.13.3 14 | github.com/mitchellh/go-homedir v1.1.0 15 | github.com/opencontainers/image-spec v1.1.1 16 | github.com/parnurzeal/gorequest v0.2.16 17 | github.com/spf13/cobra v1.9.1 18 | github.com/spf13/viper v1.20.0 19 | golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 20 | gorm.io/driver/mysql v1.5.5 21 | gorm.io/driver/postgres v1.5.7 22 | gorm.io/gorm v1.25.7 23 | oras.land/oras-go/v2 v2.5.0 24 | ) 25 | 26 | require ( 27 | github.com/VividCortex/ewma v1.2.0 // indirect 28 | github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect 29 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 30 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 31 | github.com/dustin/go-humanize v1.0.1 // indirect 32 | github.com/elazarl/goproxy v0.0.0-20230731152917-f99041a5c027 // indirect 33 | github.com/fatih/color v1.18.0 // indirect 34 | github.com/fsnotify/fsnotify v1.8.0 // indirect 35 | github.com/glebarez/go-sqlite v1.21.2 // indirect 36 | github.com/go-sql-driver/mysql v1.7.1 // indirect 37 | github.com/go-stack/stack v1.8.1 // indirect 38 | github.com/go-viper/mapstructure/v2 v2.2.1 // indirect 39 | github.com/google/uuid v1.6.0 // indirect 40 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 41 | github.com/jackc/pgpassfile v1.0.0 // indirect 42 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect 43 | github.com/jackc/pgx/v5 v5.5.4 // indirect 44 | github.com/jackc/puddle/v2 v2.2.1 // indirect 45 | github.com/jinzhu/inflection v1.0.0 // indirect 46 | github.com/jinzhu/now v1.1.5 // indirect 47 | github.com/labstack/gommon v0.4.2 // indirect 48 | github.com/mattn/go-colorable v0.1.14 // indirect 49 | github.com/mattn/go-isatty v0.0.20 // indirect 50 | github.com/mattn/go-runewidth v0.0.16 // indirect 51 | github.com/opencontainers/go-digest v1.0.0 // indirect 52 | github.com/pelletier/go-toml/v2 v2.2.3 // indirect 53 | github.com/pkg/errors v0.9.1 // indirect 54 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect 55 | github.com/rivo/uniseg v0.4.7 // indirect 56 | github.com/sagikazarmark/locafero v0.7.0 // indirect 57 | github.com/smartystreets/goconvey v1.8.0 // indirect 58 | github.com/sourcegraph/conc v0.3.0 // indirect 59 | github.com/spf13/afero v1.12.0 // indirect 60 | github.com/spf13/cast v1.7.1 // indirect 61 | github.com/spf13/pflag v1.0.6 // indirect 62 | github.com/subosito/gotenv v1.6.0 // indirect 63 | github.com/valyala/bytebufferpool v1.0.0 // indirect 64 | github.com/valyala/fasttemplate v1.2.2 // indirect 65 | github.com/yuin/gopher-lua v1.1.1 // indirect 66 | go.uber.org/atomic v1.9.0 // indirect 67 | go.uber.org/multierr v1.9.0 // indirect 68 | golang.org/x/crypto v0.35.0 // indirect 69 | golang.org/x/net v0.36.0 // indirect 70 | golang.org/x/sync v0.11.0 // indirect 71 | golang.org/x/sys v0.30.0 // indirect 72 | golang.org/x/term v0.29.0 // indirect 73 | golang.org/x/text v0.22.0 // indirect 74 | golang.org/x/time v0.8.0 // indirect 75 | gopkg.in/yaml.v3 v3.0.1 // indirect 76 | modernc.org/libc v1.22.5 // indirect 77 | modernc.org/mathutil v1.5.0 // indirect 78 | modernc.org/memory v1.5.0 // indirect 79 | modernc.org/sqlite v1.23.1 // indirect 80 | moul.io/http2curl v1.0.0 // indirect 81 | ) 82 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= 2 | github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= 3 | github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE= 4 | github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= 5 | github.com/alicebob/miniredis/v2 v2.34.0 h1:mBFWMaJSNL9RwdGRyEDoAAv8OQc5UlEhLDQggTglU/0= 6 | github.com/alicebob/miniredis/v2 v2.34.0/go.mod h1:kWShP4b58T1CW0Y5dViCd5ztzrDqRWqM3nksiyXk5s8= 7 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 8 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 9 | github.com/cheggaaa/pb/v3 v3.1.7 h1:2FsIW307kt7A/rz/ZI2lvPO+v3wKazzE4K/0LtTWsOI= 10 | github.com/cheggaaa/pb/v3 v3.1.7/go.mod h1:/Ji89zfVPeC/u5j8ukD0MBPHt2bzTYp74lQ7KlgFWTQ= 11 | github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= 12 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 13 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 14 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 15 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= 16 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= 17 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 18 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 19 | github.com/elazarl/goproxy v0.0.0-20230731152917-f99041a5c027 h1:1L0aalTpPz7YlMxETKpmQoWMBkeiuorElZIXoNmgiPE= 20 | github.com/elazarl/goproxy v0.0.0-20230731152917-f99041a5c027/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= 21 | github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= 22 | github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= 23 | github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= 24 | github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= 25 | github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= 26 | github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= 27 | github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= 28 | github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= 29 | github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= 30 | github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= 31 | github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= 32 | github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= 33 | github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= 34 | github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= 35 | github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= 36 | github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= 37 | github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= 38 | github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= 39 | github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= 40 | github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= 41 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 42 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 43 | github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= 44 | github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= 45 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 46 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 47 | github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= 48 | github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= 49 | github.com/inconshreveable/log15 v3.0.0-testing.5+incompatible h1:VryeOTiaZfAzwx8xBcID1KlJCeoWSIpsNbSk+/D2LNk= 50 | github.com/inconshreveable/log15 v3.0.0-testing.5+incompatible/go.mod h1:cOaXtrgN4ScfRrD9Bre7U1thNq5RtJ8ZoP4iXVGRj6o= 51 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 52 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 53 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= 54 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= 55 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= 56 | github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= 57 | github.com/jackc/pgx/v5 v5.5.4 h1:Xp2aQS8uXButQdnCMWNmvx6UysWQQC+u1EoizjguY+8= 58 | github.com/jackc/pgx/v5 v5.5.4/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= 59 | github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= 60 | github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= 61 | github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= 62 | github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= 63 | github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= 64 | github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= 65 | github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= 66 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 67 | github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= 68 | github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= 69 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 70 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 71 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 72 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 73 | github.com/labstack/echo/v4 v4.13.3 h1:pwhpCPrTl5qry5HRdM5FwdXnhXSLSY+WE+YQSeCaafY= 74 | github.com/labstack/echo/v4 v4.13.3/go.mod h1:o90YNEeQWjDozo584l7AwhJMHN0bOC4tAfg+Xox9q5g= 75 | github.com/labstack/gommon v0.4.2 h1:F8qTUNXgG1+6WQmqoUWnz8WiEU60mXVVw0P4ht1WRA0= 76 | github.com/labstack/gommon v0.4.2/go.mod h1:QlUFxVM+SNXhDL/Z7YhocGIBYOiwB0mXm1+1bAPHPyU= 77 | github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= 78 | github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= 79 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 80 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 81 | github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= 82 | github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 83 | github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= 84 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 85 | github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= 86 | github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= 87 | github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= 88 | github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= 89 | github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= 90 | github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= 91 | github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= 92 | github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= 93 | github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= 94 | github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= 95 | github.com/parnurzeal/gorequest v0.2.16 h1:T/5x+/4BT+nj+3eSknXmCTnEVGSzFzPGdpqmUVVZXHQ= 96 | github.com/parnurzeal/gorequest v0.2.16/go.mod h1:3Kh2QUMJoqw3icWAecsyzkpY7UzRfDhbRdTjtNwNiUE= 97 | github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= 98 | github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= 99 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 100 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 101 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 102 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 103 | github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= 104 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= 105 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= 106 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 107 | github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= 108 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 109 | github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= 110 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= 111 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 112 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 113 | github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= 114 | github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= 115 | github.com/smartystreets/assertions v1.13.1 h1:Ef7KhSmjZcK6AVf9YbJdvPYG9avaF0ZxudX+ThRdWfU= 116 | github.com/smartystreets/assertions v1.13.1/go.mod h1:cXr/IwVfSo/RbCSPhoAPv73p3hlSdrBH/b3SdnW/LMY= 117 | github.com/smartystreets/goconvey v1.8.0 h1:Oi49ha/2MURE0WexF052Z0m+BNSGirfjg5RL+JXWq3w= 118 | github.com/smartystreets/goconvey v1.8.0/go.mod h1:EdX8jtrTIj26jmjCOVNMVSIYAtgexqXKHOXW2Dx9JLg= 119 | github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= 120 | github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= 121 | github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= 122 | github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= 123 | github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= 124 | github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= 125 | github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= 126 | github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= 127 | github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= 128 | github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 129 | github.com/spf13/viper v1.20.0 h1:zrxIyR3RQIOsarIrgL8+sAvALXul9jeEPa06Y0Ph6vY= 130 | github.com/spf13/viper v1.20.0/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= 131 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 132 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 133 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 134 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 135 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 136 | github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= 137 | github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= 138 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 139 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 140 | github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= 141 | github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= 142 | github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= 143 | github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= 144 | go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= 145 | go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 146 | go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= 147 | go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= 148 | golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= 149 | golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= 150 | golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= 151 | golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= 152 | golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= 153 | golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 154 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 155 | golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= 156 | golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 157 | golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= 158 | golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= 159 | golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= 160 | golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= 161 | golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= 162 | golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= 163 | golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= 164 | golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= 165 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 166 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 167 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 168 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 169 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 170 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 171 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 172 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 173 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 174 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 175 | gorm.io/driver/mysql v1.5.5 h1:WxklwX6FozMs1gk9yVadxGfjGiJjrBKPvIIvYZOMyws= 176 | gorm.io/driver/mysql v1.5.5/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= 177 | gorm.io/driver/postgres v1.5.7 h1:8ptbNJTDbEmhdr62uReG5BGkdQyeasu/FZHxI0IMGnM= 178 | gorm.io/driver/postgres v1.5.7/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA= 179 | gorm.io/gorm v1.25.7 h1:VsD6acwRjz2zFxGO50gPO6AkNs7KKnvfzUjHQhZDz/A= 180 | gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= 181 | modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= 182 | modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= 183 | modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= 184 | modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= 185 | modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= 186 | modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= 187 | modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= 188 | modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= 189 | moul.io/http2curl v1.0.0 h1:6XwpyZOYsgZJrU8exnG87ncVkU1FVCcTRpwzOkTDUi8= 190 | moul.io/http2curl v1.0.0/go.mod h1:f6cULg+e4Md/oW1cYmwW4IWQOVl2lGbmCNGOHvzX2kE= 191 | oras.land/oras-go/v2 v2.5.0 h1:o8Me9kLY74Vp5uw07QXPiitjsw7qNXi8Twd+19Zf02c= 192 | oras.land/oras-go/v2 v2.5.0/go.mod h1:z4eisnLP530vwIOUOJeBIj0aGI0L1C3d53atvCBqZHg= 193 | -------------------------------------------------------------------------------- /integration/.gitignore: -------------------------------------------------------------------------------- 1 | go-kev.* 2 | *.sqlite3 3 | diff -------------------------------------------------------------------------------- /integration/README.md: -------------------------------------------------------------------------------- 1 | # Test Script For go-kev 2 | Documentation on testing for developers 3 | 4 | ## Getting Started 5 | ```terminal 6 | $ pip install -r requirements.txt 7 | ``` 8 | 9 | ## Run test 10 | Use `127.0.0.1:1325` and `127.0.0.1:1326` to diff the server mode between the latest tag and your working branch. 11 | 12 | If you have prepared the two addresses yourself, you can use the following Python script. 13 | ```terminal 14 | $ python diff_server_mode.py --help 15 | usage: diff_server_mode.py [-h] [--sample_rate SAMPLE_RATE] [--debug | --no-debug] {cves,multi-cves} 16 | 17 | positional arguments: 18 | {cves,multi-cves} Specify the mode to test. 19 | 20 | optional arguments: 21 | -h, --help show this help message and exit 22 | --sample_rate SAMPLE_RATE 23 | Adjust the rate of data used for testing (len(test_data) * sample_rate) 24 | --debug, --no-debug print debug message 25 | ``` 26 | 27 | [GNUmakefile](../GNUmakefile) has some tasks for testing. 28 | Please run it in the top directory of the go-kev repository. 29 | 30 | - build-integration: create the go-kev binaries needed for testing 31 | - clean-integration: delete the go-kev process, binary, and docker container used in the test 32 | - fetch-rdb: fetch data for RDB for testing 33 | - fetch-redis: fetch data for Redis for testing 34 | - diff-cveid: Run tests for CVE ID in server mode 35 | - diff-server-rdb: take the result difference of server mode using RDB 36 | - diff-server-redis: take the result difference of server mode using Redis 37 | - diff-server-rdb-redis: take the difference in server mode results between RDB and Redis 38 | -------------------------------------------------------------------------------- /integration/cveid.txt: -------------------------------------------------------------------------------- 1 | CVE-2010-5326 2 | CVE-2012-0158 3 | CVE-2012-3152 4 | CVE-2014-1812 5 | CVE-2015-1641 6 | CVE-2015-4852 7 | CVE-2016-0167 8 | CVE-2016-0185 9 | CVE-2016-3235 10 | CVE-2016-3643 11 | CVE-2016-3715 12 | CVE-2016-3718 13 | CVE-2016-3976 14 | CVE-2016-4437 15 | CVE-2016-7255 16 | CVE-2016-9563 17 | CVE-2017-0143 18 | CVE-2017-0199 19 | CVE-2017-11774 20 | CVE-2017-11882 21 | CVE-2017-16651 22 | CVE-2017-5638 23 | CVE-2017-6327 24 | CVE-2017-7269 25 | CVE-2017-7481 26 | CVE-2017-8759 27 | CVE-2017-9248 28 | CVE-2017-9805 29 | CVE-2017-9822 30 | CVE-2018-0171 31 | CVE-2018-0296 32 | CVE-2018-0798 33 | CVE-2018-0802 34 | CVE-2018-11776 35 | CVE-2018-13379 36 | CVE-2018-14558 37 | CVE-2018-15811 38 | CVE-2018-15961 39 | CVE-2018-18325 40 | CVE-2018-20062 41 | CVE-2018-2380 42 | CVE-2018-4878 43 | CVE-2018-4939 44 | CVE-2018-6789 45 | CVE-2018-7600 46 | CVE-2018-8653 47 | CVE-2019-0211 48 | CVE-2019-0541 49 | CVE-2019-0604 50 | CVE-2019-0708 51 | CVE-2019-0797 52 | CVE-2019-0803 53 | CVE-2019-0808 54 | CVE-2019-0859 55 | CVE-2019-0863 56 | CVE-2019-11510 57 | CVE-2019-11539 58 | CVE-2019-11580 59 | CVE-2019-11634 60 | CVE-2019-1214 61 | CVE-2019-1215 62 | CVE-2019-13608 63 | CVE-2019-1367 64 | CVE-2019-1429 65 | CVE-2019-15752 66 | CVE-2019-15949 67 | CVE-2019-16256 68 | CVE-2019-1653 69 | CVE-2019-16759 70 | CVE-2019-17026 71 | CVE-2019-17558 72 | CVE-2019-18187 73 | CVE-2019-18935 74 | CVE-2019-18988 75 | CVE-2019-19356 76 | CVE-2019-19781 77 | CVE-2019-20085 78 | CVE-2019-2215 79 | CVE-2019-3396 80 | CVE-2019-3398 81 | CVE-2019-4716 82 | CVE-2019-5544 83 | CVE-2019-5591 84 | CVE-2019-6223 85 | CVE-2019-8394 86 | CVE-2019-9082 87 | CVE-2019-9978 88 | CVE-2020-0041 89 | CVE-2020-0069 90 | CVE-2020-0601 91 | CVE-2020-0646 92 | CVE-2020-0674 93 | CVE-2020-0683 94 | CVE-2020-0688 95 | CVE-2020-0878 96 | CVE-2020-0938 97 | CVE-2020-0968 98 | CVE-2020-0986 99 | CVE-2020-10148 100 | CVE-2020-10181 101 | CVE-2020-10189 102 | CVE-2020-10199 103 | CVE-2020-1020 104 | CVE-2020-10221 105 | CVE-2020-1040 106 | CVE-2020-1054 107 | CVE-2020-10987 108 | CVE-2020-1147 109 | CVE-2020-11651 110 | CVE-2020-11652 111 | CVE-2020-11738 112 | CVE-2020-12271 113 | CVE-2020-12812 114 | CVE-2020-1350 115 | CVE-2020-1380 116 | CVE-2020-1464 117 | CVE-2020-1472 118 | CVE-2020-14750 119 | CVE-2020-14871 120 | CVE-2020-14882 121 | CVE-2020-14883 122 | CVE-2020-15505 123 | CVE-2020-15999 124 | CVE-2020-16009 125 | CVE-2020-16010 126 | CVE-2020-16013 127 | CVE-2020-16017 128 | CVE-2020-16846 129 | CVE-2020-17087 130 | CVE-2020-17144 131 | CVE-2020-17496 132 | CVE-2020-17530 133 | CVE-2020-24557 134 | CVE-2020-25213 135 | CVE-2020-25506 136 | CVE-2020-2555 137 | CVE-2020-26919 138 | CVE-2020-27930 139 | CVE-2020-27932 140 | CVE-2020-27950 141 | CVE-2020-29557 142 | CVE-2020-29583 143 | CVE-2020-3118 144 | CVE-2020-3161 145 | CVE-2020-3452 146 | CVE-2020-3566 147 | CVE-2020-3569 148 | CVE-2020-3580 149 | CVE-2020-3950 150 | CVE-2020-3952 151 | CVE-2020-3992 152 | CVE-2020-4006 153 | CVE-2020-4427 154 | CVE-2020-4428 155 | CVE-2020-4430 156 | CVE-2020-5735 157 | CVE-2020-5847 158 | CVE-2020-5849 159 | CVE-2020-5902 160 | CVE-2020-6207 161 | CVE-2020-6287 162 | CVE-2020-6418 163 | CVE-2020-6819 164 | CVE-2020-6820 165 | CVE-2020-7961 166 | CVE-2020-8193 167 | CVE-2020-8195 168 | CVE-2020-8196 169 | CVE-2020-8243 170 | CVE-2020-8260 171 | CVE-2020-8467 172 | CVE-2020-8468 173 | CVE-2020-8515 174 | CVE-2020-8599 175 | CVE-2020-8644 176 | CVE-2020-8655 177 | CVE-2020-8657 178 | CVE-2020-9818 179 | CVE-2020-9819 180 | CVE-2020-9859 181 | CVE-2021-1497 182 | CVE-2021-1498 183 | CVE-2021-1647 184 | CVE-2021-1675 185 | CVE-2021-1732 186 | CVE-2021-1782 187 | CVE-2021-1870 188 | CVE-2021-1871 189 | CVE-2021-1879 190 | CVE-2021-1905 191 | CVE-2021-1906 192 | CVE-2021-20016 193 | CVE-2021-20021 194 | CVE-2021-20022 195 | CVE-2021-20023 196 | CVE-2021-20090 197 | CVE-2021-21017 198 | CVE-2021-21148 199 | CVE-2021-21166 200 | CVE-2021-21193 201 | CVE-2021-21206 202 | CVE-2021-21220 203 | CVE-2021-21224 204 | CVE-2021-21972 205 | CVE-2021-21985 206 | CVE-2021-22005 207 | CVE-2021-22205 208 | CVE-2021-22502 209 | CVE-2021-22506 210 | CVE-2021-22893 211 | CVE-2021-22894 212 | CVE-2021-22899 213 | CVE-2021-22900 214 | CVE-2021-22986 215 | CVE-2021-23874 216 | CVE-2021-26084 217 | CVE-2021-26411 218 | CVE-2021-26855 219 | CVE-2021-26857 220 | CVE-2021-26858 221 | CVE-2021-27059 222 | CVE-2021-27065 223 | CVE-2021-27085 224 | CVE-2021-27101 225 | CVE-2021-27102 226 | CVE-2021-27103 227 | CVE-2021-27104 228 | CVE-2021-27561 229 | CVE-2021-27562 230 | CVE-2021-28310 231 | CVE-2021-28550 232 | CVE-2021-28663 233 | CVE-2021-28664 234 | CVE-2021-30116 235 | CVE-2021-30551 236 | CVE-2021-30554 237 | CVE-2021-30563 238 | CVE-2021-30632 239 | CVE-2021-30633 240 | CVE-2021-30657 241 | CVE-2021-30661 242 | CVE-2021-30663 243 | CVE-2021-30665 244 | CVE-2021-30666 245 | CVE-2021-30713 246 | CVE-2021-30761 247 | CVE-2021-30762 248 | CVE-2021-30807 249 | CVE-2021-30858 250 | CVE-2021-30860 251 | CVE-2021-30869 252 | CVE-2021-31199 253 | CVE-2021-31201 254 | CVE-2021-31207 255 | CVE-2021-31755 256 | CVE-2021-31955 257 | CVE-2021-31956 258 | CVE-2021-31979 259 | CVE-2021-33739 260 | CVE-2021-33742 261 | CVE-2021-33771 262 | CVE-2021-34448 263 | CVE-2021-34473 264 | CVE-2021-34523 265 | CVE-2021-34527 266 | CVE-2021-35211 267 | CVE-2021-35395 268 | CVE-2021-35464 269 | CVE-2021-36741 270 | CVE-2021-36742 271 | CVE-2021-36942 272 | CVE-2021-36948 273 | CVE-2021-36955 274 | CVE-2021-37973 275 | CVE-2021-37975 276 | CVE-2021-37976 277 | CVE-2021-38000 278 | CVE-2021-38003 279 | CVE-2021-38645 280 | CVE-2021-38647 281 | CVE-2021-38648 282 | CVE-2021-38649 283 | CVE-2021-40444 284 | CVE-2021-40539 285 | CVE-2021-41773 286 | CVE-2021-42013 287 | CVE-2021-42258 -------------------------------------------------------------------------------- /integration/diff_server_mode.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import logging 3 | from typing import Tuple 4 | from deepdiff import DeepDiff 5 | import requests 6 | from requests.adapters import HTTPAdapter 7 | from urllib3.util import Retry 8 | from urllib.parse import quote 9 | import pprint 10 | from concurrent.futures import ThreadPoolExecutor 11 | import os 12 | import random 13 | import math 14 | import json 15 | import shutil 16 | import time 17 | import uuid 18 | 19 | 20 | def diff_response(args: Tuple[str, list[str]]): 21 | session = requests.Session() 22 | retries = Retry(total=5, 23 | backoff_factor=1, 24 | status_forcelist=[503, 504]) 25 | session.mount("http://", HTTPAdapter(max_retries=retries)) 26 | 27 | # Endpoint 28 | # GET /cves/:id 29 | # POST /multi-cves 30 | path = "" 31 | if args[0] in ['cves']: 32 | path = f'{args[0]}/{args[1][0]}' 33 | try: 34 | response_old = requests.get( 35 | f'http://127.0.0.1:1325/{path}', timeout=(3.0, 10.0)).json() 36 | response_new = requests.get( 37 | f'http://127.0.0.1:1326/{path}', timeout=(3.0, 10.0)).json() 38 | except requests.ConnectionError as e: 39 | logger.error( 40 | f'Failed to Connection..., err: {e}, {pprint.pformat({"args": args, "path": path}, indent=2)}') 41 | exit(1) 42 | except requests.ReadTimeout as e: 43 | logger.warning( 44 | f'Failed to ReadTimeout..., err: {e}, {pprint.pformat({"args": args, "path": path}, indent=2)}') 45 | except Exception as e: 46 | logger.error( 47 | f'Failed to GET request..., err: {e}, {pprint.pformat({"args": args, "path": path}, indent=2)}') 48 | exit(1) 49 | 50 | diff = DeepDiff(response_old, response_new, ignore_order=True) 51 | if diff != {}: 52 | logger.warning( 53 | f'There is a difference between old and new(or RDB and Redis):\n {pprint.pformat({"args": args, "path": path}, indent=2)}') 54 | 55 | diff_path = f'integration/diff/{args[0]}/{args[1]}' 56 | with open(f'{diff_path}.old', 'w') as w: 57 | w.write(json.dumps(response_old, indent=4)) 58 | with open(f'{diff_path}.new', 'w') as w: 59 | w.write(json.dumps(response_new, indent=4)) 60 | else: 61 | path = args[0] 62 | 63 | k = math.ceil(len(args[1])/5) 64 | for _ in range(5): 65 | payload = {"args": random.sample(args[1], k)} 66 | try: 67 | response_old = session.post( 68 | f'http://127.0.0.1:1325/{path}', data=json.dumps(payload), headers={'content-type': 'application/json'}, timeout=(3.0, 10.0)).json() 69 | response_new = session.post( 70 | f'http://127.0.0.1:1326/{path}', data=json.dumps(payload), headers={'content-type': 'application/json'}, timeout=(3.0, 10.0)).json() 71 | except requests.ConnectionError as e: 72 | logger.error( 73 | f'Failed to Connection..., err: {e}, {pprint.pformat({"args": args, "path": path}, indent=2)}') 74 | exit(1) 75 | except requests.ReadTimeout as e: 76 | logger.warning( 77 | f'Failed to ReadTimeout..., err: {e}, {pprint.pformat({"args": args, "path": path}, indent=2)}') 78 | except Exception as e: 79 | logger.error( 80 | f'Failed to GET request..., err: {e}, {pprint.pformat({"args": args, "path": path}, indent=2)}') 81 | exit(1) 82 | 83 | diff = DeepDiff(response_old, response_new, ignore_order=True) 84 | if diff != {}: 85 | logger.warning( 86 | f'There is a difference between old and new(or RDB and Redis):\n {pprint.pformat({"args": args, "path": path}, indent=2)}') 87 | 88 | title = uuid.uuid4() 89 | diff_path = f'integration/diff/{args[0]}/{title}' 90 | with open(f'{diff_path}.old', 'w') as w: 91 | w.write(json.dumps( 92 | {'args': args, 'response': response_old}, indent=4)) 93 | with open(f'{diff_path}.new', 'w') as w: 94 | w.write(json.dumps( 95 | {'args': args, 'response': response_new}, indent=4)) 96 | 97 | 98 | parser = argparse.ArgumentParser() 99 | parser.add_argument('mode', choices=['cves', 'multi-cves'], 100 | help='Specify the mode to test.') 101 | parser.add_argument("--sample_rate", type=float, default=0.01, 102 | help="Adjust the rate of data used for testing (len(test_data) * sample_rate)") 103 | parser.add_argument( 104 | '--debug', action=argparse.BooleanOptionalAction, help='print debug message') 105 | args = parser.parse_args() 106 | 107 | logger = logging.getLogger(__name__) 108 | stream_handler = logging.StreamHandler() 109 | 110 | if args.debug: 111 | logger.setLevel(logging.DEBUG) 112 | stream_handler.setLevel(logging.DEBUG) 113 | else: 114 | logger.setLevel(logging.INFO) 115 | stream_handler.setLevel(logging.INFO) 116 | 117 | formatter = logging.Formatter( 118 | '%(levelname)s[%(asctime)s] %(message)s', "%m-%d|%H:%M:%S") 119 | stream_handler.setFormatter(formatter) 120 | logger.addHandler(stream_handler) 121 | 122 | logger.info( 123 | f'start server mode test(mode: {args.mode})') 124 | 125 | logger.info('check the communication with the server') 126 | for i in range(5): 127 | try: 128 | if requests.get('http://127.0.0.1:1325/health').status_code == requests.codes.ok and requests.get('http://127.0.0.1:1326/health').status_code == requests.codes.ok: 129 | logger.info('communication with the server has been confirmed') 130 | break 131 | except Exception: 132 | pass 133 | time.sleep(1) 134 | else: 135 | logger.error('Failed to communicate with server') 136 | exit(1) 137 | 138 | list_path = None 139 | if args.mode in ['cves', 'multi-cves']: 140 | list_path = f"integration/cveid.txt" 141 | if not os.path.isfile(list_path): 142 | logger.error(f'Failed to find list path..., list_path: {list_path}') 143 | exit(1) 144 | 145 | diff_path = f'integration/diff/{args.mode}' 146 | if os.path.exists(diff_path): 147 | shutil.rmtree(diff_path) 148 | os.makedirs(diff_path, exist_ok=True) 149 | 150 | with open(list_path) as f: 151 | list = [s.strip() for s in f.readlines()] 152 | list = random.sample(list, math.ceil(len(list) * args.sample_rate)) 153 | if args.mode in ['multi-cves']: 154 | diff_response((args.mode, list)) 155 | else: 156 | with ThreadPoolExecutor() as executor: 157 | ins = ((args.mode, [e]) for e in list) 158 | executor.map(diff_response, ins) 159 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/vulsio/go-kev/commands" 8 | ) 9 | 10 | func main() { 11 | if err := commands.RootCmd.Execute(); err != nil { 12 | fmt.Fprintln(os.Stderr, err) 13 | os.Exit(1) 14 | } 15 | os.Exit(0) 16 | } 17 | -------------------------------------------------------------------------------- /models/models.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "time" 5 | 6 | "gorm.io/gorm" 7 | ) 8 | 9 | // LatestSchemaVersion manages the Schema version used in the latest go-kev. 10 | const LatestSchemaVersion = 2 11 | 12 | // FetchMeta has meta information 13 | type FetchMeta struct { 14 | gorm.Model `json:"-"` 15 | GoKEVRevision string 16 | SchemaVersion uint 17 | LastFetchedAt time.Time 18 | } 19 | 20 | // OutDated checks whether last fetched feed is out dated 21 | func (f FetchMeta) OutDated() bool { 22 | return f.SchemaVersion != LatestSchemaVersion 23 | } 24 | 25 | // KEVuln : Known Exploited Vulnerabilities 26 | type KEVuln struct { 27 | ID int64 `json:"-"` 28 | CveID string `gorm:"type:varchar(255);index:idx_kev_cve_id" json:"cveID"` 29 | VendorProject string `gorm:"type:varchar(255)" json:"vendorProject"` 30 | Product string `gorm:"type:varchar(255)" json:"product"` 31 | VulnerabilityName string `gorm:"type:varchar(255)" json:"vulnerabilityName"` 32 | DateAdded time.Time `json:"dateAdded"` 33 | ShortDescription string `gorm:"type:text" json:"shortDescription"` 34 | RequiredAction string `gorm:"type:text" json:"requiredAction"` 35 | DueDate time.Time `json:"dueDate"` 36 | KnownRansomwareCampaignUse string `gorm:"type:varchar(255)" json:"knownRansomwareCampaignUse"` 37 | Notes string `gorm:"type:text" json:"notes"` 38 | } 39 | 40 | // VulnCheck : https://docs.vulncheck.com/community/vulncheck-kev/schema 41 | type VulnCheck struct { 42 | ID int64 `json:"-"` 43 | VendorProject string `gorm:"type:varchar(255)" json:"vendorProject"` 44 | Product string `gorm:"type:varchar(255)" json:"product"` 45 | Description string `gorm:"type:text" json:"shortDescription"` 46 | Name string `gorm:"type:varchar(255)" json:"vulnerabilityName"` 47 | RequiredAction string `gorm:"type:text" json:"required_action"` 48 | KnownRansomwareCampaignUse string `gorm:"type:varchar(255)" json:"knownRansomwareCampaignUse"` 49 | 50 | CVE []VulnCheckCVE `json:"cve"` 51 | 52 | VulnCheckXDB []VulnCheckXDB `json:"vulncheck_xdb"` 53 | VulnCheckReportedExploitation []VulnCheckReportedExploitation `json:"vulncheck_reported_exploitation"` 54 | 55 | DueDate *time.Time `json:"dueDate,omitempty"` 56 | CisaDateAdded *time.Time `json:"cisa_date_added,omitempty"` 57 | DateAdded time.Time `json:"date_added"` 58 | } 59 | 60 | // VulnCheckCVE : 61 | type VulnCheckCVE struct { 62 | ID int64 `json:"-"` 63 | VulnCheckID uint `json:"-" gorm:"index:idx_vulncheck_cve"` 64 | CveID string `gorm:"type:varchar(255);index:idx_vulncheck_cve_cve_id" json:"cveID"` 65 | } 66 | 67 | // VulnCheckXDB : 68 | type VulnCheckXDB struct { 69 | ID int64 `json:"-"` 70 | VulnCheckID uint `json:"-" gorm:"index:idx_vulncheck_xdb"` 71 | XDBID string `gorm:"type:varchar(255)" json:"xdb_id"` 72 | XDBURL string `gorm:"type:varchar(255)" json:"xdb_url"` 73 | DateAdded time.Time `json:"date_added"` 74 | ExploitType string `gorm:"type:varchar(255)" json:"exploit_type"` 75 | CloneSSHURL string `gorm:"type:text" json:"clone_ssh_url"` 76 | } 77 | 78 | // VulnCheckReportedExploitation : 79 | type VulnCheckReportedExploitation struct { 80 | ID int64 `json:"-"` 81 | VulnCheckID uint `json:"-" gorm:"index:idx_vulncheck_reported_exploitation"` 82 | URL string `gorm:"type:text" json:"url"` 83 | DateAdded time.Time `json:"date_added"` 84 | } 85 | -------------------------------------------------------------------------------- /models/models_test.go: -------------------------------------------------------------------------------- 1 | package models 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func Test_FetchMeta(t *testing.T) { 8 | var tests = []struct { 9 | in FetchMeta 10 | outdated bool 11 | }{ 12 | { 13 | in: FetchMeta{ 14 | SchemaVersion: 0, 15 | }, 16 | outdated: true, 17 | }, 18 | { 19 | in: FetchMeta{ 20 | SchemaVersion: LatestSchemaVersion, 21 | }, 22 | outdated: false, 23 | }, 24 | } 25 | 26 | for i, tt := range tests { 27 | if aout := tt.in.OutDated(); tt.outdated != aout { 28 | t.Errorf("[%d] outdated expected: %#v\n actual: %#v\n", i, tt.outdated, aout) 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /server/server.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/inconshreveable/log15" 10 | "github.com/labstack/echo/v4" 11 | "github.com/labstack/echo/v4/middleware" 12 | "github.com/spf13/viper" 13 | "golang.org/x/xerrors" 14 | 15 | "github.com/vulsio/go-kev/db" 16 | ) 17 | 18 | // Start : 19 | func Start(logToFile bool, logDir string, driver db.DB) error { 20 | e := echo.New() 21 | e.Debug = viper.GetBool("debug") 22 | 23 | // Middleware 24 | e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{Output: os.Stderr})) 25 | e.Use(middleware.Recover()) 26 | 27 | // setup access logger 28 | if logToFile { 29 | logPath := filepath.Join(logDir, "access.log") 30 | f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) 31 | if err != nil { 32 | return xerrors.Errorf("Failed to open a log file: %s", err) 33 | } 34 | defer f.Close() 35 | e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{Output: f})) 36 | } 37 | 38 | // Routes 39 | e.GET("/health", health()) 40 | e.GET("/cves/:cve", getVulnByCveID(driver)) 41 | e.POST("/multi-cves", getVulnByMultiCveID(driver)) 42 | 43 | bindURL := fmt.Sprintf("%s:%s", viper.GetString("bind"), viper.GetString("port")) 44 | log15.Info("Listening...", "URL", bindURL) 45 | 46 | return e.Start(bindURL) 47 | } 48 | 49 | func health() echo.HandlerFunc { 50 | return func(context echo.Context) error { 51 | return context.String(http.StatusOK, "") 52 | } 53 | } 54 | 55 | func getVulnByCveID(driver db.DB) echo.HandlerFunc { 56 | return func(context echo.Context) (err error) { 57 | cve := context.Param("cve") 58 | log15.Debug("Params", "CVE", cve) 59 | 60 | r, err := driver.GetKEVByCveID(cve) 61 | if err != nil { 62 | return xerrors.Errorf("Failed to get KEV Info by CVE. err: %w", err) 63 | } 64 | return context.JSON(http.StatusOK, r) 65 | } 66 | } 67 | 68 | type param struct { 69 | Args []string `json:"args"` 70 | } 71 | 72 | func getVulnByMultiCveID(driver db.DB) echo.HandlerFunc { 73 | return func(context echo.Context) (err error) { 74 | cveIDs := param{} 75 | if err := context.Bind(&cveIDs); err != nil { 76 | return err 77 | } 78 | log15.Debug("Params", "CVEIDs", cveIDs.Args) 79 | 80 | r, err := driver.GetKEVByMultiCveID(cveIDs.Args) 81 | if err != nil { 82 | return xerrors.Errorf("Failed to get KEV Info by CVE. err: %w", err) 83 | } 84 | return context.JSON(http.StatusOK, r) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "runtime" 7 | 8 | "github.com/inconshreveable/log15" 9 | "github.com/parnurzeal/gorequest" 10 | "github.com/spf13/viper" 11 | "golang.org/x/xerrors" 12 | ) 13 | 14 | // GetDefaultLogDir : 15 | func GetDefaultLogDir() string { 16 | defaultLogDir := "/var/log/go-kev" 17 | if runtime.GOOS == "windows" { 18 | defaultLogDir = filepath.Join(os.Getenv("APPDATA"), "go-kev") 19 | } 20 | return defaultLogDir 21 | } 22 | 23 | // SetLogger : 24 | func SetLogger(logToFile bool, logDir string, debug, logJSON bool) error { 25 | stderrHandler := log15.StderrHandler 26 | logFormat := log15.LogfmtFormat() 27 | if logJSON { 28 | logFormat = log15.JsonFormatEx(false, true) 29 | stderrHandler = log15.StreamHandler(os.Stderr, logFormat) 30 | } 31 | 32 | lvlHandler := log15.LvlFilterHandler(log15.LvlInfo, stderrHandler) 33 | if debug { 34 | lvlHandler = log15.LvlFilterHandler(log15.LvlDebug, stderrHandler) 35 | } 36 | 37 | var handler log15.Handler 38 | if logToFile { 39 | if _, err := os.Stat(logDir); err != nil { 40 | if os.IsNotExist(err) { 41 | if err := os.Mkdir(logDir, 0700); err != nil { 42 | return xerrors.Errorf("Failed to create log directory. err: %w", err) 43 | } 44 | } else { 45 | return xerrors.Errorf("Failed to check log directory. err: %w", err) 46 | } 47 | } 48 | 49 | logPath := filepath.Join(logDir, "go-kev.log") 50 | if _, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644); err != nil { 51 | return xerrors.Errorf("Failed to open a log file. err: %w", err) 52 | } 53 | handler = log15.MultiHandler( 54 | log15.Must.FileHandler(logPath, logFormat), 55 | lvlHandler, 56 | ) 57 | } else { 58 | handler = lvlHandler 59 | } 60 | log15.Root().SetHandler(handler) 61 | return nil 62 | } 63 | 64 | // FetchURL returns HTTP response body 65 | func FetchURL(url string) ([]byte, error) { 66 | httpProxy := viper.GetString("http-proxy") 67 | 68 | resp, body, errs := gorequest.New().Proxy(httpProxy).Get(url).Type("text").EndBytes() 69 | if len(errs) > 0 || resp == nil || resp.StatusCode != 200 { 70 | return nil, xerrors.Errorf("HTTP error. url: %s, err: %v", url, errs) 71 | } 72 | return body, nil 73 | } 74 | 75 | // ToPtr returns a pointer to the value passed in 76 | func ToPtr[T any](v T) *T { 77 | return &v 78 | } 79 | --------------------------------------------------------------------------------