├── .dockerignore ├── .drone.yml ├── .drone.yml.sig ├── .env.example ├── .github └── workflows │ └── release.yaml ├── .gitignore ├── .goreleaser.yaml ├── .pre-commit-config.yaml ├── .secignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── TODO.md ├── cmd └── scanrepo │ └── main.go ├── diff.go ├── docker-compose.yml ├── elastalert ├── .gitignore ├── Dockerfile ├── Makefile ├── elastalert_config.yaml ├── elasticsearch_mappings.json ├── email.tmpl ├── modules │ ├── __init__.py │ └── govuknotify.py ├── requirements-to-freeze.txt ├── requirements.txt ├── rules │ └── new_violation.yaml ├── start-elastalert.sh └── tests │ ├── __init__.py │ └── test_govuknotify.py ├── github.go ├── github_test.go ├── go.mod ├── go.sum ├── handlers.go ├── handlers_test.go ├── kube ├── deployment.yml ├── ingress.yml ├── secrets.yml └── service.yml ├── log.go ├── main.go ├── middleware.go ├── middleware_test.go ├── rules └── gitrob.json ├── test └── fixtures │ ├── github_event_push.json │ ├── github_event_push_offenses.json │ ├── no_offenses.diff │ ├── nonsense.json │ └── offenses_x1.diff ├── testhelpers_test.go ├── vendor.conf └── version /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | kube -------------------------------------------------------------------------------- /.drone.yml: -------------------------------------------------------------------------------- 1 | pipeline: 2 | 3 | my-build: 4 | privileged: true 5 | image: docker:1.13 6 | environment: 7 | - DOCKER_HOST=tcp://127.0.0.1:2375 8 | commands: 9 | - docker build -t repo-security-scanner . 10 | when: 11 | event: push 12 | 13 | image_to_quay: 14 | image: docker:1.13 15 | environment: 16 | - DOCKER_HOST=tcp://127.0.0.1:2375 17 | commands: 18 | - docker login -u="ukhomeofficedigital+repo_security_scanner_bot" -p=${DOCKER_PASSWORD} quay.io 19 | - docker tag repo-security-scanner quay.io/ukhomeofficedigital/repo-security-scanner:${DRONE_COMMIT_SHA} 20 | - docker push quay.io/ukhomeofficedigital/repo-security-scanner:${DRONE_COMMIT_SHA} 21 | when: 22 | event: push 23 | 24 | deploy_to_dev: 25 | image: quay.io/ukhomeofficedigital/kd:latest 26 | environment: 27 | - KUBE_SERVER=https://kube-dev.dsp.notprod.homeoffice.gov.uk 28 | - KUBE_NAMESPACE=repo-security-scanner 29 | - INSECURE_SKIP_TLS_VERIFY=true 30 | commands: 31 | - cd kube 32 | - kd -f deployment.yml -f service.yml -f ingress.yml 33 | when: 34 | event: [deployment, push] 35 | 36 | services: 37 | dind: 38 | image: docker:1.13-dind 39 | privileged: true 40 | command: 41 | - "-s" 42 | - "overlay" 43 | -------------------------------------------------------------------------------- /.drone.yml.sig: -------------------------------------------------------------------------------- 1 | eyJhbGciOiJIUzI1NiJ9.cGlwZWxpbmU6CgogIG15LWJ1aWxkOgogICAgcHJpdmlsZWdlZDogdHJ1ZQogICAgaW1hZ2U6IGRvY2tlcjoxLjEzCiAgICBlbnZpcm9ubWVudDoKICAgICAgLSBET0NLRVJfSE9TVD10Y3A6Ly8xMjcuMC4wLjE6MjM3NQogICAgY29tbWFuZHM6CiAgICAgIC0gZG9ja2VyIGJ1aWxkIC10IHJlcG8tc2VjdXJpdHktc2Nhbm5lciAuCiAgICB3aGVuOgogICAgICBldmVudDogcHVzaAoKICBpbWFnZV90b19xdWF5OgogICAgaW1hZ2U6IGRvY2tlcjoxLjEzCiAgICBlbnZpcm9ubWVudDoKICAgICAgLSBET0NLRVJfSE9TVD10Y3A6Ly8xMjcuMC4wLjE6MjM3NQogICAgY29tbWFuZHM6CiAgICAgIC0gZG9ja2VyIGxvZ2luIC11PSJ1a2hvbWVvZmZpY2VkaWdpdGFsK3JlcG9fc2VjdXJpdHlfc2Nhbm5lcl9ib3QiIC1wPSR7RE9DS0VSX1BBU1NXT1JEfSBxdWF5LmlvCiAgICAgIC0gZG9ja2VyIHRhZyByZXBvLXNlY3VyaXR5LXNjYW5uZXIgcXVheS5pby91a2hvbWVvZmZpY2VkaWdpdGFsL3JlcG8tc2VjdXJpdHktc2Nhbm5lcjoke0RST05FX0NPTU1JVF9TSEF9CiAgICAgIC0gZG9ja2VyIHB1c2ggcXVheS5pby91a2hvbWVvZmZpY2VkaWdpdGFsL3JlcG8tc2VjdXJpdHktc2Nhbm5lcjoke0RST05FX0NPTU1JVF9TSEF9CiAgICB3aGVuOgogICAgICBldmVudDogcHVzaAoKICBkZXBsb3lfdG9fZGV2OgogICAgaW1hZ2U6IHF1YXkuaW8vdWtob21lb2ZmaWNlZGlnaXRhbC9rZDpsYXRlc3QKICAgIGVudmlyb25tZW50OgogICAgICAtIEtVQkVfU0VSVkVSPWh0dHBzOi8va3ViZS1kZXYuZHNwLm5vdHByb2QuaG9tZW9mZmljZS5nb3YudWsKICAgICAgLSBLVUJFX05BTUVTUEFDRT1yZXBvLXNlY3VyaXR5LXNjYW5uZXIKICAgICAgLSBJTlNFQ1VSRV9TS0lQX1RMU19WRVJJRlk9dHJ1ZQogICAgY29tbWFuZHM6CiAgICAgIC0gY2Qga3ViZQogICAgICAtIGtkIC1mIGRlcGxveW1lbnQueW1sIC1mIHNlcnZpY2UueW1sIC1mIGluZ3Jlc3MueW1sCiAgICB3aGVuOgogICAgICBldmVudDogW2RlcGxveW1lbnQsIHB1c2hdCgpzZXJ2aWNlczoKICBkaW5kOgogICAgaW1hZ2U6IGRvY2tlcjoxLjEzLWRpbmQKICAgIHByaXZpbGVnZWQ6IHRydWUKICAgIGNvbW1hbmQ6CiAgICAgIC0gIi1zIgogICAgICAtICJvdmVybGF5Igo.9nJkptCzCAycV8_iUGYdJ_T5h1awr2C7BJ2NQi2l-Mo -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | export GITHUB_WEBHOOKSECRET=blah 2 | export ELASTICSEARCH_URL="http://localhost:9200" 3 | export GOVUK_NOTIFY_API_KEY=<_update_me_> 4 | export GOVUK_NOTIFY_TEMPLATE_ID=<_update_me_> 5 | export NOTIFICATION_EMAILS=test@example.com 6 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: goreleaser 2 | 3 | on: 4 | push: 5 | tags: 6 | - "*" 7 | 8 | jobs: 9 | goreleaser: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v2 14 | with: 15 | fetch-depth: 0 16 | - name: Set up Go 17 | uses: actions/setup-go@v2 18 | - name: Run GoReleaser 19 | uses: goreleaser/goreleaser-action@v2 20 | with: 21 | distribution: goreleaser 22 | version: latest 23 | args: release --rm-dist 24 | env: 25 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | 3 | scanrepo 4 | release 5 | /repo-security-scanner 6 | logs 7 | *.log 8 | npm-debug.log* 9 | 10 | # Runtime data 11 | pids 12 | *.pid 13 | *.seed 14 | 15 | # Directory for instrumented libs generated by jscoverage/JSCover 16 | lib-cov 17 | 18 | # Coverage directory used by tools like istanbul 19 | coverage 20 | 21 | # nyc test coverage 22 | .nyc_output 23 | 24 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 25 | .grunt 26 | 27 | # node-waf configuration 28 | .lock-wscript 29 | 30 | # Compiled binary addons (http://nodejs.org/api/addons.html) 31 | build/Release 32 | 33 | # Dependency directories 34 | node_modules 35 | jspm_packages 36 | 37 | # Optional npm cache directory 38 | .npm 39 | 40 | # Optional REPL history 41 | .node_repl_history 42 | 43 | # Created by https://www.gitignore.io/api/go 44 | 45 | ### Go ### 46 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 47 | *.o 48 | *.a 49 | *.so 50 | 51 | # Folders 52 | _obj 53 | _test 54 | 55 | # Architecture specific extensions/prefixes 56 | *.[568vq] 57 | [568vq].out 58 | 59 | *.cgo1.go 60 | *.cgo2.c 61 | _cgo_defun.c 62 | _cgo_gotypes.go 63 | _cgo_export.* 64 | 65 | _testmain.go 66 | 67 | *.exe 68 | *.test 69 | *.prof 70 | 71 | # Output of the go coverage tool, specifically when used with LiteIDE 72 | *.out 73 | 74 | # External packages folder 75 | vendor/ 76 | 77 | # End of https://www.gitignore.io/api/go 78 | 79 | dist/ 80 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | # This is an example .goreleaser.yml file with some sensible defaults. 2 | # Make sure to check the documentation at https://goreleaser.com 3 | before: 4 | hooks: 5 | # You may remove this if you don't use go modules. 6 | - go mod tidy 7 | # you may remove this if you don't need go generate 8 | - go generate ./... 9 | builds: 10 | - env: 11 | - CGO_ENABLED=0 12 | main: ./cmd/scanrepo 13 | goos: 14 | - linux 15 | - windows 16 | - darwin 17 | archives: 18 | - replacements: 19 | darwin: Darwin 20 | linux: Linux 21 | windows: Windows 22 | 386: i386 23 | amd64: x86_64 24 | checksum: 25 | name_template: 'checksums.txt' 26 | snapshot: 27 | name_template: "{{ incpatch .Version }}-next" 28 | changelog: 29 | sort: asc 30 | filters: 31 | exclude: 32 | - '^docs:' 33 | - '^test:' 34 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | - repo: git://github.com/pre-commit/pre-commit-hooks 2 | sha: 46251c9523506b68419aefdf5ff6ff2fbc4506a4 3 | hooks: 4 | - id: check-added-large-files 5 | - id: check-json 6 | - id: check-merge-conflict 7 | - id: detect-private-key 8 | - id: end-of-file-fixer 9 | - id: forbid-new-submodules 10 | - id: trailing-whitespace 11 | - repo: https://bitbucket.org/samwhited/go-pre-commit.git 12 | sha: cab517ac1132ea76603bd51ba5a95305f81bb2ba 13 | hooks: 14 | - id: gofmt 15 | 16 | - repo: local 17 | hooks: 18 | - id: kubeval 19 | name: kubeval 20 | description: Lint kube files with system. 21 | entry: kubeval 22 | language: system 23 | files: kube\/(?!secrets\.yml) 24 | -------------------------------------------------------------------------------- /.secignore: -------------------------------------------------------------------------------- 1 | .env 2 | kube/secrets.yml 3 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.8-onbuild 2 | 3 | EXPOSE 8080 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PORT = 8080 2 | 3 | GITHUB_SECRET = $(shell echo $(GITHUB_WEBHOOKSECRET)) 4 | 5 | USER = ukhomeoffice-bot-test 6 | REPO = testgithubintegration 7 | OFFENSES_X0 = 47797c0123bc0f5adfcae3d3467a2ed12e72b2cb 8 | OFFENSES_X1 = f591c33a1b9500d0721b6664cfb6033d47a00793 9 | 10 | FIXT_DIR = test/fixtures 11 | RULES_FILE = $(FIXT_DIR)/rules/gitrob.json 12 | DIFF_FILE = $(FIXT_DIR)/github_event_push.json 13 | DIFF_FILE_OFFENSES = $(FIXT_DIR)/github_event_push_offenses.json 14 | RULES_URL = https://raw.githubusercontent.com/michenriksen/gitrob/master/signatures.json 15 | 16 | release-local: 17 | @goreleaser release --snapshot --rm-dist 18 | 19 | release: 20 | @goreleaser release 21 | 22 | cli: 23 | @go install -race ./cmd/scanrepo 24 | 25 | install: deps 26 | @go install -race --ldflags=\"-s\" . 27 | 28 | deps: get-tools 29 | @trash 30 | 31 | get-tools: 32 | @go get -u github.com/rancher/trash 33 | 34 | lint: 35 | @golint 36 | @go vet 37 | 38 | rules: 39 | @curl -s $(RULES_URL) > $(RULES_DIR)/gitrob.json 40 | 41 | # curl -s https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/commits/f591c33a1b9500d0721b6664cfb6033d47a00793 -H "Accept: application/vnd.github.VERSION.diff" 42 | diff-no-offenses: 43 | @curl -s \ 44 | -H "Accept: application/vnd.github.VERSION.diff" \ 45 | https://api.github.com/repos/$(USER)/$(REPO)/commits/$(OFFENSES_X0) 46 | 47 | diff-offenses: 48 | @curl -s \ 49 | -H "Accept: application/vnd.github.VERSION.diff" \ 50 | https://api.github.com/repos/$(USER)/$(REPO)/commits/$(OFFENSES_X1) 51 | 52 | struct: 53 | @gojson \ 54 | -name githubResponseFull \ 55 | -input test/fixtures/github_event_push.json 56 | 57 | watch: 58 | @realize run 59 | 60 | run: 61 | @go build -race . && ./repo-security-scanner 62 | 63 | mac-diff-file: 64 | @cat $(DIFF_FILE) | openssl sha1 -hmac $(GITHUB_SECRET) | sed 's/^.* //' 65 | 66 | mac-diff-file-offenses: 67 | @cat $(DIFF_FILE_OFFENSES) | openssl sha1 -hmac $(GITHUB_SECRET) | sed 's/^.* //' 68 | 69 | 70 | test-run: 71 | @wget -O- \ 72 | -X POST \ 73 | --header="X-GitHub-Event: push" \ 74 | --header="X-Hub-Signature: sha1=$(shell make mac-diff-file)" \ 75 | --post-file "$(DIFF_FILE)" \ 76 | http://localhost:$(PORT)/github 77 | 78 | test-run-offenses: 79 | @wget -O- \ 80 | -X POST \ 81 | --header="X-GitHub-Event: push" \ 82 | --header="X-Hub-Signature: sha1=$(shell make mac-diff-file-offenses)" \ 83 | --post-file "$(DIFF_FILE_OFFENSES)" \ 84 | http://localhost:$(PORT)/github 85 | 86 | test-run-fail: 87 | @wget -O- \ 88 | -X POST \ 89 | --header="X-GitHub-Event: push" \ 90 | --header="X-Hub-Signature: sha1=123456" \ 91 | --post-file "$(DIFF_FILE)" \ 92 | http://localhost:$(PORT)/github 93 | 94 | test-run-dev: 95 | @curl \ 96 | -X POST \ 97 | -H "x-github-event: push" \ 98 | --header="X-Hub-Signature: sha1=$(shell make mac-diff-file)" \ 99 | -d @$(DIFF_FILE) \ 100 | http://repo-security-scanner.notprod.homeoffice.gov.uk/github 101 | 102 | test: 103 | @go test 104 | 105 | .PHONY: release* test* run deps install lint rules struct diff 106 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![goreleaser](https://github.com/techjacker/repo-security-scanner/actions/workflows/release.yaml/badge.svg)](https://github.com/techjacker/repo-security-scanner/actions/workflows/release.yaml) 2 | 3 | # repo-security-scanner 4 | 5 | - CLI tool that finds secrets accidentally committed to a git repo, eg passwords, private keys 6 | - Run it against your entire repo's history by piping the output from `git log -p` 7 | 8 | ----------------------------------------------------------- 9 | 10 | ## Installation 11 | 1. [Download](../../releases) the latest stable release of the CLI tool for your architecture 12 | 2. Extract the tar and move the ```scanrepo``` binary to somewhere in your `$PATH`, eg `/usr/bin` 13 | 14 | ----------------------------------------------------------- 15 | 16 | ## Usage 17 | 18 | Check the entire history of the current branch for secrets. 19 | 20 | ``` 21 | $ git log -p | scanrepo 22 | 23 | ------------------ 24 | Violation 1 25 | Commit: 4cc087a1b4731d1017844cc86323df43068b0409 26 | File: web/src/db/seed.sql 27 | Reason: "SQL dump file" 28 | 29 | ------------------ 30 | Violation 2 31 | Commit: 142e6019248c0d53a5240242ed1a75c0cc110a0b 32 | File: config/passwords.ini 33 | Reason: "Contains word: password" 34 | 35 | ... 36 | ``` 37 | 38 | ----------------------------------------------------------- 39 | ### Add false positives to `.secignore` 40 | 41 | ``` 42 | $ cat .secignore 43 | file/that/is/not/really/a/secret/but/looks/like/one/to/diffence 44 | these/pems/are/ok/*.pem 45 | ``` 46 | 47 | [See example in this repo](./.secignore). 48 | 49 | 50 | ----------------------------------------------------------- 51 | ## Notifications 52 | Work in progress. 53 | 54 | ### Local Testing 55 | #### Set environment variables needed 56 | Create `env` file and update environment variables. 57 | ``` 58 | $ cp .env{.example,} 59 | # update .env values 60 | $ vi .env 61 | $ source .env 62 | ``` 63 | 64 | #### Launch containers 65 | ``` 66 | $ docker-compose up -d 67 | ``` 68 | 69 | #### Run test offenses 70 | ``` 71 | $ make test-run-offenses 72 | ``` 73 | 74 | 75 | ### Debugging Elastalert 76 | ``` 77 | $ docker exec -it sh 78 | # run elastalert test rule utility within elastalert container 79 | $ elastalert-test-rule --config $ELASTALERT_CONFIG --count-only "$RULES_DIRECTORY/new_violation.yaml" 80 | $ elastalert-test-rule --alert --config $ELASTALERT_CONFIG "$RULES_DIRECTORY/new_violation.yaml" 81 | # run elastalert in debug mode 82 | $ elastalert --config "$ELASTALERT_CONFIG" --rule "$RULES_DIRECTORY/new_violation.yaml" --debug 83 | ``` 84 | 85 | #### Logs 86 | ``` 87 | $ tail -f /log/elastalert_new_violation_rule.log 88 | ``` 89 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | ### bufio.NewScanner Limitations 2 | ``` 3 | // Programs that need more control over error handling or large tokens, 4 | // or must run sequential scans on a reader, should use bufio.Reader instead. 5 | ``` 6 | 7 | ### TODO 8 | - [ ] Analyze body of commits (added/removed lines) 9 | - [ ] Add concurrency (parallelize requests to github API) 10 | - [ ] Add context + timeout to requests to github API 11 | -------------------------------------------------------------------------------- /cmd/scanrepo/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "log" 8 | "os" 9 | 10 | "github.com/techjacker/diffence" 11 | ) 12 | 13 | func main() { 14 | 15 | rPath := flag.String("rules", "", "path to custom rules in JSON format") 16 | flag.Parse() 17 | 18 | info, _ := os.Stdin.Stat() 19 | if (info.Mode() & os.ModeCharDevice) == os.ModeCharDevice { 20 | log.Fatalln("The command is intended to work with pipes.") 21 | return 22 | } 23 | 24 | var ( 25 | err error 26 | rules *[]diffence.Rule 27 | ) 28 | 29 | if len(*rPath) > 0 { 30 | rules, err = diffence.LoadRulesJSON(*rPath) 31 | } else { 32 | rules, err = diffence.LoadDefaultRules() 33 | } 34 | if err != nil { 35 | log.Fatalf("Cannot load rules\n%s", err) 36 | return 37 | } 38 | 39 | diff := diffence.DiffChecker{ 40 | Rules: rules, 41 | Ignorer: diffence.NewIgnorerFromFile(".secignore"), 42 | } 43 | res, err := diff.Check(bufio.NewReader(os.Stdin)) 44 | if err != nil { 45 | log.Fatalf("Error reading diff\n%s\n", err) 46 | return 47 | } 48 | 49 | matches := res.Matches() 50 | if matches < 1 { 51 | fmt.Printf("Diff contains NO offenses\n\n") 52 | return 53 | } 54 | 55 | i := 1 56 | fmt.Fprintf(os.Stderr, "Diff contains %d offenses\n\n", matches) 57 | for diffKey, rule := range res.MatchedRules { 58 | fmt.Fprintf(os.Stderr, "------------------\n") 59 | fmt.Fprintf(os.Stderr, "Violation %d\n", i) 60 | commit, filename := diffence.SplitDiffHashKey(diffKey) 61 | if commit != "" { 62 | fmt.Fprintf(os.Stderr, "Commit: %s\n", commit) 63 | } 64 | fmt.Fprintf(os.Stderr, "File: %s\n", filename) 65 | fmt.Fprintf(os.Stderr, "Reason: %#v\n\n", rule[0].Caption) 66 | i++ 67 | } 68 | // finding violations constitutes an error 69 | os.Exit(1) 70 | } 71 | -------------------------------------------------------------------------------- /diff.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | ) 7 | 8 | // DiffGetterHTTP gets diff text for a commit 9 | type DiffGetterHTTP interface { 10 | Get(string) (*http.Response, error) 11 | } 12 | 13 | type diffGetterGithub struct{} 14 | 15 | func (g diffGetterGithub) Get(url string) (*http.Response, error) { 16 | req, err := http.NewRequest("GET", url, nil) 17 | if err != nil { 18 | return &http.Response{}, fmt.Errorf("could not get diff: %s", err) 19 | } 20 | req.Header.Set("Accept", "application/vnd.github.VERSION.diff") 21 | return http.DefaultClient.Do(req) 22 | } 23 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: "2.1" 3 | 4 | services: 5 | 6 | server: 7 | build: 8 | context: . 9 | env_file: .env 10 | environment: 11 | - GITHUB_WEBHOOKSECRET 12 | - ELASTICSEARCH_URL=http://elasticsearch:9200 13 | depends_on: 14 | elasticsearch: 15 | condition: service_healthy 16 | ports: 17 | - 8080:8080 18 | 19 | elasticsearch: 20 | restart: always 21 | image: elasticsearch:5 22 | ports: 23 | - 9200:9200 24 | expose: 25 | - "9200" 26 | healthcheck: 27 | test: ["CMD", "curl", "--fail", "http://localhost:9200"] 28 | interval: 5s 29 | timeout: 3s 30 | retries: 5 31 | 32 | elastalert: 33 | # privileged needed for ntpd 34 | privileged: true 35 | build: 36 | context: elastalert 37 | environment: 38 | - GOVUK_NOTIFY_API_KEY 39 | - GOVUK_NOTIFY_TEMPLATE_ID 40 | - NOTIFICATION_EMAILS 41 | - ELASTICSEARCH_HOST=elasticsearch 42 | - ELASTICSEARCH_PORT=9200 43 | depends_on: 44 | elasticsearch: 45 | condition: service_healthy 46 | -------------------------------------------------------------------------------- /elastalert/.gitignore: -------------------------------------------------------------------------------- 1 | env 2 | 3 | # Created by https://www.gitignore.io/api/python 4 | 5 | ### Python ### 6 | # Byte-compiled / optimized / DLL files 7 | __pycache__/ 8 | *.py[cod] 9 | *$py.class 10 | 11 | # C extensions 12 | *.so 13 | 14 | # Distribution / packaging 15 | .Python 16 | env/ 17 | build/ 18 | develop-eggs/ 19 | dist/ 20 | downloads/ 21 | eggs/ 22 | .eggs/ 23 | lib/ 24 | lib64/ 25 | parts/ 26 | sdist/ 27 | var/ 28 | wheels/ 29 | *.egg-info/ 30 | .installed.cfg 31 | *.egg 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .coverage 47 | .coverage.* 48 | .cache 49 | nosetests.xml 50 | coverage.xml 51 | *,cover 52 | .hypothesis/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | 62 | # Flask stuff: 63 | instance/ 64 | .webassets-cache 65 | 66 | # Scrapy stuff: 67 | .scrapy 68 | 69 | # Sphinx documentation 70 | docs/_build/ 71 | 72 | # PyBuilder 73 | target/ 74 | 75 | # Jupyter Notebook 76 | .ipynb_checkpoints 77 | 78 | # pyenv 79 | .python-version 80 | 81 | # celery beat schedule file 82 | celerybeat-schedule 83 | 84 | # SageMath parsed files 85 | *.sage.py 86 | 87 | # dotenv 88 | .env 89 | 90 | # virtualenv 91 | .venv 92 | venv/ 93 | ENV/ 94 | 95 | # Spyder project settings 96 | .spyderproject 97 | .spyproject 98 | 99 | # Rope project settings 100 | .ropeproject 101 | 102 | # mkdocs documentation 103 | /site 104 | 105 | # End of https://www.gitignore.io/api/python 106 | -------------------------------------------------------------------------------- /elastalert/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM iron/python:2 2 | 3 | RUN mkdir -p /var/empty 4 | # Set this environment variable to true to set timezone on container start. 5 | ENV SET_CONTAINER_TIMEZONE true 6 | # Default container timezone as found under the directory /usr/share/zoneinfo/. 7 | ENV CONTAINER_TIMEZONE Europe/London 8 | # URL from which to download Elastalert. 9 | 10 | WORKDIR /opt 11 | RUN apk update && \ 12 | apk upgrade && \ 13 | apk add ca-certificates openssl-dev libffi-dev python-dev gcc musl-dev tzdata openntpd curl && \ 14 | rm -rf /var/cache/apk/* && \ 15 | # Install pip - required for installation of Elastalert. 16 | wget https://bootstrap.pypa.io/get-pip.py && \ 17 | python get-pip.py && \ 18 | rm get-pip.py && \ 19 | # Download and unpack Elastalert. 20 | wget https://github.com/Yelp/elastalert/archive/master.zip && \ 21 | unzip *.zip && \ 22 | rm *.zip && \ 23 | mv e* elastalert 24 | 25 | # Install Elastalert. 26 | ENV ELASTALERT_ROOT /opt/elastalert 27 | WORKDIR ${ELASTALERT_ROOT} 28 | RUN python setup.py install && \ 29 | pip install -e . && \ 30 | pip install notifications-python-client && \ 31 | pip uninstall twilio --yes && \ 32 | pip install twilio==6.0.0 33 | 34 | WORKDIR /opt 35 | ENV RULES_DIRECTORY /opt/rules 36 | COPY rules ${RULES_DIRECTORY} 37 | ENV ELASTALERT_CONFIG /opt/elastalert_config.yaml 38 | COPY elastalert_config.yaml ${ELASTALERT_CONFIG} 39 | COPY modules ${ELASTALERT_ROOT}/elastalert_modules 40 | 41 | COPY elasticsearch_mappings.json /tmp 42 | COPY start-elastalert.sh /opt/ 43 | RUN chmod +x /opt/start-elastalert.sh 44 | CMD ["/opt/start-elastalert.sh"] 45 | -------------------------------------------------------------------------------- /elastalert/Makefile: -------------------------------------------------------------------------------- 1 | env: 2 | @python3 -m venv env 3 | 4 | deps-update: 5 | @pip install -r requirements-to-freeze.txt --upgrade 6 | @pip freeze > requirements.txt 7 | 8 | deps: 9 | @pip install -r requirements.txt 10 | @pre-commit install 11 | 12 | clean: 13 | @pip uninstall -yr requirements.txt 14 | @pip freeze > requirements.txt 15 | 16 | autopep8: 17 | @autopep8 . --recursive --in-place --pep8-passes 2000 --verbose 18 | 19 | autopep8-stats: 20 | @pep8 --quiet --statistics . 21 | 22 | test: 23 | @pytest tests -vv 24 | 25 | .PHONY: deps lint test* debug clean 26 | -------------------------------------------------------------------------------- /elastalert/elastalert_config.yaml: -------------------------------------------------------------------------------- 1 | # How often ElastAlert will query Elasticsearch 2 | # The unit can be anything from weeks to seconds 3 | run_every: 4 | seconds: 10 5 | 6 | # ElastAlert will buffer results from the most recent 7 | # period of time, in case some log sources are not in real time 8 | buffer_time: 9 | minutes: 15 10 | 11 | # The index on es_host which is used for metadata storage 12 | # This can be a unmapped index, but it is recommended that you run 13 | # elastalert-create-index to set a mapping 14 | writeback_index: elastalert_status 15 | 16 | # If an alert fails for some reason, ElastAlert will retry 17 | # sending the alert until this time period has elapsed 18 | alert_time_limit: 19 | days: 2 20 | 21 | timestamp_field: Timestamp 22 | -------------------------------------------------------------------------------- /elastalert/elasticsearch_mappings.json: -------------------------------------------------------------------------------- 1 | { 2 | "mappings": { 3 | "log": { 4 | "properties": { 5 | "Data": { 6 | "properties": { 7 | "filename": { 8 | "type": "text", 9 | "fields": { 10 | "keyword": { 11 | "type": "keyword", 12 | "index": false, 13 | "ignore_above": 256 14 | } 15 | }, 16 | "fielddata": false 17 | }, 18 | "organisation": { 19 | "type": "text", 20 | "fields": { 21 | "keyword": { 22 | "type": "keyword", 23 | "index": false, 24 | "ignore_above": 256 25 | } 26 | }, 27 | "fielddata": false 28 | }, 29 | "reason": { 30 | "type": "text", 31 | "fields": { 32 | "keyword": { 33 | "type": "keyword", 34 | "index": false, 35 | "ignore_above": 256 36 | } 37 | }, 38 | "fielddata": false 39 | }, 40 | "repo": { 41 | "type": "text", 42 | "fields": { 43 | "keyword": { 44 | "type": "keyword", 45 | "index": false, 46 | "ignore_above": 256 47 | } 48 | }, 49 | "fielddata": false 50 | }, 51 | "url": { 52 | "type": "text", 53 | "fields": { 54 | "keyword": { 55 | "type": "keyword", 56 | "index": false, 57 | "ignore_above": 256 58 | } 59 | }, 60 | "fielddata": false 61 | } 62 | } 63 | }, 64 | "Host": { 65 | "type": "text", 66 | "fields": { 67 | "keyword": { 68 | "type": "keyword", 69 | "index": false, 70 | "ignore_above": 256 71 | } 72 | }, 73 | "fielddata": false 74 | }, 75 | "Level": { 76 | "type": "text", 77 | "fields": { 78 | "keyword": { 79 | "type": "keyword", 80 | "index": false, 81 | "ignore_above": 256 82 | } 83 | }, 84 | "fielddata": false 85 | }, 86 | "Message": { 87 | "type": "text", 88 | "fields": { 89 | "keyword": { 90 | "type": "keyword", 91 | "index": false, 92 | "ignore_above": 256 93 | } 94 | }, 95 | "fielddata": false 96 | }, 97 | "Timestamp": { 98 | "type": "date", 99 | "format": "strict_date_optional_time||epoch_millis" 100 | } 101 | } 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /elastalert/email.tmpl: -------------------------------------------------------------------------------- 1 | Message: ((Message)) 2 | Timestamp: ((Timestamp)) 3 | 4 | Filename: ((Filename)) 5 | Reason: ((Reason)) 6 | Organisation: ((Organisation)) 7 | Repo: ((Repo)) 8 | URL: ((URL)) 9 | 10 | Elasticsearch Index: ((ElasticsearchIndex)) 11 | Elasticsearch ID: ((ElasticsearchId)) 12 | -------------------------------------------------------------------------------- /elastalert/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/techjacker/repo-security-scanner/2583094f81731073454eddeed38228512e68fec9/elastalert/modules/__init__.py -------------------------------------------------------------------------------- /elastalert/modules/govuknotify.py: -------------------------------------------------------------------------------- 1 | import os 2 | from elastalert.alerts import Alerter, BasicMatchString 3 | from notifications_python_client.notifications import NotificationsAPIClient 4 | 5 | 6 | class GovNotifyAlerter(Alerter): 7 | 8 | required_options = set(['log_file_path', 'email']) 9 | 10 | def __init__(self, rule): 11 | Alerter.__init__(self, rule) 12 | self.template_id = os.environ['GOVUK_NOTIFY_TEMPLATE_ID'] 13 | self.email_addresses = os.environ['NOTIFICATION_EMAILS'].split(',') 14 | api_key = os.environ['GOVUK_NOTIFY_API_KEY'] 15 | self.notifications_client = NotificationsAPIClient(api_key) 16 | 17 | @staticmethod 18 | def _generate_personalisation(match_items): 19 | personalisation = {} 20 | for i, v in enumerate(match_items): 21 | if v[0] == 'Message': 22 | personalisation['Message'] = v[1] 23 | elif v[0] == 'Timestamp': 24 | personalisation['Timestamp'] = v[1] 25 | elif v[0] == '_index': 26 | personalisation['ElasticsearchIndex'] = v[1] 27 | elif v[0] == '_id': 28 | personalisation['ElasticsearchId'] = v[1] 29 | elif v[0] == 'Data': 30 | personalisation['Filename'] = v[1]['filename'] 31 | personalisation['Reason'] = v[1]['reason'] 32 | personalisation['Organisation'] = v[1]['organisation'] 33 | personalisation['Repo'] = v[1]['repo'] 34 | personalisation['URL'] = v[1]['url'] 35 | return personalisation 36 | 37 | def _send_notification(self, email_address, personalisation): 38 | return self.notifications_client.send_email_notification( 39 | email_address=email_address, 40 | template_id=self.template_id, 41 | personalisation=personalisation, 42 | reference=None 43 | ) 44 | 45 | def alert(self, matches): 46 | # Matches is a list of match dictionaries. 47 | # It contains more than one match when the alert has 48 | # the aggregation option set 49 | for match in matches: 50 | personalisation = self._generate_personalisation(match.items()) 51 | for email_address in self.email_addresses: 52 | self._send_notification( 53 | email_address, personalisation) 54 | 55 | with open(self.rule['log_file_path'], 'a') as output_file: 56 | # basic_match_string will transform the match into the default 57 | # human readable string format 58 | # https://github.com/Yelp/elastalert/blob/3931d7feaf0d07b6531fb53042b9284bb46712ce/elastalert/alerts.py#L128 59 | match_string = str(BasicMatchString(self.rule, match)) 60 | output_file.write(match_string) 61 | 62 | # get_info is called after an alert is sent to get 63 | # data that is written back to Elasticsearch in the field "alert_info" 64 | # It should return a dict of information relevant to what the alert does 65 | def get_info(self): 66 | return {'type': 'GovUK Notify Alerter', 67 | 'email': self.rule['email'], 68 | 'log_file_path': self.rule['log_file_path']} 69 | -------------------------------------------------------------------------------- /elastalert/requirements-to-freeze.txt: -------------------------------------------------------------------------------- 1 | elastalert 2 | notifications_python_client 3 | pytest 4 | requests_mock 5 | -------------------------------------------------------------------------------- /elastalert/requirements.txt: -------------------------------------------------------------------------------- 1 | aws-requests-auth==0.3.3 2 | blist==1.3.6 3 | boto3==1.4.4 4 | botocore==1.5.77 5 | certifi==2017.4.17 6 | chardet==3.0.4 7 | configparser==3.5.0 8 | croniter==0.3.17 9 | defusedxml==0.5.0 10 | docopt==0.6.2 11 | docutils==0.13.1 12 | elastalert==0.1.16 13 | elasticsearch==5.4.0 14 | exotel==0.1.4 15 | future==0.16.0 16 | idna==2.5 17 | jira==1.0.10 18 | jmespath==0.9.3 19 | jsonschema==2.6.0 20 | mock==2.0.0 21 | monotonic==1.3 22 | notifications-python-client==4.3.1 23 | oauthlib==2.0.2 24 | pbr==3.1.1 25 | py==1.10.0 26 | PyJWT==2.4.0 27 | PySocks==1.6.7 28 | PyStaticConfiguration==0.10.3 29 | pytest==3.1.2 30 | python-dateutil==2.6.0 31 | pytz==2017.2 32 | PyYAML==5.4 33 | requests==2.20.0 34 | requests-mock==1.3.0 35 | requests-oauthlib==0.8.0 36 | requests-toolbelt==0.8.0 37 | s3transfer==0.1.10 38 | simplejson==3.11.1 39 | six==1.10.0 40 | stomp.py==4.1.18 41 | texttable==0.9.1 42 | twilio==6.0.0 43 | urllib3==1.26.5 44 | -------------------------------------------------------------------------------- /elastalert/rules/new_violation.yaml: -------------------------------------------------------------------------------- 1 | # Alert when a new violation is written to the githubintegration elasticsearch index 2 | name: New violation added to githubintegration index rule 3 | 4 | type: new_term 5 | 6 | index: githubintegration 7 | 8 | fields: 9 | - Level 10 | - Timestamp 11 | 12 | aggregation_key: "Data.repo" 13 | 14 | log_file_path: "/var/log/elastalert_new_violation_rule.log" 15 | 16 | alert: 17 | - "elastalert_modules.govuknotify.GovNotifyAlerter" 18 | -------------------------------------------------------------------------------- /elastalert/start-elastalert.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | #################### 5 | # create elasticsearch mappings for index monitoring 6 | # elastlaert will throw an error if these do not already exist 7 | #################### 8 | curl -s "$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/githubintegration/_search" 9 | if [[ $? != 0 ]]; then 10 | curl \ 11 | --upload-file /tmp/elasticsearch_mappings.json \ 12 | "$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/githubintegration" | python -m json.tool 13 | fi 14 | 15 | #################### 16 | # Set the timezone 17 | #################### 18 | if [ "$SET_CONTAINER_TIMEZONE" = "true" ]; then 19 | cp /usr/share/zoneinfo/${CONTAINER_TIMEZONE} /etc/localtime && \ 20 | echo "${CONTAINER_TIMEZONE}" > /etc/timezone && \ 21 | echo "Container timezone set to: $CONTAINER_TIMEZONE" 22 | else 23 | echo "Container timezone not modified" 24 | fi 25 | # Force immediate synchronisation of the time and start the time-synchronization service. 26 | # In order to be able to use ntpd in the container, it must be run with the SYS_TIME capability. 27 | # In addition you may want to add the SYS_NICE capability, in order for ntpd to be able to modify its priority. 28 | ntpd -s 29 | 30 | #################### 31 | # Main config 32 | #################### 33 | echo "" 34 | echo "rules_folder: $RULES_DIRECTORY" >> "$ELASTALERT_CONFIG" 35 | echo "" 36 | echo "es_host: $ELASTICSEARCH_HOST" >> "$ELASTALERT_CONFIG" 37 | echo "" 38 | echo "es_port: $ELASTICSEARCH_PORT" >> "$ELASTALERT_CONFIG" 39 | 40 | 41 | #################### 42 | # Rules config 43 | #################### 44 | # NOTIFICATION_EMAILS = comma separated list of email addresses 45 | echo "" 46 | echo "email:" >> "$RULES_DIRECTORY/new_violation.yaml" 47 | for i in $(echo $NOTIFICATION_EMAILS | tr "," "\n"); do 48 | echo "- $i" >> "$RULES_DIRECTORY/new_violation.yaml" 49 | done 50 | 51 | 52 | # Wait until Elasticsearch is online since otherwise Elastalert will fail. 53 | rm -f garbage_file 54 | while ! wget -O garbage_file "$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT" 2>/dev/null 55 | do 56 | echo "Waiting for Elasticsearch..." 57 | rm -f garbage_file 58 | sleep 1 59 | done 60 | rm -f garbage_file 61 | sleep 5 62 | 63 | # Check if the Elastalert index exists in Elasticsearch and create it if it does not. 64 | if ! wget -O garbage_file "$ELASTICSEARCH_HOST:$ELASTICSEARCH_PORT/elastalert_status" 2>/dev/null 65 | then 66 | echo "Creating Elastalert index in Elasticsearch..." 67 | elastalert-create-index \ 68 | --host "$ELASTICSEARCH_HOST" \ 69 | --port "$ELASTICSEARCH_PORT" \ 70 | --config "$ELASTALERT_CONFIG" \ 71 | --index elastalert_status \ 72 | --old-index "" 73 | else 74 | echo "Elastalert index already exists in Elasticsearch." 75 | fi 76 | rm -f garbage_file 77 | 78 | echo "Starting Elastalert..." 79 | exec elastalert --config "$ELASTALERT_CONFIG" --verbose 80 | -------------------------------------------------------------------------------- /elastalert/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/techjacker/repo-security-scanner/2583094f81731073454eddeed38228512e68fec9/elastalert/tests/__init__.py -------------------------------------------------------------------------------- /elastalert/tests/test_govuknotify.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from modules.govuknotify import GovNotifyAlerter 3 | 4 | 5 | @pytest.mark.parametrize(('match_items', 'expected'), [ 6 | ( 7 | [ 8 | ('Message', 'some message'), 9 | ('Timestamp', '12:30'), 10 | ('_index', 'githubintegration'), 11 | ('_id', '12345'), 12 | ('Data', { 13 | 'filename': 'secret.txt', 14 | 'reason': 'it is a secret', 15 | 'organisation': 'homeoffice', 16 | 'repo': 'greatrepo', 17 | 'url': 'https://github.com/homeoffice/greatrepo', 18 | }), 19 | ], 20 | { 21 | 'Message': 'some message', 22 | 'Timestamp': '12:30', 23 | 'ElasticsearchIndex': 'githubintegration', 24 | 'ElasticsearchId': '12345', 25 | 'Filename': 'secret.txt', 26 | 'Reason': 'it is a secret', 27 | 'Organisation': 'homeoffice', 28 | 'Repo': 'greatrepo', 29 | 'URL': 'https://github.com/homeoffice/greatrepo', 30 | } 31 | ), 32 | ]) 33 | def test_personalisation(match_items, expected): 34 | personalisation = GovNotifyAlerter._generate_personalisation( 35 | match_items=match_items) 36 | assert personalisation == expected 37 | -------------------------------------------------------------------------------- /github.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "io" 8 | ) 9 | 10 | // Valid exposes a method to validate a type 11 | type Valid interface { 12 | OK() error 13 | } 14 | 15 | // DecodeJSON marshalls JSON into a struct 16 | func DecodeJSON(r io.Reader, v interface{}) error { 17 | err := json.NewDecoder(r).Decode(v) 18 | if err != nil { 19 | return err 20 | } 21 | obj, ok := v.(Valid) 22 | if !ok { 23 | return nil // no OK method 24 | } 25 | err = obj.OK() 26 | if err != nil { 27 | return err 28 | } 29 | return nil 30 | } 31 | 32 | // GithubResponse is for marshalling Github push event JSON payloads 33 | type GithubResponse struct { 34 | Compare string 35 | Commits []struct { 36 | Added []string `json:"added"` 37 | ID string `json:"id"` 38 | } `json:"commits"` 39 | Repository struct { 40 | Name string `json:"name"` 41 | Owner struct { 42 | Email interface{} `json:"email"` 43 | Name string `json:"name"` 44 | } `json:"owner"` 45 | } `json:"repository"` 46 | } 47 | 48 | // OK validates a marshalled struct 49 | func (g *GithubResponse) OK() error { 50 | if len(g.Compare) < 1 { 51 | return errors.New("missing compare URL") 52 | } 53 | if len(g.Commits) < 1 { 54 | return errors.New("empty payload") 55 | } 56 | for _, c := range g.Commits { 57 | if len(c.ID) < 1 { 58 | return errors.New("missing commit ID") 59 | } 60 | } 61 | if len(g.Repository.Name) < 1 { 62 | return errors.New("missing repository name") 63 | } 64 | return nil 65 | } 66 | 67 | // getDiffURL returns the URL of the Github Diff API endpoint for a particular commit 68 | func (g GithubResponse) getDiffURL(commitID string) string { 69 | return fmt.Sprintf( 70 | "%s/%s", 71 | g.getDiffURLStem(), 72 | commitID, 73 | ) 74 | } 75 | 76 | func (g *GithubResponse) getDiffURLStem() string { 77 | return fmt.Sprintf( 78 | "https://api.github.com/repos/%s/%s/commits", 79 | g.Repository.Owner.Name, 80 | g.Repository.Name, 81 | ) 82 | } 83 | -------------------------------------------------------------------------------- /github_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | func Test_decodeGithubJSON(t *testing.T) { 11 | type args struct { 12 | body io.Reader 13 | } 14 | type want struct { 15 | CommitsID string 16 | CommitsAdded []string 17 | RepositoryName string 18 | RepositoryOwnerName string 19 | RepositoryOwnerEmail interface{} 20 | HeadersXGithubEvent string 21 | GithubAPIDiffURL string 22 | } 23 | tests := []struct { 24 | name string 25 | args args 26 | want want 27 | wantErr bool 28 | }{ 29 | { 30 | name: "Decodes github diff JSON response into struct", 31 | args: args{getFixture("test/fixtures/github_event_push.json")}, 32 | want: want{ 33 | CommitsID: "47797c0123bc0f5adfcae3d3467a2ed12e72b2cb", 34 | CommitsAdded: []string{"ba.txt"}, 35 | RepositoryName: "testgithubintegration", 36 | RepositoryOwnerName: "ukhomeoffice-bot-test", 37 | RepositoryOwnerEmail: nil, 38 | GithubAPIDiffURL: "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/commits", 39 | }, 40 | wantErr: false, 41 | }, 42 | { 43 | name: "Factory method validates decoded JSON", 44 | args: args{ 45 | strings.NewReader(`{"missing": "everything"}`), 46 | }, 47 | want: want{ 48 | CommitsID: "", 49 | CommitsAdded: []string{""}, 50 | RepositoryName: "", 51 | RepositoryOwnerName: "", 52 | RepositoryOwnerEmail: nil, 53 | GithubAPIDiffURL: "", 54 | }, 55 | wantErr: true, 56 | }, 57 | } 58 | for _, tt := range tests { 59 | t.Run(tt.name, func(t *testing.T) { 60 | gitRes := &GithubResponse{} 61 | err := DecodeJSON(tt.args.body, gitRes) 62 | if (err != nil) != tt.wantErr { 63 | t.Fatalf("decodeGithubJSON() error = %v, wantErr %v", err, tt.wantErr) 64 | return 65 | } 66 | if len(gitRes.Commits) > 0 { 67 | equals(t, gitRes.Commits[0].ID, tt.want.CommitsID) 68 | equals(t, gitRes.Commits[0].Added, tt.want.CommitsAdded) 69 | equals(t, gitRes.getDiffURLStem(), tt.want.GithubAPIDiffURL) 70 | equals(t, gitRes.getDiffURL(tt.want.CommitsID), fmt.Sprintf("%s/%s", tt.want.GithubAPIDiffURL, tt.want.CommitsID)) 71 | } 72 | equals(t, gitRes.Repository.Name, tt.want.RepositoryName) 73 | equals(t, gitRes.Repository.Owner.Name, tt.want.RepositoryOwnerName) 74 | equals(t, gitRes.Repository.Owner.Email, tt.want.RepositoryOwnerEmail) 75 | }) 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/UKHomeOffice-attic/repo-security-scanner 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/julienschmidt/httprouter v1.3.0 7 | github.com/mailru/easyjson v0.7.7 // indirect 8 | github.com/sirupsen/logrus v1.8.1 9 | github.com/techjacker/diffence v0.0.0-20170522155311-9be9b8826a62 10 | golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect 11 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect 12 | gopkg.in/olivere/elastic.v5 v5.0.86 13 | gopkg.in/sohlich/elogrus.v2 v2.0.2 14 | ) 15 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/aws/aws-sdk-go v1.29.11/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= 4 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= 9 | github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= 10 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 11 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 12 | github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 13 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 14 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 15 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 16 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 17 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 18 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 19 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 20 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 21 | github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= 22 | github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 23 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 24 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 25 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 26 | github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= 27 | github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= 28 | github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 29 | github.com/olivere/elastic/v7 v7.0.12/go.mod h1:14rWX28Pnh3qCKYRVnSGXWLf9MbLonYS/4FDCY3LAPo= 30 | github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= 31 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 32 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 33 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 34 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 35 | github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= 36 | github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= 37 | github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= 38 | github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= 39 | github.com/smartystreets/gunit v1.1.3/go.mod h1:EH5qMBab2UclzXUcpR8b93eHsIlp9u+pDQIRp5DZNzQ= 40 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 41 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 42 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 43 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 44 | github.com/techjacker/diffence v0.0.0-20170522155311-9be9b8826a62 h1:2F9fIaOPcGSWciwkE3c0tgwOyYSTJBLOsaPGPxTeoGA= 45 | github.com/techjacker/diffence v0.0.0-20170522155311-9be9b8826a62/go.mod h1:rtSSvrXSkiGyhBIxArCyhtb2QADhLBuPi0h4q35IVbA= 46 | go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= 47 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 48 | golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= 49 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 50 | golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= 51 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 52 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 53 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 54 | golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 55 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 56 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 57 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 58 | golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= 59 | golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 60 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 61 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 62 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 63 | golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 64 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 65 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 66 | golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 67 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 68 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 69 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 70 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= 71 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 72 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 73 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 74 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 75 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 76 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 77 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 78 | golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= 79 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 80 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 81 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 82 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 83 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 84 | google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= 85 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 86 | google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= 87 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 88 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 89 | gopkg.in/olivere/elastic.v5 v5.0.86 h1:xFy6qRCGAmo5Wjx96srho9BitLhZl2fcnpuidPwduXM= 90 | gopkg.in/olivere/elastic.v5 v5.0.86/go.mod h1:M3WNlsF+WhYn7api4D87NIflwTV/c0iVs8cqfWhK+68= 91 | gopkg.in/sohlich/elogrus.v2 v2.0.2 h1:qbqPT0cJjj4EsrCalvusqHgQeSy4pAR5mlVBKJeqtfE= 92 | gopkg.in/sohlich/elogrus.v2 v2.0.2/go.mod h1:Q9jBJmlG0MjXTOJoMaWFTWbxY/B3LCA2pmBEQRywiLo= 93 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 94 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 95 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 96 | -------------------------------------------------------------------------------- /handlers.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | 7 | "github.com/techjacker/diffence" 8 | ) 9 | 10 | const ( 11 | msgHealthOk = "ok" 12 | msgBadRequest = "bad request" 13 | msgIgnore = "Not a push evt; ignoring" 14 | msgNoViolationsFound = "Push contains no offenses" 15 | msgViolationFound = "Push contains violations" 16 | ) 17 | 18 | // GithubHandler is a github integration HTTP handler 19 | func GithubHandler(dc diffence.Checker, dg DiffGetterHTTP, log Log) http.Handler { 20 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 21 | // decode github push event payload 22 | gitRes := &GithubResponse{} 23 | err := DecodeJSON(r.Body, gitRes) 24 | if err != nil { 25 | http.Error(w, msgBadRequest, http.StatusBadRequest) 26 | return 27 | } 28 | // analyse all pushed commits 29 | results := diffence.Results{} 30 | for _, commit := range gitRes.Commits { 31 | resp, err := dg.Get(gitRes.getDiffURL(commit.ID)) 32 | if err != nil { 33 | http.Error(w, msgBadRequest, http.StatusBadRequest) 34 | return 35 | } 36 | diffRes, err := dc.Check(resp.Body) 37 | resp.Body.Close() 38 | if err != nil { 39 | http.Error(w, msgBadRequest, http.StatusBadRequest) 40 | return 41 | } 42 | results = append(results, diffRes) 43 | } 44 | 45 | if results.Matches() < 1 { 46 | fmt.Fprintf(w, "%s\n", msgNoViolationsFound) 47 | return 48 | } 49 | 50 | for _, res := range results { 51 | if res.Matches() > 0 { 52 | log.Log( 53 | res.MatchedRules, 54 | gitRes.Repository.Owner.Name, 55 | gitRes.Repository.Name, 56 | gitRes.Compare, 57 | ) 58 | } 59 | } 60 | fmt.Fprintf(w, "%s\n", msgViolationFound) 61 | }) 62 | } 63 | 64 | // HealthHandler is an endpoint for healthchecks 65 | func HealthHandler(w http.ResponseWriter, r *http.Request) { 66 | w.WriteHeader(http.StatusOK) 67 | w.Write([]byte(msgHealthOk)) 68 | } 69 | -------------------------------------------------------------------------------- /handlers_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http" 6 | "net/http/httptest" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/julienschmidt/httprouter" 11 | "github.com/sirupsen/logrus" 12 | "github.com/techjacker/diffence" 13 | ) 14 | 15 | type testDiffGetter struct { 16 | fixt string 17 | } 18 | 19 | func (t testDiffGetter) Get(_ string) (*http.Response, error) { 20 | return &http.Response{ 21 | Body: ioutil.NopCloser(getFixture(t.fixt)), 22 | }, nil 23 | } 24 | 25 | type testLogger struct { 26 | log *logrus.Logger 27 | } 28 | 29 | func (l testLogger) Log(v ...interface{}) { 30 | return 31 | } 32 | 33 | func TestGithubHandler(t *testing.T) { 34 | const ( 35 | testPath = "/github" 36 | ) 37 | 38 | type args struct { 39 | githubPayloadPath string 40 | rulesPath string 41 | diffPath string 42 | } 43 | tests := []struct { 44 | name string 45 | args args 46 | wantStatusCode int 47 | wantResBody string 48 | }{ 49 | { 50 | name: "Incorrect JSON payload returns 4xx status code", 51 | args: args{ 52 | githubPayloadPath: "test/fixtures/nonsense.json", 53 | rulesPath: gitrobRules, 54 | diffPath: "", 55 | }, 56 | wantStatusCode: http.StatusBadRequest, 57 | wantResBody: msgBadRequest, 58 | }, 59 | { 60 | name: "No offenses in diff", 61 | args: args{ 62 | githubPayloadPath: "test/fixtures/github_event_push.json", 63 | rulesPath: gitrobRules, 64 | diffPath: "test/fixtures/no_offenses.diff", 65 | }, 66 | wantStatusCode: http.StatusOK, 67 | wantResBody: msgNoViolationsFound, 68 | }, 69 | { 70 | name: "1 offense in diff", 71 | args: args{ 72 | githubPayloadPath: "test/fixtures/github_event_push.json", 73 | rulesPath: gitrobRules, 74 | diffPath: "test/fixtures/offenses_x1.diff", 75 | }, 76 | wantStatusCode: http.StatusOK, 77 | wantResBody: msgViolationFound, 78 | }, 79 | } 80 | for _, tt := range tests { 81 | t.Run(tt.name, func(t *testing.T) { 82 | 83 | router := httprouter.New() 84 | router.Handler("POST", testPath, GithubHandler( 85 | diffence.DiffChecker{Rules: getTestRules(t, tt.args.rulesPath)}, 86 | testDiffGetter{tt.args.diffPath}, 87 | testLogger{}, 88 | )) 89 | 90 | params := getFixture(tt.args.githubPayloadPath) 91 | r, err := http.NewRequest("POST", testPath, params) 92 | if err != nil { 93 | t.Error(err) 94 | } 95 | w := httptest.NewRecorder() 96 | router.ServeHTTP(w, r) 97 | 98 | if w.Code != tt.wantStatusCode { 99 | t.Fatalf("Githubhandler returned unexpected status code: got %v want %v", 100 | w.Code, tt.wantStatusCode) 101 | } 102 | if strings.EqualFold(strings.TrimSpace(w.Body.String()), tt.wantResBody) != true { 103 | t.Fatalf("Githubhandler returned unexpected body: got %v want %v", 104 | w.Body.String(), tt.wantResBody) 105 | } 106 | }) 107 | } 108 | 109 | } 110 | 111 | func TestHealthHandler(t *testing.T) { 112 | const ( 113 | testPath = "/healthz" 114 | ) 115 | 116 | tests := []struct { 117 | name string 118 | wantStatusCode int 119 | wantResBody string 120 | }{ 121 | { 122 | name: "Healthcheck handler", 123 | wantStatusCode: http.StatusOK, 124 | wantResBody: msgHealthOk, 125 | }, 126 | } 127 | for _, tt := range tests { 128 | t.Run(tt.name, func(t *testing.T) { 129 | 130 | router := httprouter.New() 131 | router.Handler("GET", testPath, http.HandlerFunc(HealthHandler)) 132 | 133 | r, err := http.NewRequest("GET", testPath, nil) 134 | if err != nil { 135 | t.Error(err) 136 | } 137 | w := httptest.NewRecorder() 138 | router.ServeHTTP(w, r) 139 | 140 | if w.Code != tt.wantStatusCode { 141 | t.Fatalf("Healthhandler returned unexpected status code: got %v want %v", 142 | w.Code, tt.wantStatusCode) 143 | } 144 | if strings.EqualFold(strings.TrimSpace(w.Body.String()), tt.wantResBody) != true { 145 | t.Fatalf("Healthhandler returned unexpected body: got %v want %v", 146 | w.Body.String(), tt.wantResBody) 147 | } 148 | }) 149 | } 150 | 151 | } 152 | -------------------------------------------------------------------------------- /kube/deployment.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: extensions/v1beta1 3 | kind: Deployment 4 | metadata: 5 | name: repo-security-scanner 6 | spec: 7 | replicas: 3 8 | template: 9 | metadata: 10 | labels: 11 | name: repo-security-scanner 12 | spec: 13 | containers: 14 | - name: repo-security-scanner 15 | image: quay.io/ukhomeofficedigital/repo-security-scanner:{{.DRONE_COMMIT_SHA}} 16 | ports: 17 | - name: http 18 | containerPort: 8080 19 | # https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/ 20 | livenessProbe: 21 | httpGet: 22 | path: /healthz 23 | port: http 24 | readinessProbe: 25 | httpGet: 26 | path: /healthz 27 | port: http 28 | env: 29 | - name: GITHUB_PRIVATEKEY 30 | valueFrom: 31 | secretKeyRef: 32 | name: repo-security-scanner 33 | key: GITHUB_PRIVATEKEY 34 | - name: GITHUB_WEBHOOKSECRET 35 | valueFrom: 36 | secretKeyRef: 37 | name: repo-security-scanner 38 | key: GITHUB_WEBHOOKSECRET 39 | resources: 40 | limits: 41 | cpu: 250m 42 | memory: 256Mi 43 | -------------------------------------------------------------------------------- /kube/ingress.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: extensions/v1beta1 3 | kind: Ingress 4 | metadata: 5 | name: repo-security-scanner-ingress 6 | spec: 7 | rules: 8 | - host: repo-security-scanner.notprod.homeoffice.gov.uk 9 | http: 10 | paths: 11 | - backend: 12 | serviceName: repo-security-scanner-service 13 | servicePort: http 14 | path: / 15 | -------------------------------------------------------------------------------- /kube/secrets.yml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | metadata: 4 | name: repo-security-scanner 5 | type: Opaque 6 | data: 7 | GITHUB_PRIVATEKEY: {{.GITHUB_PRIVATEKEY}} 8 | GITHUB_WEBHOOKSECRET: {{.GITHUB_WEBHOOKSECRET}} 9 | -------------------------------------------------------------------------------- /kube/service.yml: -------------------------------------------------------------------------------- 1 | --- 2 | apiVersion: v1 3 | kind: Service 4 | metadata: 5 | labels: 6 | name: repo-security-scanner-service 7 | name: repo-security-scanner-service 8 | spec: 9 | selector: 10 | name: repo-security-scanner 11 | ports: 12 | - name: http 13 | port: 8080 14 | targetPort: 8080 15 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/sirupsen/logrus" 7 | elastic "gopkg.in/olivere/elastic.v5" 8 | elogrus "gopkg.in/sohlich/elogrus.v2" 9 | 10 | "github.com/techjacker/diffence" 11 | ) 12 | 13 | // Log is the log interface for the app 14 | type Log interface { 15 | Log(v ...interface{}) 16 | } 17 | 18 | // Log is the log for the app 19 | type Logger struct { 20 | log *logrus.Logger 21 | } 22 | 23 | func (l Logger) Log(v ...interface{}) { 24 | i := 1 25 | for filename, rule := range v[0].(diffence.MatchedRules) { 26 | i++ 27 | l.log.WithFields(logrus.Fields{ 28 | "organisation": v[1], 29 | "repo": v[2], 30 | "url": v[3], 31 | "filename": filename, 32 | "reason": rule[0].Caption, 33 | }).Error(fmt.Sprintf("Violation found in %s:%s", v[1], v[2])) 34 | } 35 | } 36 | 37 | // NewESLogger is a factory for Elasticsearch loggers 38 | func NewESLogger(esUrl, esIndex string) (Logger, error) { 39 | log := logrus.New() 40 | client, err := elastic.NewClient(elastic.SetURL(esUrl)) 41 | if err != nil { 42 | return Logger{}, err 43 | } 44 | hook, err := elogrus.NewElasticHook(client, "localhost", logrus.DebugLevel, esIndex) 45 | if err != nil { 46 | return Logger{}, err 47 | } 48 | log.Hooks.Add(hook) 49 | return Logger{log: log}, nil 50 | } 51 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "os" 7 | "path" 8 | "runtime" 9 | 10 | "github.com/julienschmidt/httprouter" 11 | "github.com/sirupsen/logrus" 12 | "github.com/techjacker/diffence" 13 | ) 14 | 15 | const ( 16 | elasticSearchIndex = "githubintegration" 17 | headerGithubMAC = "X-Hub-Signature" 18 | headerGithubEvt = "X-Github-Event" 19 | githubWebhookSecret = "GITHUB_WEBHOOKSECRET" 20 | gitrobRules = "rules/gitrob.json" 21 | serverPort = 8080 22 | ) 23 | 24 | func getRules(rulesPath string) *[]diffence.Rule { 25 | _, cmd, _, _ := runtime.Caller(0) 26 | rules, err := diffence.LoadRulesJSON(path.Join(path.Dir(cmd), rulesPath)) 27 | if err != nil { 28 | panic(fmt.Sprintf("Cannot read rule file: %s\n", err)) 29 | } 30 | return rules 31 | } 32 | 33 | func getRequiredEnvVar(name string) []byte { 34 | val, ok := os.LookupEnv(name) 35 | if !ok { 36 | panic(fmt.Sprintf("Env var:%s not set in environment", name)) 37 | } 38 | return []byte(val) 39 | } 40 | 41 | func getLogger() Log { 42 | esURL, ok := os.LookupEnv("ELASTICSEARCH_URL") 43 | if ok { 44 | fmt.Printf("Logging to elasticsearch: %s\n", esURL) 45 | logger, err := NewESLogger(esURL, elasticSearchIndex) 46 | if err != nil { 47 | panic(err) 48 | } 49 | return logger 50 | } 51 | return Logger{ 52 | log: logrus.New(), 53 | } 54 | } 55 | 56 | func main() { 57 | router := httprouter.New() 58 | router.Handler("GET", "/healthz", http.HandlerFunc(HealthHandler)) 59 | router.Handler("POST", "/github", Adapt( 60 | GithubHandler( 61 | diffence.DiffChecker{Rules: getRules(gitrobRules)}, 62 | diffGetterGithub{}, 63 | getLogger(), 64 | ), 65 | AuthMiddleware(GithubAuthenticator{getRequiredEnvVar(githubWebhookSecret)}), 66 | )) 67 | 68 | fmt.Printf("Server listening on port: %d\n", serverPort) 69 | http.ListenAndServe(fmt.Sprintf(":%d", serverPort), router) 70 | } 71 | -------------------------------------------------------------------------------- /middleware.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "crypto/hmac" 6 | "crypto/sha1" 7 | "encoding/hex" 8 | "errors" 9 | "io/ioutil" 10 | "net/http" 11 | ) 12 | 13 | // Adapter defines a middleware handler 14 | type Adapter func(http.Handler) http.Handler 15 | 16 | // Adapt chains a series of middleware handlers 17 | func Adapt(h http.Handler, adapters ...Adapter) http.Handler { 18 | for _, adapter := range adapters { 19 | h = adapter(h) 20 | } 21 | return h 22 | } 23 | 24 | // Middleware wraps a function and returns an http.Handler fn type 25 | type Middleware func(http.Handler) http.Handler 26 | 27 | // Authenticator is an interface for authorizing streams 28 | type Authenticator interface { 29 | CheckMAC([]byte, []byte) (bool, error) 30 | } 31 | 32 | // GithubAuthenticator authorizes a payload with a shared secret 33 | type GithubAuthenticator struct { 34 | secret []byte 35 | } 36 | 37 | // CheckMAC checks text matches a HMAC signature 38 | func (g GithubAuthenticator) CheckMAC(body, expectedMAC []byte) (bool, error) { 39 | if len(g.secret) < 1 { 40 | return false, errors.New("secret not set") 41 | } 42 | mac := hmac.New(sha1.New, g.secret) 43 | mac.Write(body) 44 | actualMAC := []byte(hex.EncodeToString(mac.Sum(nil))) 45 | return hmac.Equal(expectedMAC, append([]byte("sha1="), actualMAC...)), nil 46 | } 47 | 48 | // AuthMiddleware authenticates a request body 49 | func AuthMiddleware(ag Authenticator) Adapter { 50 | return func(next http.Handler) http.Handler { 51 | fn := func(w http.ResponseWriter, r *http.Request) { 52 | if r.Header.Get(headerGithubEvt) != "push" { 53 | w.WriteHeader(http.StatusOK) 54 | w.Write([]byte(msgIgnore)) 55 | return 56 | } 57 | buf, err := ioutil.ReadAll(r.Body) 58 | rdr1 := ioutil.NopCloser(bytes.NewBuffer(buf)) 59 | r.Body = rdr1 60 | if err != nil { 61 | w.WriteHeader(http.StatusInternalServerError) 62 | w.Write([]byte(http.StatusText(http.StatusInternalServerError))) 63 | return 64 | } 65 | authorized, err := ag.CheckMAC( 66 | buf, 67 | []byte(r.Header.Get(headerGithubMAC)), 68 | ) 69 | if err != nil { 70 | w.WriteHeader(http.StatusInternalServerError) 71 | w.Write([]byte(http.StatusText(http.StatusInternalServerError))) 72 | return 73 | } 74 | if authorized != true { 75 | w.WriteHeader(http.StatusUnauthorized) 76 | w.Write([]byte(http.StatusText(http.StatusUnauthorized))) 77 | return 78 | } 79 | next.ServeHTTP(w, r) 80 | } 81 | return http.HandlerFunc(fn) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /middleware_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/julienschmidt/httprouter" 10 | ) 11 | 12 | const ( 13 | sharedSecret = "blah" 14 | testPath = "/auth" 15 | testHandlerMsg = "test handler message body" 16 | ) 17 | 18 | func TestAuthMiddleware(t *testing.T) { 19 | type args struct { 20 | headerGithubEvt string 21 | headerGithubMAC string 22 | body string 23 | } 24 | // authTestHandler returns a http.Handler for testing http middleware 25 | authTestHandler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { 26 | w.WriteHeader(http.StatusOK) 27 | w.Write([]byte(testHandlerMsg)) 28 | }) 29 | 30 | tests := []struct { 31 | name string 32 | args args 33 | wantStatusCode int 34 | wantResBody string 35 | }{ 36 | { 37 | name: "Not a push event", 38 | args: args{ 39 | headerGithubEvt: "not-push", 40 | headerGithubMAC: "sha1=033fa0645f29a6f8a4decd8e8aee8a9e341151cc", 41 | body: "hello world", 42 | }, 43 | wantStatusCode: http.StatusOK, 44 | wantResBody: msgIgnore, 45 | }, 46 | { 47 | name: "Push event with valid signature", 48 | args: args{ 49 | headerGithubEvt: "push", 50 | headerGithubMAC: "sha1=033fa0645f29a6f8a4decd8e8aee8a9e341151cc", 51 | body: "hello world", 52 | }, 53 | wantStatusCode: http.StatusOK, 54 | wantResBody: testHandlerMsg, 55 | }, 56 | { 57 | name: "Push event with invalid signature", 58 | args: args{ 59 | headerGithubEvt: "push", 60 | headerGithubMAC: "sha1=0000000000000000000000000000000000000000", 61 | body: "hello world", 62 | }, 63 | wantStatusCode: http.StatusUnauthorized, 64 | wantResBody: http.StatusText(http.StatusUnauthorized), 65 | }, 66 | } 67 | for _, tt := range tests { 68 | t.Run(tt.name, func(t *testing.T) { 69 | 70 | router := httprouter.New() 71 | auth := AuthMiddleware(GithubAuthenticator{[]byte(sharedSecret)}) 72 | router.Handler("POST", testPath, Adapt(authTestHandler, auth)) 73 | 74 | r, err := http.NewRequest("POST", testPath, strings.NewReader(tt.args.body)) 75 | r.Header.Set(headerGithubEvt, tt.args.headerGithubEvt) 76 | r.Header.Set(headerGithubMAC, tt.args.headerGithubMAC) 77 | if err != nil { 78 | t.Error(err) 79 | } 80 | w := httptest.NewRecorder() 81 | router.ServeHTTP(w, r) 82 | 83 | if w.Code != tt.wantStatusCode { 84 | t.Fatalf("TestAuthHandler returned unexpected status code: got %v want %v", 85 | w.Code, tt.wantStatusCode) 86 | } 87 | if strings.EqualFold(strings.TrimSpace(w.Body.String()), tt.wantResBody) != true { 88 | t.Fatalf("TestAuthHandler returned unexpected body: got %v want %v", 89 | w.Body.String(), tt.wantResBody) 90 | } 91 | }) 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /rules/gitrob.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "part": "filename", 4 | "type": "regex", 5 | "pattern": "\\A.*_rsa\\z", 6 | "caption": "Private SSH key", 7 | "description": null 8 | }, 9 | { 10 | "part": "filename", 11 | "type": "regex", 12 | "pattern": "\\A.*_dsa\\z", 13 | "caption": "Private SSH key", 14 | "description": null 15 | }, 16 | { 17 | "part": "filename", 18 | "type": "regex", 19 | "pattern": "\\A.*_ed25519\\z", 20 | "caption": "Private SSH key", 21 | "description": null 22 | }, 23 | { 24 | "part": "filename", 25 | "type": "regex", 26 | "pattern": "\\A.*_ecdsa\\z", 27 | "caption": "Private SSH key", 28 | "description": null 29 | }, 30 | { 31 | "part": "path", 32 | "type": "regex", 33 | "pattern": "\\.?ssh/config\\z", 34 | "caption": "SSH configuration file", 35 | "description": null 36 | }, 37 | { 38 | "part": "extension", 39 | "type": "match", 40 | "pattern": "pem", 41 | "caption": "Potential cryptographic private key", 42 | "description": null 43 | }, 44 | { 45 | "part": "extension", 46 | "type": "regex", 47 | "pattern": "\\Akey(pair)?\\z", 48 | "caption": "Potential cryptographic private key", 49 | "description": null 50 | }, 51 | { 52 | "part": "extension", 53 | "type": "match", 54 | "pattern": "pkcs12", 55 | "caption": "Potential cryptographic key bundle", 56 | "description": null 57 | }, 58 | { 59 | "part": "extension", 60 | "type": "match", 61 | "pattern": "pfx", 62 | "caption": "Potential cryptographic key bundle", 63 | "description": null 64 | }, 65 | { 66 | "part": "extension", 67 | "type": "match", 68 | "pattern": "p12", 69 | "caption": "Potential cryptographic key bundle", 70 | "description": null 71 | }, 72 | { 73 | "part": "extension", 74 | "type": "match", 75 | "pattern": "asc", 76 | "caption": "Potential cryptographic key bundle", 77 | "description": null 78 | }, 79 | { 80 | "part": "filename", 81 | "type": "match", 82 | "pattern": "otr.private_key", 83 | "caption": "Pidgin OTR private key", 84 | "description": null 85 | }, 86 | { 87 | "part": "filename", 88 | "type": "regex", 89 | "pattern": "\\A\\.?(bash_|zsh_|z)?history\\z", 90 | "caption": "Shell command history file", 91 | "description": null 92 | }, 93 | { 94 | "part": "filename", 95 | "type": "regex", 96 | "pattern": "\\A\\.?mysql_history\\z", 97 | "caption": "MySQL client command history file", 98 | "description": null 99 | }, 100 | { 101 | "part": "filename", 102 | "type": "regex", 103 | "pattern": "\\A\\.?psql_history\\z", 104 | "caption": "PostgreSQL client command history file", 105 | "description": null 106 | }, 107 | { 108 | "part": "filename", 109 | "type": "regex", 110 | "pattern": "\\A\\.?pgpass\\z", 111 | "caption": "PostgreSQL password file", 112 | "description": null 113 | }, 114 | { 115 | "part": "filename", 116 | "type": "regex", 117 | "pattern": "\\A\\.?irb_history\\z", 118 | "caption": "Ruby IRB console history file", 119 | "description": null 120 | }, 121 | { 122 | "part": "path", 123 | "type": "regex", 124 | "pattern": "\\.?purple\\/accounts\\.xml\\z", 125 | "caption": "Pidgin chat client account configuration file", 126 | "description": null 127 | }, 128 | { 129 | "part": "path", 130 | "type": "regex", 131 | "pattern": "\\.?xchat2?\\/servlist_?\\.conf\\z", 132 | "caption": "Hexchat/XChat IRC client server list configuration file", 133 | "description": null 134 | }, 135 | { 136 | "part": "path", 137 | "type": "regex", 138 | "pattern": "\\.?irssi\\/config\\z", 139 | "caption": "Irssi IRC client configuration file", 140 | "description": null 141 | }, 142 | { 143 | "part": "path", 144 | "type": "regex", 145 | "pattern": "\\.?recon-ng\\/keys\\.db\\z", 146 | "caption": "Recon-ng web reconnaissance framework API key database", 147 | "description": null 148 | }, 149 | { 150 | "part": "filename", 151 | "type": "regex", 152 | "pattern": "\\A\\.?dbeaver-data-sources.xml\\z", 153 | "caption": "DBeaver SQL database manager configuration file", 154 | "description": null 155 | }, 156 | { 157 | "part": "filename", 158 | "type": "regex", 159 | "pattern": "\\A\\.?muttrc\\z", 160 | "caption": "Mutt e-mail client configuration file", 161 | "description": null 162 | }, 163 | { 164 | "part": "filename", 165 | "type": "regex", 166 | "pattern": "\\A\\.?s3cfg\\z", 167 | "caption": "S3cmd configuration file", 168 | "description": null 169 | }, 170 | { 171 | "part": "path", 172 | "type": "regex", 173 | "pattern": "\\.?aws/credentials\\z", 174 | "caption": "AWS CLI credentials file", 175 | "description": null 176 | }, 177 | { 178 | "part": "filename", 179 | "type": "regex", 180 | "pattern": "\\A\\.?trc\\z", 181 | "caption": "T command-line Twitter client configuration file", 182 | "description": null 183 | }, 184 | { 185 | "part": "extension", 186 | "type": "match", 187 | "pattern": "ovpn", 188 | "caption": "OpenVPN client configuration file", 189 | "description": null 190 | }, 191 | { 192 | "part": "filename", 193 | "type": "regex", 194 | "pattern": "\\A\\.?gitrobrc\\z", 195 | "caption": "Well, this is awkward... Gitrob configuration file", 196 | "description": null 197 | }, 198 | { 199 | "part": "filename", 200 | "type": "regex", 201 | "pattern": "\\A\\.?(bash|zsh)rc\\z", 202 | "caption": "Shell configuration file", 203 | "description": "Shell configuration files might contain information such as server hostnames, passwords and API keys." 204 | }, 205 | { 206 | "part": "filename", 207 | "type": "regex", 208 | "pattern": "\\A\\.?(bash_|zsh_)?profile\\z", 209 | "caption": "Shell profile configuration file", 210 | "description": "Shell configuration files might contain information such as server hostnames, passwords and API keys." 211 | }, 212 | { 213 | "part": "filename", 214 | "type": "regex", 215 | "pattern": "\\A\\.?(bash_|zsh_)?aliases\\z", 216 | "caption": "Shell command alias configuration file", 217 | "description": "Shell configuration files might contain information such as server hostnames, passwords and API keys." 218 | }, 219 | { 220 | "part": "filename", 221 | "type": "match", 222 | "pattern": "secret_token.rb", 223 | "caption": "Ruby On Rails secret token configuration file", 224 | "description": "If the Rails secret token is known, it can allow for remote code execution. (http://www.exploit-db.com/exploits/27527/)" 225 | }, 226 | { 227 | "part": "filename", 228 | "type": "match", 229 | "pattern": "omniauth.rb", 230 | "caption": "OmniAuth configuration file", 231 | "description": "The OmniAuth configuration file might contain client application secrets." 232 | }, 233 | { 234 | "part": "filename", 235 | "type": "match", 236 | "pattern": "carrierwave.rb", 237 | "caption": "Carrierwave configuration file", 238 | "description": "Can contain credentials for online storage systems such as Amazon S3 and Google Storage." 239 | }, 240 | { 241 | "part": "filename", 242 | "type": "match", 243 | "pattern": "schema.rb", 244 | "caption": "Ruby On Rails database schema file", 245 | "description": "Contains information on the database schema of a Ruby On Rails application." 246 | }, 247 | { 248 | "part": "filename", 249 | "type": "match", 250 | "pattern": "database.yml", 251 | "caption": "Potential Ruby On Rails database configuration file", 252 | "description": "Might contain database credentials." 253 | }, 254 | { 255 | "part": "filename", 256 | "type": "match", 257 | "pattern": "settings.py", 258 | "caption": "Django configuration file", 259 | "description": "Might contain database credentials, online storage system credentials, secret keys, etc." 260 | }, 261 | { 262 | "part": "filename", 263 | "type": "regex", 264 | "pattern": "\\A(.*)?config(\\.inc)?\\.php\\z", 265 | "caption": "PHP configuration file", 266 | "description": "Might contain credentials and keys." 267 | }, 268 | { 269 | "part": "extension", 270 | "type": "match", 271 | "pattern": "kdb", 272 | "caption": "KeePass password manager database file", 273 | "description": null 274 | }, 275 | { 276 | "part": "extension", 277 | "type": "match", 278 | "pattern": "agilekeychain", 279 | "caption": "1Password password manager database file", 280 | "description": null 281 | }, 282 | { 283 | "part": "extension", 284 | "type": "match", 285 | "pattern": "keychain", 286 | "caption": "Apple Keychain database file", 287 | "description": null 288 | }, 289 | { 290 | "part": "extension", 291 | "type": "regex", 292 | "pattern": "\\Akey(store|ring)\\z", 293 | "caption": "GNOME Keyring database file", 294 | "description": null 295 | }, 296 | { 297 | "part": "extension", 298 | "type": "match", 299 | "pattern": "log", 300 | "caption": "Log file", 301 | "description": "Log files might contain information such as references to secret HTTP endpoints, session IDs, user information, passwords and API keys." 302 | }, 303 | { 304 | "part": "extension", 305 | "type": "match", 306 | "pattern": "pcap", 307 | "caption": "Network traffic capture file", 308 | "description": null 309 | }, 310 | { 311 | "part": "extension", 312 | "type": "regex", 313 | "pattern": "\\Asql(dump)?\\z", 314 | "caption": "SQL dump file", 315 | "description": null 316 | }, 317 | { 318 | "part": "extension", 319 | "type": "match", 320 | "pattern": "gnucash", 321 | "caption": "GnuCash database file", 322 | "description": null 323 | }, 324 | { 325 | "part": "filename", 326 | "type": "regex", 327 | "pattern": "backup", 328 | "caption": "Contains word: backup", 329 | "description": null 330 | }, 331 | { 332 | "part": "filename", 333 | "type": "regex", 334 | "pattern": "dump", 335 | "caption": "Contains word: dump", 336 | "description": null 337 | }, 338 | { 339 | "part": "filename", 340 | "type": "regex", 341 | "pattern": "password", 342 | "caption": "Contains word: password", 343 | "description": null 344 | }, 345 | { 346 | "part": "filename", 347 | "type": "regex", 348 | "pattern": "credential", 349 | "caption": "Contains word: credential", 350 | "description": null 351 | }, 352 | { 353 | "part": "filename", 354 | "type": "regex", 355 | "pattern": "secret", 356 | "caption": "Contains word: secret", 357 | "description": null 358 | }, 359 | { 360 | "part": "filename", 361 | "type": "regex", 362 | "pattern": "private.*key", 363 | "caption": "Contains words: private, key", 364 | "description": null 365 | }, 366 | { 367 | "part": "filename", 368 | "type": "match", 369 | "pattern": "jenkins.plugins.publish_over_ssh.BapSshPublisherPlugin.xml", 370 | "caption": "Jenkins publish over SSH plugin file", 371 | "description": null 372 | }, 373 | { 374 | "part": "filename", 375 | "type": "match", 376 | "pattern": "credentials.xml", 377 | "caption": "Potential Jenkins credentials file", 378 | "description": null 379 | }, 380 | { 381 | "part": "filename", 382 | "type": "regex", 383 | "pattern": "\\A\\.?htpasswd\\z", 384 | "caption": "Apache htpasswd file", 385 | "description": null 386 | }, 387 | { 388 | "part": "filename", 389 | "type": "regex", 390 | "pattern": "\\A(\\.|_)?netrc\\z", 391 | "caption": "Configuration file for auto-login process", 392 | "description": "Might contain username and password." 393 | }, 394 | { 395 | "part": "extension", 396 | "type": "match", 397 | "pattern": "kwallet", 398 | "caption": "KDE Wallet Manager database file", 399 | "description": null 400 | }, 401 | { 402 | "part": "filename", 403 | "type": "match", 404 | "pattern": "LocalSettings.php", 405 | "caption": "Potential MediaWiki configuration file", 406 | "description": null 407 | }, 408 | { 409 | "part": "extension", 410 | "type": "match", 411 | "pattern": "tblk", 412 | "caption": "Tunnelblick VPN configuration file", 413 | "description": null 414 | }, 415 | { 416 | "part": "path", 417 | "type": "regex", 418 | "pattern": "\\.?gem/credentials\\z", 419 | "caption": "Rubygems credentials file", 420 | "description": "Might contain API key for a rubygems.org account." 421 | }, 422 | { 423 | "part": "filename", 424 | "type": "regex", 425 | "pattern": "\\A*\\.pubxml(\\.user)?\\z", 426 | "caption": "Potential MSBuild publish profile", 427 | "description": null 428 | }, 429 | { 430 | "part": "filename", 431 | "type": "match", 432 | "pattern": "Favorites.plist", 433 | "caption": "Sequel Pro MySQL database manager bookmark file", 434 | "description": null 435 | }, 436 | { 437 | "part": "filename", 438 | "type": "match", 439 | "pattern": "configuration.user.xpl", 440 | "caption": "Little Snitch firewall configuration file", 441 | "description": "Contains traffic rules for applications" 442 | }, 443 | { 444 | "part": "extension", 445 | "type": "match", 446 | "pattern": "dayone", 447 | "caption": "Day One journal file", 448 | "description": null 449 | }, 450 | { 451 | "part": "filename", 452 | "type": "match", 453 | "pattern": "journal.txt", 454 | "caption": "Potential jrnl journal file", 455 | "description": null 456 | }, 457 | { 458 | "part": "filename", 459 | "type": "regex", 460 | "pattern": "\\A\\.?tugboat\\z", 461 | "caption": "Tugboat DigitalOcean management tool configuration", 462 | "description": null 463 | }, 464 | { 465 | "part": "filename", 466 | "type": "regex", 467 | "pattern": "\\A\\.?git-credentials\\z", 468 | "caption": "git-credential-store helper credentials file", 469 | "description": null 470 | }, 471 | { 472 | "part": "filename", 473 | "type": "regex", 474 | "pattern": "\\A\\.?gitconfig\\z", 475 | "caption": "Git configuration file", 476 | "description": null 477 | }, 478 | { 479 | "part": "filename", 480 | "type": "match", 481 | "pattern": "knife.rb", 482 | "caption": "Chef Knife configuration file", 483 | "description": "Might contain references to Chef servers" 484 | }, 485 | { 486 | "part": "path", 487 | "type": "regex", 488 | "pattern": "\\.?chef/(.*)\\.pem\\z", 489 | "caption": "Chef private key", 490 | "description": "Can be used to authenticate against Chef servers" 491 | }, 492 | { 493 | "part": "filename", 494 | "type": "match", 495 | "pattern": "proftpdpasswd", 496 | "caption": "cPanel backup ProFTPd credentials file", 497 | "description": "Contains usernames and password hashes for FTP accounts" 498 | }, 499 | { 500 | "part": "filename", 501 | "type": "match", 502 | "pattern": "robomongo.json", 503 | "caption": "Robomongo MongoDB manager configuration file", 504 | "description": "Might contain credentials for MongoDB databases" 505 | }, 506 | { 507 | "part": "filename", 508 | "type": "match", 509 | "pattern": "filezilla.xml", 510 | "caption": "FileZilla FTP configuration file", 511 | "description": "Might contain credentials for FTP servers" 512 | }, 513 | { 514 | "part": "filename", 515 | "type": "match", 516 | "pattern": "recentservers.xml", 517 | "caption": "FileZilla FTP recent servers file", 518 | "description": "Might contain credentials for FTP servers" 519 | }, 520 | { 521 | "part": "filename", 522 | "type": "match", 523 | "pattern": "ventrilo_srv.ini", 524 | "caption": "Ventrilo server configuration file", 525 | "description": "Might contain passwords" 526 | }, 527 | { 528 | "part": "filename", 529 | "type": "regex", 530 | "pattern": "\\A\\.?dockercfg\\z", 531 | "caption": "Docker configuration file", 532 | "description": "Might contain credentials for public or private Docker registries" 533 | }, 534 | { 535 | "part": "filename", 536 | "type": "regex", 537 | "pattern": "\\A\\.?npmrc\\z", 538 | "caption": "NPM configuration file", 539 | "description": "Might contain credentials for NPM registries" 540 | }, 541 | { 542 | "part": "filename", 543 | "type": "match", 544 | "pattern": "terraform.tfvars", 545 | "caption": "Terraform variable config file", 546 | "description": "Might contain credentials for terraform providers" 547 | }, 548 | { 549 | "part": "filename", 550 | "type": "regex", 551 | "pattern": "\\A\\.?env\\z", 552 | "caption": "Environment configuration file", 553 | "description": null 554 | } 555 | ] 556 | -------------------------------------------------------------------------------- /test/fixtures/github_event_push.json: -------------------------------------------------------------------------------- 1 | { 2 | "ref": "refs/heads/master", 3 | "before": "f591c33a1b9500d0721b6664cfb6033d47a00793", 4 | "after": "47797c0123bc0f5adfcae3d3467a2ed12e72b2cb", 5 | "created": false, 6 | "deleted": false, 7 | "forced": false, 8 | "base_ref": null, 9 | "compare": "https://github.com/ukhomeoffice-bot-test/testgithubintegration/compare/f591c33a1b95...47797c0123bc", 10 | "commits": [{ 11 | "id": "47797c0123bc0f5adfcae3d3467a2ed12e72b2cb", 12 | "tree_id": "740aa75764922d3ef8b6b6f64b4ab981d3a6d140", 13 | "distinct": true, 14 | "message": "h", 15 | "timestamp": "2017-01-19T10:30:29Z", 16 | "url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration/commit/47797c0123bc0f5adfcae3d3467a2ed12e72b2cb", 17 | "author": { 18 | "name": "Andrew Griffiths", 19 | "email": "mail@andrewgriffithsonline.com", 20 | "username": "techjacker" 21 | }, 22 | "committer": { 23 | "name": "Andrew Griffiths", 24 | "email": "mail@andrewgriffithsonline.com", 25 | "username": "techjacker" 26 | }, 27 | "added": [ 28 | "ba.txt" 29 | ], 30 | "removed": [ 31 | 32 | ], 33 | "modified": [ 34 | 35 | ] 36 | }], 37 | "head_commit": { 38 | "id": "47797c0123bc0f5adfcae3d3467a2ed12e72b2cb", 39 | "tree_id": "740aa75764922d3ef8b6b6f64b4ab981d3a6d140", 40 | "distinct": true, 41 | "message": "h", 42 | "timestamp": "2017-01-19T10:30:29Z", 43 | "url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration/commit/47797c0123bc0f5adfcae3d3467a2ed12e72b2cb", 44 | "author": { 45 | "name": "Andrew Griffiths", 46 | "email": "mail@andrewgriffithsonline.com", 47 | "username": "techjacker" 48 | }, 49 | "committer": { 50 | "name": "Andrew Griffiths", 51 | "email": "mail@andrewgriffithsonline.com", 52 | "username": "techjacker" 53 | }, 54 | "added": [ 55 | "ba.txt" 56 | ], 57 | "removed": [ 58 | 59 | ], 60 | "modified": [ 61 | 62 | ] 63 | }, 64 | "repository": { 65 | "id": 79224801, 66 | "name": "testgithubintegration", 67 | "full_name": "ukhomeoffice-bot-test/testgithubintegration", 68 | "owner": { 69 | "name": "ukhomeoffice-bot-test", 70 | "email": null 71 | }, 72 | "private": false, 73 | "html_url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration", 74 | "description": null, 75 | "fork": false, 76 | "url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration", 77 | "forks_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/forks", 78 | "keys_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/keys{/key_id}", 79 | "collaborators_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/collaborators{/collaborator}", 80 | "teams_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/teams", 81 | "hooks_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/hooks", 82 | "issue_events_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/issues/events{/number}", 83 | "events_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/events", 84 | "assignees_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/assignees{/user}", 85 | "branches_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/branches{/branch}", 86 | "tags_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/tags", 87 | "blobs_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/git/blobs{/sha}", 88 | "git_tags_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/git/tags{/sha}", 89 | "git_refs_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/git/refs{/sha}", 90 | "trees_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/git/trees{/sha}", 91 | "statuses_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/statuses/{sha}", 92 | "languages_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/languages", 93 | "stargazers_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/stargazers", 94 | "contributors_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/contributors", 95 | "subscribers_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/subscribers", 96 | "subscription_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/subscription", 97 | "commits_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/commits{/sha}", 98 | "git_commits_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/git/commits{/sha}", 99 | "comments_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/comments{/number}", 100 | "issue_comment_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/issues/comments{/number}", 101 | "contents_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/contents/{+path}", 102 | "compare_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/compare/{base}...{head}", 103 | "merges_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/merges", 104 | "archive_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/{archive_format}{/ref}", 105 | "downloads_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/downloads", 106 | "issues_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/issues{/number}", 107 | "pulls_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/pulls{/number}", 108 | "milestones_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/milestones{/number}", 109 | "notifications_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/notifications{?since,all,participating}", 110 | "labels_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/labels{/name}", 111 | "releases_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/releases{/id}", 112 | "deployments_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/deployments", 113 | "created_at": 1484655622, 114 | "updated_at": "2017-01-17T12:20:22Z", 115 | "pushed_at": 1484821838, 116 | "git_url": "git://github.com/ukhomeoffice-bot-test/testgithubintegration.git", 117 | "ssh_url": "git@github.com:ukhomeoffice-bot-test/testgithubintegration.git", 118 | "clone_url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration.git", 119 | "svn_url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration", 120 | "homepage": null, 121 | "size": 5, 122 | "stargazers_count": 0, 123 | "watchers_count": 0, 124 | "language": null, 125 | "has_issues": true, 126 | "has_downloads": true, 127 | "has_wiki": true, 128 | "has_pages": false, 129 | "forks_count": 0, 130 | "mirror_url": null, 131 | "open_issues_count": 0, 132 | "forks": 0, 133 | "open_issues": 0, 134 | "watchers": 0, 135 | "default_branch": "master", 136 | "stargazers": 0, 137 | "master_branch": "master", 138 | "organization": "ukhomeoffice-bot-test" 139 | }, 140 | "pusher": { 141 | "name": "techjacker", 142 | "email": "mail@andrewgriffithsonline.com" 143 | }, 144 | "organization": { 145 | "login": "ukhomeoffice-bot-test", 146 | "id": 22323467, 147 | "url": "https://api.github.com/orgs/ukhomeoffice-bot-test", 148 | "repos_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/repos", 149 | "events_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/events", 150 | "hooks_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/hooks", 151 | "issues_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/issues", 152 | "members_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/members{/member}", 153 | "public_members_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/public_members{/member}", 154 | "avatar_url": "https://avatars.githubusercontent.com/u/22323467?v=3", 155 | "description": null 156 | }, 157 | "sender": { 158 | "login": "techjacker", 159 | "id": 923307, 160 | "avatar_url": "https://avatars.githubusercontent.com/u/923307?v=3", 161 | "gravatar_id": "", 162 | "url": "https://api.github.com/users/techjacker", 163 | "html_url": "https://github.com/techjacker", 164 | "followers_url": "https://api.github.com/users/techjacker/followers", 165 | "following_url": "https://api.github.com/users/techjacker/following{/other_user}", 166 | "gists_url": "https://api.github.com/users/techjacker/gists{/gist_id}", 167 | "starred_url": "https://api.github.com/users/techjacker/starred{/owner}{/repo}", 168 | "subscriptions_url": "https://api.github.com/users/techjacker/subscriptions", 169 | "organizations_url": "https://api.github.com/users/techjacker/orgs", 170 | "repos_url": "https://api.github.com/users/techjacker/repos", 171 | "events_url": "https://api.github.com/users/techjacker/events{/privacy}", 172 | "received_events_url": "https://api.github.com/users/techjacker/received_events", 173 | "type": "User", 174 | "site_admin": false 175 | }, 176 | "installation": { 177 | "id": 6074 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /test/fixtures/github_event_push_offenses.json: -------------------------------------------------------------------------------- 1 | { 2 | "ref": "refs/heads/master", 3 | "before": "3cbc791b208b9557fbf61d83a2af057537d909b5", 4 | "after": "ae842c75ecb4ffacc888d7a83304c157a229625c", 5 | "created": false, 6 | "deleted": false, 7 | "forced": true, 8 | "base_ref": null, 9 | "compare": "https://github.com/ukhomeoffice-bot-test/testgithubintegration/compare/3cbc791b208b...ae842c75ecb4", 10 | "commits": [ 11 | { 12 | "id": "b421aaf0767e236aaeb20258f0a70d2228b83f16", 13 | "tree_id": "9bc182221a87f1d849d3e84c011d3afc587a01b9", 14 | "distinct": true, 15 | "message": "18", 16 | "timestamp": "2017-02-21T20:13:03Z", 17 | "url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration/commit/b421aaf0767e236aaeb20258f0a70d2228b83f16", 18 | "author": { 19 | "name": "Andrew Griffiths", 20 | "email": "mail@andrewgriffithsonline.com", 21 | "username": "techjacker" 22 | }, 23 | "committer": { 24 | "name": "Andrew Griffiths", 25 | "email": "mail@andrewgriffithsonline.com", 26 | "username": "techjacker" 27 | }, 28 | "added": [ 29 | "18" 30 | ], 31 | "removed": [ 32 | 33 | ], 34 | "modified": [ 35 | 36 | ] 37 | }, 38 | { 39 | "id": "ae842c75ecb4ffacc888d7a83304c157a229625c", 40 | "tree_id": "ddba7ffb347ca2934b2ed0891bb06b5b777b0f4d", 41 | "distinct": true, 42 | "message": "secrets99", 43 | "timestamp": "2017-03-15T22:21:30Z", 44 | "url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration/commit/ae842c75ecb4ffacc888d7a83304c157a229625c", 45 | "author": { 46 | "name": "Andrew Griffiths", 47 | "email": "mail@andrewgriffithsonline.com", 48 | "username": "techjacker" 49 | }, 50 | "committer": { 51 | "name": "Andrew Griffiths", 52 | "email": "mail@andrewgriffithsonline.com", 53 | "username": "techjacker" 54 | }, 55 | "added": [ 56 | "secrets99" 57 | ], 58 | "removed": [ 59 | 60 | ], 61 | "modified": [ 62 | 63 | ] 64 | } 65 | ], 66 | "head_commit": { 67 | "id": "ae842c75ecb4ffacc888d7a83304c157a229625c", 68 | "tree_id": "ddba7ffb347ca2934b2ed0891bb06b5b777b0f4d", 69 | "distinct": true, 70 | "message": "secrets99", 71 | "timestamp": "2017-03-15T22:21:30Z", 72 | "url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration/commit/ae842c75ecb4ffacc888d7a83304c157a229625c", 73 | "author": { 74 | "name": "Andrew Griffiths", 75 | "email": "mail@andrewgriffithsonline.com", 76 | "username": "techjacker" 77 | }, 78 | "committer": { 79 | "name": "Andrew Griffiths", 80 | "email": "mail@andrewgriffithsonline.com", 81 | "username": "techjacker" 82 | }, 83 | "added": [ 84 | "secrets99" 85 | ], 86 | "removed": [ 87 | 88 | ], 89 | "modified": [ 90 | 91 | ] 92 | }, 93 | "repository": { 94 | "id": 79224801, 95 | "name": "testgithubintegration", 96 | "full_name": "ukhomeoffice-bot-test/testgithubintegration", 97 | "owner": { 98 | "name": "ukhomeoffice-bot-test", 99 | "email": null, 100 | "login": "ukhomeoffice-bot-test", 101 | "id": 22323467, 102 | "avatar_url": "https://avatars1.githubusercontent.com/u/22323467?v=3", 103 | "gravatar_id": "", 104 | "url": "https://api.github.com/users/ukhomeoffice-bot-test", 105 | "html_url": "https://github.com/ukhomeoffice-bot-test", 106 | "followers_url": "https://api.github.com/users/ukhomeoffice-bot-test/followers", 107 | "following_url": "https://api.github.com/users/ukhomeoffice-bot-test/following{/other_user}", 108 | "gists_url": "https://api.github.com/users/ukhomeoffice-bot-test/gists{/gist_id}", 109 | "starred_url": "https://api.github.com/users/ukhomeoffice-bot-test/starred{/owner}{/repo}", 110 | "subscriptions_url": "https://api.github.com/users/ukhomeoffice-bot-test/subscriptions", 111 | "organizations_url": "https://api.github.com/users/ukhomeoffice-bot-test/orgs", 112 | "repos_url": "https://api.github.com/users/ukhomeoffice-bot-test/repos", 113 | "events_url": "https://api.github.com/users/ukhomeoffice-bot-test/events{/privacy}", 114 | "received_events_url": "https://api.github.com/users/ukhomeoffice-bot-test/received_events", 115 | "type": "Organization", 116 | "site_admin": false 117 | }, 118 | "private": false, 119 | "html_url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration", 120 | "description": null, 121 | "fork": false, 122 | "url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration", 123 | "forks_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/forks", 124 | "keys_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/keys{/key_id}", 125 | "collaborators_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/collaborators{/collaborator}", 126 | "teams_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/teams", 127 | "hooks_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/hooks", 128 | "issue_events_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/issues/events{/number}", 129 | "events_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/events", 130 | "assignees_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/assignees{/user}", 131 | "branches_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/branches{/branch}", 132 | "tags_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/tags", 133 | "blobs_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/git/blobs{/sha}", 134 | "git_tags_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/git/tags{/sha}", 135 | "git_refs_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/git/refs{/sha}", 136 | "trees_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/git/trees{/sha}", 137 | "statuses_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/statuses/{sha}", 138 | "languages_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/languages", 139 | "stargazers_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/stargazers", 140 | "contributors_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/contributors", 141 | "subscribers_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/subscribers", 142 | "subscription_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/subscription", 143 | "commits_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/commits{/sha}", 144 | "git_commits_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/git/commits{/sha}", 145 | "comments_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/comments{/number}", 146 | "issue_comment_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/issues/comments{/number}", 147 | "contents_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/contents/{+path}", 148 | "compare_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/compare/{base}...{head}", 149 | "merges_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/merges", 150 | "archive_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/{archive_format}{/ref}", 151 | "downloads_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/downloads", 152 | "issues_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/issues{/number}", 153 | "pulls_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/pulls{/number}", 154 | "milestones_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/milestones{/number}", 155 | "notifications_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/notifications{?since,all,participating}", 156 | "labels_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/labels{/name}", 157 | "releases_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/releases{/id}", 158 | "deployments_url": "https://api.github.com/repos/ukhomeoffice-bot-test/testgithubintegration/deployments", 159 | "created_at": 1484655622, 160 | "updated_at": "2017-01-17T12:20:22Z", 161 | "pushed_at": 1489616498, 162 | "git_url": "git://github.com/ukhomeoffice-bot-test/testgithubintegration.git", 163 | "ssh_url": "git@github.com:ukhomeoffice-bot-test/testgithubintegration.git", 164 | "clone_url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration.git", 165 | "svn_url": "https://github.com/ukhomeoffice-bot-test/testgithubintegration", 166 | "homepage": null, 167 | "size": 8, 168 | "stargazers_count": 0, 169 | "watchers_count": 0, 170 | "language": null, 171 | "has_issues": true, 172 | "has_downloads": true, 173 | "has_wiki": true, 174 | "has_pages": false, 175 | "forks_count": 0, 176 | "mirror_url": null, 177 | "open_issues_count": 0, 178 | "forks": 0, 179 | "open_issues": 0, 180 | "watchers": 0, 181 | "default_branch": "master", 182 | "stargazers": 0, 183 | "master_branch": "master", 184 | "organization": "ukhomeoffice-bot-test" 185 | }, 186 | "pusher": { 187 | "name": "techjacker", 188 | "email": "mail@andrewgriffithsonline.com" 189 | }, 190 | "organization": { 191 | "login": "ukhomeoffice-bot-test", 192 | "id": 22323467, 193 | "url": "https://api.github.com/orgs/ukhomeoffice-bot-test", 194 | "repos_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/repos", 195 | "events_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/events", 196 | "hooks_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/hooks", 197 | "issues_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/issues", 198 | "members_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/members{/member}", 199 | "public_members_url": "https://api.github.com/orgs/ukhomeoffice-bot-test/public_members{/member}", 200 | "avatar_url": "https://avatars1.githubusercontent.com/u/22323467?v=3", 201 | "description": null 202 | }, 203 | "sender": { 204 | "login": "techjacker", 205 | "id": 923307, 206 | "avatar_url": "https://avatars2.githubusercontent.com/u/923307?v=3", 207 | "gravatar_id": "", 208 | "url": "https://api.github.com/users/techjacker", 209 | "html_url": "https://github.com/techjacker", 210 | "followers_url": "https://api.github.com/users/techjacker/followers", 211 | "following_url": "https://api.github.com/users/techjacker/following{/other_user}", 212 | "gists_url": "https://api.github.com/users/techjacker/gists{/gist_id}", 213 | "starred_url": "https://api.github.com/users/techjacker/starred{/owner}{/repo}", 214 | "subscriptions_url": "https://api.github.com/users/techjacker/subscriptions", 215 | "organizations_url": "https://api.github.com/users/techjacker/orgs", 216 | "repos_url": "https://api.github.com/users/techjacker/repos", 217 | "events_url": "https://api.github.com/users/techjacker/events{/privacy}", 218 | "received_events_url": "https://api.github.com/users/techjacker/received_events", 219 | "type": "User", 220 | "site_admin": false 221 | }, 222 | "installation": { 223 | "id": 5873 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /test/fixtures/no_offenses.diff: -------------------------------------------------------------------------------- 1 | diff --git a/ba.txt b/ba.txt 2 | new file mode 100644 3 | index 0000000..be33f17 4 | --- /dev/null 5 | +++ b/ba.txt 6 | @@ -0,0 +1 @@ 7 | +hee 8 | -------------------------------------------------------------------------------- /test/fixtures/nonsense.json: -------------------------------------------------------------------------------- 1 | { 2 | "not": "expected json schema", 3 | "body": {} 4 | } 5 | -------------------------------------------------------------------------------- /test/fixtures/offenses_x1.diff: -------------------------------------------------------------------------------- 1 | diff --git a/secrets11.txt b/secrets11.txt 2 | index 2bd7976..6b97068 100644 3 | --- a/secrets11.txt 4 | +++ b/secrets11.txt 5 | @@ -1 +1 @@ 6 | -blah SECRET_KEYfsdfs 7 | +blah _KEYfsdfs 8 | -------------------------------------------------------------------------------- /testhelpers_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "os" 8 | "path" 9 | "path/filepath" 10 | "reflect" 11 | "runtime" 12 | "testing" 13 | 14 | "github.com/techjacker/diffence" 15 | ) 16 | 17 | func getFixture(filename string) io.Reader { 18 | if filename == "" { 19 | return bytes.NewReader([]byte("")) 20 | // return ioutil.Discard 21 | } 22 | file, err := os.Open(filename) 23 | if err != nil { 24 | panic(err) 25 | } 26 | return file 27 | } 28 | 29 | func getTestRules(t *testing.T, rulesPath string) *[]diffence.Rule { 30 | // get rules 31 | _, cmd, _, _ := runtime.Caller(0) 32 | rules, err := diffence.LoadRulesJSON(path.Join(path.Dir(cmd), rulesPath)) 33 | if err != nil { 34 | t.Fatal(fmt.Sprintf("Cannot read rule file: %s\n", err)) 35 | } 36 | return rules 37 | } 38 | 39 | // assert fails the test if the condition is false. 40 | func assert(tb testing.TB, condition bool, msg string, v ...interface{}) { 41 | if !condition { 42 | _, file, line, _ := runtime.Caller(1) 43 | fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) 44 | tb.FailNow() 45 | } 46 | } 47 | 48 | // ok fails the test if an err is not nil. 49 | func ok(tb testing.TB, err error) { 50 | if err != nil { 51 | _, file, line, _ := runtime.Caller(1) 52 | fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error()) 53 | tb.FailNow() 54 | } 55 | } 56 | 57 | // equals fails the test if exp is not equal to act. 58 | func equals(tb testing.TB, exp, act interface{}) { 59 | if !reflect.DeepEqual(exp, act) { 60 | _, file, line, _ := runtime.Caller(1) 61 | fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act) 62 | tb.FailNow() 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /vendor.conf: -------------------------------------------------------------------------------- 1 | # package 2 | github.com/UKHomeOffice/repo-security-scanner 3 | 4 | github.com/techjacker/diffence 12c4abbcb3c837367860b4bfa55f01e86dc5b377 5 | github.com/julienschmidt/httprouter 8a45e95fc75cb77048068a62daed98cc22fdac7c 6 | -------------------------------------------------------------------------------- /version: -------------------------------------------------------------------------------- 1 | RELEASE_VERSION=0.4.0 2 | RELEASE_BUILD_PATH=./cmd/scanrepo 3 | --------------------------------------------------------------------------------