├── .gitignore ├── Makefile ├── README.md ├── matchers.json ├── scan-list.go ├── scanner.go └── security-scan.go /.gitignore: -------------------------------------------------------------------------------- 1 | security-scan.csv 2 | build 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | default: build 2 | 3 | .PHONY: build 4 | 5 | build: fmt vet 6 | go build -o ./build/security-scan 7 | 8 | install: 9 | go install github.com/onetwopunch/security-scan 10 | 11 | fmt: 12 | go fmt 13 | 14 | vet: 15 | go vet 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Security Scan 2 | 3 | This simple Go tool originally forked from https://github.com/stefansundin/secrets-scanner scans a Git history for things that look like passwords or secrets, and outputs the results complete with filename, commit SHA, author, etc to a CSV file. 4 | 5 | It is completely customizable in that you can input your own custom matcher if those that are given are not enough. 6 | 7 | ### Installation 8 | 9 | To install just run: 10 | 11 | ``` 12 | go install github.com/onetwopunch 13 | ``` 14 | 15 | ### Usage 16 | 17 | Navigate to the git repo working directory you want to scan and run: 18 | 19 | ``` 20 | security-scan 21 | ``` 22 | 23 | You'll get a text report that look like this: 24 | 25 | ``` 26 | Reading Git history from /Users/ryan/Workspace/repo... 27 | [Password or Secret Assignment] Matches Found: 37 28 | [AWS Access Key ID] Matches Found: 85 29 | [Redis URL with Password] Matches Found: 19 30 | [URL Basic auth] Matches Found: 18 31 | [Google Access Token] Matches Found: 8 32 | [Google API] Matches Found: 0 33 | [Slack API] Matches Found: 0 34 | [Slack Bot] Matches Found: 0 35 | [Gem Fury v1] Matches Found: 0 36 | [Gem Fury v2] Matches Found: 0 37 | ``` 38 | 39 | And a CSV file will be generated in the current directory. For more details about usage, just run: 40 | 41 | ``` 42 | $ security-scan -h 43 | 44 | Usage of security-scan: 45 | -git string 46 | Git working directory to scan (defaults to current working directory) 47 | -h Usage 48 | -m string 49 | JSON file containing a list of matchers 50 | [ 51 | { 52 | "description":string, 53 | "regex":string 54 | }, ... 55 | ] 56 | (default "$GOPATH/src/github.com/onetwopunch/security-scan/matchers.json") 57 | -o string 58 | Output CSV filename (default "security-scan.csv") 59 | ``` 60 | 61 | The matchers file is pretty self-explanatory: you can add or remove matchers to fit your org's needs. If you need to make changes, just run: 62 | 63 | ``` 64 | cp $GOPATH/src/github.com/onetwopunch/security-scan/matchers.json my_matchers.json 65 | ``` 66 | 67 | Then edit it to your heart's desire and to see the results, run: 68 | 69 | ``` 70 | security-scan -m my_matchers.json -git /path/to/repo 71 | ``` 72 | 73 | ### Contributing 74 | 75 | If you feel like your matcher is generic enough to add to the default, please feel free to submit a PR. As well, PR's are always welcome. 76 | 77 | ### Development 78 | 79 | I like to use Make to encapsulate some common commands...sue me. It makes things nice when dealing with Go, so if you want to build just run: 80 | 81 | ``` 82 | make 83 | ``` 84 | 85 | Which will also run `fmt` and `vet`. The output will be in build, so to test just run `make` and then execute `./build/security-scan` with whatever args you need. 86 | 87 | ``` 88 | make install 89 | ``` 90 | 91 | Will install into your Go path 92 | -------------------------------------------------------------------------------- /matchers.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "description": "Password or Secret Assignment", 4 | "regex": "(password|secret) = ['\"].*?" 5 | }, 6 | { 7 | "description": "AWS Access Key ID", 8 | "regex": "AKIA[0-9A-Z]{16}" 9 | }, 10 | { 11 | "description": "Redis URL with Password", 12 | "regex": "redis://[0-9a-zA-Z:@.\\-]+" 13 | }, 14 | { 15 | "description": "URL Basic auth", 16 | "regex": "https?://[0-9a-zA-z_]+?:[0-9a-zA-z_]+?@.+?" 17 | }, 18 | { 19 | "description":"Google Access Token", 20 | "regex": "ya29.[0-9a-zA-Z_\\-]{68}" 21 | }, 22 | { 23 | "description": "Google API", 24 | "regex": "AIzaSy[0-9a-zA-Z_\\-]{33}" 25 | }, 26 | { 27 | "description": "Slack API", 28 | "regex": "xoxp-\\d+-\\d+-\\d+-[0-9a-f]+" 29 | }, 30 | { 31 | "description": "Slack Bot", 32 | "regex": "xoxb-\\d+-[0-9a-zA-Z]+" 33 | }, 34 | { 35 | "description": "Gem Fury v1", 36 | "regex": "https?://[0-9a-zA-Z]+@[a-z]+\\.(gemfury.com|fury.io)(/[a-z]+)?" 37 | }, 38 | { 39 | "description": "Gem Fury v2", 40 | "regex": "https?://[a-z]+\\.(gemfury.com|fury.io)/[0-9a-zA-Z]{20}" 41 | } 42 | ] 43 | -------------------------------------------------------------------------------- /scan-list.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "io/ioutil" 6 | "log" 7 | ) 8 | 9 | type scannerEntry struct { 10 | Description string `json:"description"` 11 | Regex string `json:"regex"` 12 | } 13 | 14 | type scanList []scannerEntry 15 | 16 | func NewScanList(filepath string) []*Scanner { 17 | var list scanList 18 | var data []byte 19 | var err error 20 | 21 | if data, err = ioutil.ReadFile(filepath); err != nil { 22 | log.Fatal(err) 23 | } 24 | 25 | if err = json.Unmarshal(data, &list); err != nil { 26 | log.Fatal(err) 27 | } 28 | var scanners []*Scanner 29 | for _, entry := range list { 30 | scanner := NewScanner(entry.Description, entry.Regex) 31 | scanners = append(scanners, scanner) 32 | } 33 | return scanners 34 | } 35 | -------------------------------------------------------------------------------- /scanner.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | ) 7 | 8 | type Match struct { 9 | line string 10 | filename string 11 | commit string 12 | author string 13 | } 14 | 15 | func (match *Match) ArrayWithDescription(description string) []string { 16 | return []string{ 17 | description, 18 | match.filename, 19 | match.commit, 20 | match.author, 21 | match.line, 22 | } 23 | } 24 | 25 | type Scanner struct { 26 | description string 27 | re *regexp.Regexp 28 | matches []Match 29 | currAuthor string 30 | currFilename string 31 | currCommit string 32 | } 33 | 34 | var reFilename *regexp.Regexp = regexp.MustCompile("[+-]{3} (a/(.*?$)|b/(.*?$))") 35 | var reCommit *regexp.Regexp = regexp.MustCompile("^commit ([a-f0-9]{40})") 36 | var reAuthor *regexp.Regexp = regexp.MustCompile("Author: (.*?) <") 37 | 38 | func NewScanner(description string, pattern string) *Scanner { 39 | scanner := Scanner{description: description} 40 | scanner.re = regexp.MustCompile(pattern) 41 | return &scanner 42 | } 43 | 44 | func (me *Scanner) ScanLine(line string) bool { 45 | var ok bool 46 | var filename, commit, author string 47 | 48 | if ok, filename = getFilename(line); ok { 49 | me.currFilename = filename 50 | return false 51 | } 52 | 53 | if ok, commit = getCommit(line); ok { 54 | me.currCommit = commit 55 | return false 56 | } 57 | 58 | if ok, author = getAuthor(line); ok { 59 | me.currAuthor = author 60 | return false 61 | } 62 | 63 | match := me.re.FindString(line) 64 | if len(match) > 0 { 65 | trimmed := strings.Trim(line, "\n") 66 | match := Match{ 67 | line: trimmed, 68 | author: me.currAuthor, 69 | commit: me.currCommit, 70 | filename: me.currFilename, 71 | } 72 | me.matches = append(me.matches, match) 73 | return true 74 | } 75 | return false 76 | } 77 | 78 | func (me *Scanner) Records() [][]string { 79 | res := [][]string{} 80 | for _, match := range me.matches { 81 | res = append(res, match.ArrayWithDescription(me.description)) 82 | } 83 | return res 84 | } 85 | 86 | func getFilename(line string) (bool, string) { 87 | // We can do it this way because if the filename changes 88 | // we will care what it changed to more than what it 89 | // changed from so the b/filename will overwrite unless 90 | // the file was deleted in which case it will be /dev/null 91 | // Otherwise, the filenames will be the same 92 | 93 | matches := reFilename.FindStringSubmatch(line) 94 | if len(matches) < 4 { 95 | return false, "" 96 | } 97 | if len(matches[2]) > 0 { 98 | return true, matches[2] 99 | } 100 | return true, matches[3] 101 | } 102 | 103 | func getCommit(line string) (bool, string) { 104 | matches := reCommit.FindStringSubmatch(line) 105 | if len(matches) > 1 { 106 | return true, matches[1] 107 | } 108 | return false, "" 109 | } 110 | 111 | func getAuthor(line string) (bool, string) { 112 | matches := reAuthor.FindStringSubmatch(line) 113 | if len(matches) > 1 { 114 | return true, matches[1] 115 | } 116 | return false, "" 117 | } 118 | -------------------------------------------------------------------------------- /security-scan.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/csv" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "os" 10 | "os/exec" 11 | "path/filepath" 12 | "strings" 13 | ) 14 | 15 | func main() { 16 | var err error 17 | var outfile *os.File 18 | var lines []string 19 | 20 | dir := flag.String("git", "", "Git working directory to scan (defaults to current working directory)") 21 | output := flag.String("o", "security-scan.csv", "Output CSV filename") 22 | gopath := os.Getenv("GOPATH") 23 | defaultMatchersPath := filepath.Join(gopath, "src", "github.com", "onetwopunch", "security-scan", "matchers.json") 24 | matchers := flag.String("m", defaultMatchersPath, "JSON file containing a list of matchers\n\t[\n\t {\n\t \"description\":string,\n\t \"regex\":string\n\t }, ...\n\t]\n\t") 25 | scanners := NewScanList(*matchers) 26 | help := flag.Bool("h", false, "Usage") 27 | flag.Parse() 28 | if *help { 29 | os.Exit(Usage()) 30 | } 31 | 32 | os.MkdirAll(filepath.Dir(*output), 0755) 33 | outfile, err = os.OpenFile(*output, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666) 34 | if err != nil { 35 | log.Fatal(err) 36 | } 37 | fmt.Printf("Reading Git history from %s...\n", *dir) 38 | lines, err = gitLog(*dir) 39 | if err != nil { 40 | log.Fatal(err) 41 | Usage() 42 | } 43 | writer := csv.NewWriter(outfile) 44 | header := []string{"Description", "Filename", "Commit", "Author", "Line"} 45 | writer.Write(header) 46 | writer.Flush() 47 | 48 | for _, scanner := range scanners { 49 | fmt.Printf("[%v] Matches Found: 0\r", scanner.description) 50 | for _, line := range lines { 51 | if scanner.ScanLine(line) { 52 | fmt.Printf("[%v] Matches Found: %v\r", scanner.description, len(scanner.matches)) 53 | } 54 | } 55 | fmt.Printf("\n") 56 | writer.WriteAll(scanner.Records()) 57 | } 58 | } 59 | 60 | func gitLog(dir string) ([]string, error) { 61 | var out bytes.Buffer 62 | var err bytes.Buffer 63 | 64 | cmd := exec.Command("git", "log", "-p") 65 | cmd.Stderr = &err 66 | cmd.Stdout = &out 67 | cmd.Dir = dir 68 | cmd.Run() 69 | if err.Len() > 0 { 70 | return []string{}, fmt.Errorf("%v", err.String()) 71 | } 72 | lines := strings.Split(out.String(), "\n") 73 | return lines, nil 74 | } 75 | 76 | func Usage() int { 77 | fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0]) 78 | flag.PrintDefaults() 79 | return 1 80 | } 81 | --------------------------------------------------------------------------------