├── rules
    ├── unix.rule
    ├── aws.rule
    ├── certs.rule
    └── password.rule
├── AUTHORS
├── .travis.yml
├── source.go
├── go.mod
├── cmd
    └── seekret
    │   ├── known.go
    │   ├── format.go
    │   └── main.go
├── models
    ├── secret.go
    ├── rule.go
    ├── exception.go
    ├── rule_test.go
    ├── exception_test.go
    ├── object.go
    └── object_test.go
├── inspect.go
├── doc.go
├── go.sum
├── README.rst
├── seekret.go
└── LICENSE


/rules/unix.rule:
--------------------------------------------------------------------------------
1 | passwd:
2 |   match: .*\w+:[^:]:\d+:\d+:[\w ]+:[\w\/]+:[\w\/]+.*


--------------------------------------------------------------------------------
/rules/aws.rule:
--------------------------------------------------------------------------------
1 | secret_key:
2 |   match: .*secret(_)?key\s*=\s*(.*)?[A-Za-z0-9\/]{40}(.*)?.*
3 | 
4 | access_key:
5 |   match: .*access(_)?key\*=\*(.*)?[A-Z0-9]{20}(.*)?.*


--------------------------------------------------------------------------------
/rules/certs.rule:
--------------------------------------------------------------------------------
1 | generic:
2 |   match: .*-----BEGIN PRIVATE KEY-----.*
3 | 
4 | pgp:
5 |   match: .*-----BEGIN PGP PRIVATE KEY BLOCK-----.*
6 | 
7 | rsa:
8 |   match: .*-----BEGIN RSA PRIVATE KEY-----.*


--------------------------------------------------------------------------------
/AUTHORS:
--------------------------------------------------------------------------------
1 | # This is the official list of seekret authors for copyright purposes.
2 | 
3 | # Names should be added to this file as
4 | #	Name or Organization <email address> contribution
5 | #		Contribution
6 | # The email address is not required for organizations.
7 | 
8 | Albert Puigsech Galicia <albert@puigsech.com>
9 | 	Main developer


--------------------------------------------------------------------------------
/rules/password.rule:
--------------------------------------------------------------------------------
 1 | password:
 2 |   match: .*password\s*=\s*.*[^ \'";,\*].*
 3 |   unmatch: 
 4 |     - =.*password.*
 5 | 
 6 | pwd:
 7 |   match: .*pwd\s*=\s*.*[^ \'";,\*].*
 8 |   unmatch: 
 9 |     - =.*password.*
10 | 
11 | pass:
12 |   match: .*pass\s*=\s*.*[^ \'";,\*].*
13 |   unmatch: 
14 |     - =.*password.*
15 | 
16 | cred:
17 |   match: .*cred\s*=\s*.*[^ \'";,\*].*
18 |   unmatch: 
19 |     - =.*password.*


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: go
 2 | go:
 3 |   - 1.13.3
 4 | dist: bionic
 5 | sudo: required
 6 | 
 7 | before_install:
 8 |   - sudo sh -c 'echo "deb http://archive.ubuntu.com/ubuntu/ xenial main restricted universe" >> /etc/apt/sources.list'
 9 |   - sudo apt-get update
10 |   - sudo apt-get install -y pkg-config libgit2-dev
11 |   - sudo apt-get clean
12 | 
13 | install:
14 |   - go get github.com/apuigsech/seekret/cmd/seekret
15 | 


--------------------------------------------------------------------------------
/source.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2016 - Authors included on AUTHORS file.
 2 | //
 3 | // Use of this source code is governed by a Apache License
 4 | // that can be found in the LICENSE file.
 5 | 
 6 | package seekret
 7 | 
 8 | import (
 9 | 	"github.com/apuigsech/seekret/models"
10 | )
11 | 
12 | type LoadOptions map[string]interface{}
13 | 
14 | type SourceType interface {
15 | 	LoadObjects(source string, opt LoadOptions) ([]models.Object, error)
16 | }
17 | 


--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
 1 | module github.com/apuigsech/seekret
 2 | 
 3 | go 1.13
 4 | 
 5 | require (
 6 | 	github.com/apuigsech/seekret v0.0.0-20161231092511-9b1f7ea1b3fd
 7 | 	github.com/apuigsech/seekret-source-dir v0.0.0-20161101151956-464d81254a35
 8 | 	github.com/apuigsech/seekret-source-git v0.0.0-20191113160547-6cc674ca4101
 9 | 	github.com/codahale/blake2 v0.0.0-20150924215134-8d10d0420cbf // indirect
10 | 	github.com/emptyinterface/sshconfig v0.0.0-20161024154117-9c4bfc0173e2 // indirect
11 | 	github.com/urfave/cli v1.22.1
12 | 	golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708
13 | 	gopkg.in/libgit2/git2go.v26 v26.0.0-20190104134959-c9f7fd544d39 // indirect
14 | 	gopkg.in/yaml.v2 v2.2.5
15 | )
16 | 


--------------------------------------------------------------------------------
/cmd/seekret/known.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2016 - Authors included on AUTHORS file.
 2 | //
 3 | // Use of this source code is governed by a Apache License
 4 | // that can be found in the LICENSE file.
 5 | 
 6 | package main
 7 | 
 8 | import (
 9 | 	"bufio"
10 | 	"github.com/apuigsech/seekret"
11 | 	"github.com/apuigsech/seekret/models"
12 | 	"os"
13 | 	"path/filepath"
14 | )
15 | 
16 | func LoadKnownFromFile(s *seekret.Seekret, file string) error {
17 | 	if file == "" {
18 | 		return nil
19 | 	}
20 | 
21 | 	filename, _ := filepath.Abs(file)
22 | 
23 | 	fh, err := os.Open(filename)
24 | 	if err != nil {
25 | 		return err
26 | 	}
27 | 	defer fh.Close()
28 | 
29 | 	scanner := bufio.NewScanner(fh)
30 | 	for scanner.Scan() {
31 | 		rule, err := models.NewRule("known", scanner.Text())
32 | 		if err != nil {
33 | 			return err
34 | 		}
35 | 		s.AddRule(*rule, true)
36 | 	}
37 | 
38 | 	if err := scanner.Err(); err != nil {
39 | 		return err
40 | 	}
41 | 
42 | 	return nil
43 | }
44 | 


--------------------------------------------------------------------------------
/models/secret.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2016 - Authors included on AUTHORS file.
 2 | //
 3 | // Use of this source code is governed by a Apache License
 4 | // that can be found in the LICENSE file.
 5 | 
 6 | package models
 7 | 
 8 | // Represents a found secret.
 9 | type Secret struct {
10 | 	// Object in witch the secret is found.
11 | 	Object *Object
12 | 
13 | 	// Rule that matches.
14 | 	Rule *Rule
15 | 
16 | 	// Number of line in the content that contains the secret.
17 | 	Nline int
18 | 
19 | 	// Content of the specific line.
20 | 	Line string
21 | 
22 | 	// Specifies if this matches an exception too.
23 | 	Exception bool
24 | }
25 | 
26 | // NewSecret creates a new secret.
27 | func NewSecret(object *Object, rule *Rule, nLine int, line string) *Secret {
28 | 	s := &Secret{
29 | 		Object: object,
30 | 		Rule:   rule,
31 | 		Nline:  nLine,
32 | 		Line:   line,
33 | 	}
34 | 	return s
35 | }
36 | 
37 | // SetException specifies that a found secret is an exception (of false positive).
38 | func (s *Secret) SetException(exception bool) {
39 | 	s.Exception = exception
40 | }
41 | 


--------------------------------------------------------------------------------
/cmd/seekret/format.go:
--------------------------------------------------------------------------------
 1 | // Copyright 2016 - Authors included on AUTHORS file.
 2 | //
 3 | // Use of this source code is governed by a Apache License
 4 | // that can be found in the LICENSE file.
 5 | 
 6 | package main
 7 | 
 8 | import (
 9 | 	"fmt"
10 | 	"github.com/apuigsech/seekret/models"
11 | )
12 | 
13 | func FormatOutput(secretList []models.Secret, format string) string {
14 | 	var out string
15 | 
16 | 	switch format {
17 | 	case "human":
18 | 		out, _ = formatOutputHuman(secretList)
19 | 	case "json":
20 | 		out, _ = formatOutputJSON(secretList)
21 | 	case "xml":
22 | 		out, _ = formatOutputJSON(secretList)
23 | 	case "csv":
24 | 		out, _ = formatOutputCSV(secretList)
25 | 	default:
26 | 		out, _ = formatOutputHuman(secretList)
27 | 	}
28 | 
29 | 	return out
30 | }
31 | 
32 | func formatOutputHuman(secretList []models.Secret) (string, error) {
33 | 	var out string
34 | 	for _, s := range secretList {
35 | 		out = out + fmt.Sprintf("%s\n\t%d: [%s] %s %t\n", s.Object.Name, s.Nline, s.Rule.Name, s.Line, s.Exception)
36 | 	}
37 | 	return out, nil
38 | }
39 | 
40 | // TODO: Implement
41 | func formatOutputJSON(secretList []models.Secret) (string, error) {
42 | 	return "Not implemented", nil
43 | }
44 | 
45 | func formatOutputXML(secretList []models.Secret) (string, error) {
46 | 	return "Not implemented", nil
47 | }
48 | 
49 | func formatOutputCSV(secretList []models.Secret) (string, error) {
50 | 	return "Not implemented", nil
51 | }
52 | 


--------------------------------------------------------------------------------
/models/rule.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2016 - Authors included on AUTHORS file.
  2 | //
  3 | // Use of this source code is governed by a Apache License
  4 | // that can be found in the LICENSE file.
  5 | 
  6 | package models
  7 | 
  8 | import (
  9 | 	"bufio"
 10 | 	"bytes"
 11 | 	"fmt"
 12 | 	"regexp"
 13 | )
 14 | 
 15 | // Represents a Rule.
 16 | type Rule struct {
 17 | 	// Contains the name of the rule.
 18 | 	Name string
 19 | 
 20 | 	// Specifies if the rule is enabled or not.
 21 | 	Enabled bool
 22 | 
 23 | 	// All lines of the content are analised separatelly.
 24 | 	// For a line to be considered a secret it should match the Match regular
 25 | 	// expression and not match any of the regular expressions contained on the
 26 | 	// Unmacth array.
 27 | 	Match   *regexp.Regexp
 28 | 	Unmatch []*regexp.Regexp
 29 | }
 30 | 
 31 | type RunResult struct {
 32 | 	Line  string
 33 | 	Nline int
 34 | }
 35 | 
 36 | // NewRule creates a new rule.
 37 | func NewRule(name string, match string) (*Rule, error) {
 38 | 	matchRegexp, err := regexp.Compile("(?i)" + match)
 39 | 	if err != nil {
 40 | 		return nil, err
 41 | 	}
 42 | 	if err != nil {
 43 | 		fmt.Println(err)
 44 | 	}
 45 | 
 46 | 	r := &Rule{
 47 | 		Enabled: false,
 48 | 		Name:    name,
 49 | 		Match:   matchRegexp,
 50 | 	}
 51 | 	return r, nil
 52 | }
 53 | 
 54 | // Enable marks the rule as enabled.
 55 | func (r *Rule) Enable() {
 56 | 	r.Enabled = true
 57 | }
 58 | 
 59 | // Enable marks the rule as disabled.
 60 | func (r *Rule) Disable() {
 61 | 	r.Enabled = false
 62 | }
 63 | 
 64 | // AddUnmatch adds a refular expression into the unmatch list.
 65 | func (r *Rule) AddUnmatch(unmatch string) error {
 66 | 	unmatchRegexp, err := regexp.Compile("(?i)" + unmatch)
 67 | 	if err != nil {
 68 | 		return err
 69 | 	}
 70 | 
 71 | 	r.Unmatch = append(r.Unmatch, unmatchRegexp)
 72 | 
 73 | 	return nil
 74 | }
 75 | 
 76 | // Run executes the rule into a content to find all lines that matches it.
 77 | func (r *Rule) Run(content []byte) []RunResult {
 78 | 	var results []RunResult
 79 | 
 80 | 	b := bufio.NewScanner(bytes.NewReader(content))
 81 | 
 82 | 	nLine := 0
 83 | 	for b.Scan() {
 84 | 		nLine = nLine + 1
 85 | 		line := b.Text()
 86 | 
 87 | 		if r.Match.MatchString(line) {
 88 | 			unmatch := false
 89 | 			for _, Unmatch := range r.Unmatch {
 90 | 				if Unmatch.MatchString(line) {
 91 | 					unmatch = true
 92 | 				}
 93 | 			}
 94 | 
 95 | 			if !unmatch {
 96 | 				results = append(results, RunResult{
 97 | 					Line:  line,
 98 | 					Nline: nLine,
 99 | 				})
100 | 			}
101 | 		}
102 | 	}
103 | 
104 | 	return results
105 | }
106 | 


--------------------------------------------------------------------------------
/inspect.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2016 - Authors included on AUTHORS file.
  2 | //
  3 | // Use of this source code is governed by a Apache License
  4 | // that can be found in the LICENSE file.
  5 | 
  6 | package seekret
  7 | 
  8 | import (
  9 | 	"bufio"
 10 | 	"bytes"
 11 | 	"github.com/apuigsech/seekret/models"
 12 | )
 13 | 
 14 | type workerJob struct {
 15 | 	objectGroup   []models.Object
 16 | 	ruleList      []models.Rule
 17 | 	exceptionList []models.Exception
 18 | }
 19 | 
 20 | type workerResult struct {
 21 | 	wid        int
 22 | 	secretList []models.Secret
 23 | }
 24 | 
 25 | // Inspect executes the inspection into all loaded objects, by checking all
 26 | // rules and exceptions loaded.
 27 | func (s *Seekret) Inspect(Nworkers int) {
 28 | 	jobs := make(chan workerJob)
 29 | 	results := make(chan workerResult)
 30 | 
 31 | 	for w := 1; w <= Nworkers; w++ {
 32 | 		go inspect_worker(w, jobs, results)
 33 | 	}
 34 | 
 35 | 	objectGroupMap := s.GroupObjectsByPrimaryKeyHash()
 36 | 
 37 | 	go func() {
 38 | 		for _, objectGroup := range objectGroupMap {
 39 | 			jobs <- workerJob{
 40 | 				objectGroup:   objectGroup,
 41 | 				ruleList:      s.ruleList,
 42 | 				exceptionList: s.exceptionList,
 43 | 			}
 44 | 		}
 45 | 		close(jobs)
 46 | 	}()
 47 | 
 48 | 	for i := 0; i < len(objectGroupMap); i++ {
 49 | 		result := <-results
 50 | 		s.secretList = append(s.secretList, result.secretList...)
 51 | 	}
 52 | }
 53 | 
 54 | func inspect_worker(id int, jobs <-chan workerJob, results chan<- workerResult) {
 55 | 	lid := 0
 56 | 	for job := range jobs {
 57 | 		lid = lid + 1
 58 | 		result := workerResult{
 59 | 			wid: id,
 60 | 		}
 61 | 
 62 | 		content := job.objectGroup[0].Content
 63 | 
 64 | 		for ri,r := range job.ruleList {
 65 | 			if r.Enabled == false {
 66 | 				continue
 67 | 			}
 68 | 
 69 | 			fs := bufio.NewScanner(bytes.NewReader(content))
 70 | 			buf := []byte{}
 71 | 
 72 | 			// INFO: Remove the next line if using golang < 1.6
 73 | 			fs.Buffer(buf, models.MaxObjectContentLen)
 74 | 
 75 | 			nLine := 0
 76 | 			for fs.Scan() {
 77 | 				nLine = nLine + 1
 78 | 				line := fs.Text()
 79 | 
 80 | 				runResultList := r.Run([]byte(line))
 81 | 
 82 | 				for oi,_ := range job.objectGroup {
 83 | 					for _, runResult := range runResultList {
 84 | 						secret := models.NewSecret(&job.objectGroup[oi], &job.ruleList[ri], runResult.Nline, runResult.Line)
 85 | 						secret.SetException(exceptionCheck(job.exceptionList, *secret))
 86 | 						result.secretList = append(result.secretList, *secret)
 87 | 					}
 88 | 				}
 89 | 
 90 | 			}
 91 | 		}
 92 | 
 93 | 		results <- result
 94 | 	}
 95 | }
 96 | 
 97 | func exceptionCheck(exceptionList []models.Exception, secret models.Secret) bool {
 98 | 	for _, x := range exceptionList {
 99 | 		match := x.Run(&secret)
100 | 
101 | 		if match {
102 | 			return true
103 | 		}
104 | 	}
105 | 	return false
106 | }
107 | 


--------------------------------------------------------------------------------
/models/exception.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2016 - Authors included on AUTHORS file.
  2 | //
  3 | // Use of this source code is governed by a Apache License
  4 | // that can be found in the LICENSE file.
  5 | 
  6 | package models
  7 | 
  8 | import (
  9 | 	"regexp"
 10 | )
 11 | 
 12 | // Represents an Exception. In order for a secret to be considered as exception
 13 | // all non-nill attributes should match with the secret information. That means
 14 | // it's considered like and AND statement.
 15 | type Exception struct {
 16 | 	Name string
 17 | 
 18 | 	// Regular expresion that should match the name of the rule.
 19 | 	Rule *regexp.Regexp
 20 | 
 21 | 	// Regular expresion that should match the name of the object.
 22 | 	Object *regexp.Regexp
 23 | 
 24 | 	// Number of line where the secret is found in the contect of the object.
 25 | 	Nline *int
 26 | 
 27 | 	// Regular expresion that should match the content of the object.
 28 | 	Content *regexp.Regexp
 29 | }
 30 | 
 31 | // NewException creates a new exception.
 32 | func NewException() *Exception {
 33 | 	x := &Exception{
 34 | 		Name: "anonymous",
 35 | 	}
 36 | 	return x
 37 | }
 38 | 
 39 | // SetRule sets the regular expresion that should match the name of the rule.
 40 | func (x *Exception) SetRule(rule string) error {
 41 | 	ruleRegexp, err := regexp.Compile("(?i)" + rule)
 42 | 	if err != nil {
 43 | 		return err
 44 | 	}
 45 | 	x.Rule = ruleRegexp
 46 | 	return nil
 47 | }
 48 | 
 49 | // SetObject sets the regular expresion that should match the name of the
 50 | // object.
 51 | func (x *Exception) SetObject(object string) error {
 52 | 	objectRegexp, err := regexp.Compile("(?i)" + object)
 53 | 	if err != nil {
 54 | 		return err
 55 | 	}
 56 | 	x.Object = objectRegexp
 57 | 	return nil
 58 | }
 59 | 
 60 | // SetNline sets the number of line where secret should be found.
 61 | func (x *Exception) SetNline(nLine int) error {
 62 | 	x.Nline = &nLine
 63 | 	return nil
 64 | }
 65 | 
 66 | // SetContent sets the regular expresion that should match the content of the
 67 | // object.
 68 | func (x *Exception) SetContent(content string) error {
 69 | 	contentRegexp, err := regexp.Compile("(?i)" + content)
 70 | 	if err != nil {
 71 | 		return err
 72 | 	}
 73 | 	x.Content = contentRegexp
 74 | 	return nil
 75 | }
 76 | 
 77 | // Run executes the exception into a secret to determine if it's an exception
 78 | // or not.
 79 | func (x *Exception) Run(s *Secret) bool {
 80 | 	match := true
 81 | 
 82 | 	if match && x.Rule != nil && !x.Rule.MatchString(s.Rule.Name) {
 83 | 		match = false
 84 | 	}
 85 | 
 86 | 	if match && x.Object != nil && !x.Object.MatchString(s.Object.Name) {
 87 | 		match = false
 88 | 	}
 89 | 
 90 | 	if match && x.Nline != nil && *x.Nline != s.Nline {
 91 | 		match = false
 92 | 	}
 93 | 
 94 | 	if match && x.Content != nil && !x.Content.MatchString(s.Line) {
 95 | 		match = false
 96 | 	}
 97 | 
 98 | 	return match
 99 | }
100 | 


--------------------------------------------------------------------------------
/doc.go:
--------------------------------------------------------------------------------
  1 | /*
  2 | 
  3 | Package seekret provides a framework to create tools to inspect information
  4 | looking for sensitive information like passwords, tokens, private keys,
  5 | certificates, etc.
  6 | 
  7 | 
  8 | 
  9 | Basics
 10 | 
 11 | The current trend of automation of all things and de DevOps culture are very 
 12 | beneficial for efficiency but also come with several  problems, being one of
 13 | them the secret provisioning. Bootstrapping secrets into systems and
 14 | applications may be complicated and sometimes the straightforward way is to 
 15 | store them into a insecure storage, like github repository, embedded into an
 16 | artifact or system image, etc. That means that an AWS secret_key end up into a
 17 | Github repository.
 18 | 
 19 | Seekret is an extensible framework that gelps in creating tools for detecting
 20 | secrets on different sources. The secrets to detect are defined by a set of
 21 | rules that can help detect passwords, tokens, private keys, certificates, etc.
 22 | 
 23 | 
 24 | 
 25 | Tools Using Seekret
 26 | 
 27 | Seekret is extensible and can cover various use cases. Below there are some
 28 | tools that uses seekret:
 29 | 
 30 |   git-seekret: https://github.com/apuigsech/git-seekret
 31 | 
 32 |     Git module that uses local hooks to help develepers to prevent leaking 
 33 |     sensitive information in a commit.
 34 | 
 35 | 
 36 | 
 37 | Using It
 38 | 
 39 | Seekret API is very simple and easy to use. This section shows some snippets of
 40 | code that shows the basic operations you can do with it.
 41 | 
 42 | The first thing to be done is to create a new Seekret context:
 43 | 
 44 | 	s := seekret.NewSeekret()
 45 | 
 46 | Then the rules must to be loaded. They can be loaded from a path definition, a
 47 | directory or a single file:
 48 | 
 49 | 	s.LoadRulesFromPath("/path/to/main/rues:/path/to/other/rules:/path/to/more/rules")
 50 | 
 51 | 	s.LoadRulesFromDir("/path/to/rules")
 52 | 
 53 | 	s.LoadRulesFromFile("/path/to/file.rule")
 54 | 
 55 | Optionally, exceptions (or false positives) can also be loaded from a file:
 56 | 
 57 | 	s.LoadExceptionsFromFile("/path/to/exceptions/file")
 58 | 
 59 | After that, must be loaded the objects to be inspected searching for secrets.
 60 | 
 61 | 	opts := map[string]interface{} {
 62 |   		// Loading options.
 63 | 	}
 64 | 	s.LoadObjects(sourceType, source, opts)
 65 | 
 66 | sourceType is an interface that implements the interface shown below. We offer
 67 | sourceType's for Directories and Git Repositories, but you are able to extend
 68 | it by creating your own.
 69 | 
 70 | 	type SourceType interface {
 71 | 		LoadObjects(source string, opt LoadOptions) ([]models.Object, error)
 72 | 	}
 73 | 
 74 | Currently, there are the following different sources supported:
 75 | 
 76 |   Directories (and files): https://github.com/apuigsech/seekret-source-dir
 77 | 
 78 |     Load all files contained in a directory (and its sub-directories).
 79 | 
 80 |   Git Repositories: https://github.com/apuigsech/seekret-source-git
 81 | 
 82 |     Load git objects from commits or staging area.
 83 | 
 84 | Having all the rules, exceptions and objects loaded into the contects, it's
 85 | possible to start the inspection with the following code:
 86 | 
 87 | 	s.Inspect(Nworkers)
 88 | 
 89 | Nworkers is an integuer that specify the number of goroutines used on the
 90 | inspection. The recommended value is runtime.NumCPU().
 91 | 
 92 | Finally, it is possible to obtain the list of secrets located and do
 93 | something with them:
 94 | 
 95 | 		secretsList := s.ListSecrets()
 96 | 		for secret := range secretsList {
 97 | 			// Do something
 98 | 		}
 99 | 
100 | */
101 | package seekret
102 | 


--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 2 | github.com/apuigsech/seekret v0.0.0-20161231092511-9b1f7ea1b3fd h1:fC+kH7BK48k8xmBAQq/LkP9HZ51BoKS+JSb+yBPLx2k=
 3 | github.com/apuigsech/seekret v0.0.0-20161231092511-9b1f7ea1b3fd/go.mod h1:MQe9ewnl3lqr6Ien+Pr9iZRlDIhzXdu4HjtPI6PCGlI=
 4 | github.com/apuigsech/seekret-source-dir v0.0.0-20161101151956-464d81254a35 h1:9oE1yQZuU8OIf90XU5cqHLwiIhKsPGJGIzyR6+Ctikw=
 5 | github.com/apuigsech/seekret-source-dir v0.0.0-20161101151956-464d81254a35/go.mod h1:yBvRALbfKC6BkRLBfI3MjtsmnjvfmKaFBZlOn2PDeHQ=
 6 | github.com/apuigsech/seekret-source-git v0.0.0-20191113160547-6cc674ca4101 h1:ZsvqPJ8jbjxZ3sv/FyHkNKf7leTS3HrpztWEml5UBUQ=
 7 | github.com/apuigsech/seekret-source-git v0.0.0-20191113160547-6cc674ca4101/go.mod h1:2rtO0huhwkRu6mLvXYBLAp2M5fyocBhpMb1+YbkZMJA=
 8 | github.com/codahale/blake2 v0.0.0-20150924215134-8d10d0420cbf h1:5ZeQB3mThuz5C2MSER6T5GdtXTF9CMMk42F9BOyRsEQ=
 9 | github.com/codahale/blake2 v0.0.0-20150924215134-8d10d0420cbf/go.mod h1:BO2rLUAZMrpgh6GBVKi0Gjdqw2MgCtJrtmUdDeZRKjY=
10 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
11 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
12 | github.com/emptyinterface/sshconfig v0.0.0-20161024154117-9c4bfc0173e2 h1:r68g22c8zYcLao8uTCHkt0AZ60P484MPz+BxuBN0WXM=
13 | github.com/emptyinterface/sshconfig v0.0.0-20161024154117-9c4bfc0173e2/go.mod h1:dFF2pt8AyGGENN5JzxKeVjTf2ISP9gpgshao7mmOL88=
14 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
15 | github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
16 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
17 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
18 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
19 | github.com/urfave/cli v1.22.1 h1:+mkCCcOFKPnCmVYVcURKps1Xe+3zP90gSYGNfRkjoIY=
20 | github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
21 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
22 | golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4=
23 | golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
24 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
25 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
26 | golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
27 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
28 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
29 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
30 | gopkg.in/libgit2/git2go.v26 v26.0.0-20190104134959-c9f7fd544d39 h1:t8ZonYWkloKoMg2pBBSH/LYqtlQ6bs7uySKYqXYEre8=
31 | gopkg.in/libgit2/git2go.v26 v26.0.0-20190104134959-c9f7fd544d39/go.mod h1:Fx2mSUbkLzBXfxYSP0L8/Iz7h0fIUJNDUYeRHFuzKH0=
32 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
33 | gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c=
34 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
35 | 


--------------------------------------------------------------------------------
/models/rule_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2016 - Authors included on AUTHORS file.
  2 | //
  3 | // Use of this source code is governed by a Apache License
  4 | // that can be found in the LICENSE file.
  5 | 
  6 | package models
  7 | 
  8 | import (
  9 | 	"testing"
 10 | )
 11 | 
 12 | type NewRuleSample struct {
 13 | 	name    string
 14 | 	match   string
 15 | 	unmatch []string
 16 | 	ok      bool
 17 | }
 18 | 
 19 | func TestNewRule(t *testing.T) {
 20 | 	testSamples := []NewRuleSample{
 21 | 		{
 22 | 			name:  "rule_1",
 23 | 			match: "match_1",
 24 | 			ok:    true,
 25 | 		},
 26 | 		{
 27 | 			name:  "rule_2",
 28 | 			match: "*",
 29 | 			ok:    false,
 30 | 		},
 31 | 		{
 32 | 			name:  "rule_3",
 33 | 			match: "match_3",
 34 | 			unmatch: []string{
 35 | 				"unmatch_3_1",
 36 | 				"unmatch_3_2",
 37 | 				"unmatch_3_3",
 38 | 			},
 39 | 			ok: true,
 40 | 		},
 41 | 		{
 42 | 			name:  "rule_4",
 43 | 			match: "match_4",
 44 | 			unmatch: []string{
 45 | 				"unmatch_4_1",
 46 | 				"*",
 47 | 				"unmatch_4_3",
 48 | 			},
 49 | 			ok: false,
 50 | 		},
 51 | 	}
 52 | 
 53 | 	for _, ts := range testSamples {
 54 | 		if !testNewRuleSample(ts) {
 55 | 			t.Error("unexpected new rule")
 56 | 		}
 57 | 	}
 58 | }
 59 | 
 60 | func testNewRuleSample(ts NewRuleSample) bool {
 61 | 	r, err := NewRule(ts.name, ts.match)
 62 | 
 63 | 	ok_1 := (err == nil)
 64 | 
 65 | 	ok_2 := true
 66 | 	for _, unmatch := range ts.unmatch {
 67 | 		err := r.AddUnmatch(unmatch)
 68 | 
 69 | 		if err != nil {
 70 | 			ok_2 = false
 71 | 		}
 72 | 	}
 73 | 
 74 | 	ok := (ok_1 && ok_2)
 75 | 
 76 | 	if ok == ts.ok {
 77 | 		return true
 78 | 	} else {
 79 | 		return false
 80 | 	}
 81 | 
 82 | 	return true
 83 | }
 84 | 
 85 | type RunRuleSample struct {
 86 | 	match      string
 87 | 	unmatch    []string
 88 | 	content    []byte
 89 | 	expResults []RunResult
 90 | 	ok         bool
 91 | }
 92 | 
 93 | func TestRunRule(t *testing.T) {
 94 | 	testSamples := []RunRuleSample{
 95 | 		{
 96 | 			match: ".*TEST_1.*",
 97 | 			content: []byte(
 98 | 				"xxx\n" +
 99 | 					"yyy\n" +
100 | 					"xxx TEST_1 yyy\n" +
101 | 					"TEST_1",
102 | 			),
103 | 			expResults: []RunResult{
104 | 				{
105 | 					Line:  "xxx TEST_1 yyy",
106 | 					Nline: 3,
107 | 				},
108 | 				{
109 | 					Line:  "TEST_1",
110 | 					Nline: 4,
111 | 				},
112 | 			},
113 | 			ok: true,
114 | 		},
115 | 		{
116 | 			match: ".*TEST_2.*",
117 | 			unmatch: []string{
118 | 				".*yyy.*",
119 | 				".*zzz.*",
120 | 			},
121 | 			content: []byte(
122 | 				"xxx\n" +
123 | 					"yyy\n" +
124 | 					"xxx TEST_2 yyy\n" +
125 | 					"xxx TEST_2 zzz\n" +
126 | 					"xxx TEST_2 www\n" +
127 | 					"TEST_2",
128 | 			),
129 | 			expResults: []RunResult{
130 | 				{
131 | 					Line:  "xxx TEST_2 www",
132 | 					Nline: 5,
133 | 				},
134 | 				{
135 | 					Line:  "TEST_2",
136 | 					Nline: 6,
137 | 				},
138 | 			},
139 | 			ok: true,
140 | 		},
141 | 	}
142 | 
143 | 	for _, ts := range testSamples {
144 | 		if !testRunRuleSample(ts) {
145 | 			t.Error("unexpected run rule results")
146 | 		}
147 | 	}
148 | }
149 | 
150 | func testRunRuleSample(ts RunRuleSample) bool {
151 | 	r, err := NewRule("rule", ts.match)
152 | 	if err != nil {
153 | 		return false
154 | 	}
155 | 
156 | 	for _, unmatch := range ts.unmatch {
157 | 		err := r.AddUnmatch(unmatch)
158 | 		if err != nil {
159 | 			return false
160 | 		}
161 | 	}
162 | 
163 | 	results := r.Run(ts.content)
164 | 
165 | 	ok := true
166 | 	for _, res := range results {
167 | 		ok_s := false
168 | 		for _, expRes := range ts.expResults {
169 | 			if res.Nline == expRes.Nline && res.Line == expRes.Line {
170 | 				ok_s = true
171 | 			}
172 | 		}
173 | 
174 | 		if ok_s == false {
175 | 			ok = false
176 | 		}
177 | 	}
178 | 
179 | 	if ok == ts.ok {
180 | 		return true
181 | 	} else {
182 | 		return false
183 | 	}
184 | }
185 | 


--------------------------------------------------------------------------------
/models/exception_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2016 - Authors included on AUTHORS file.
  2 | //
  3 | // Use of this source code is governed by a Apache License
  4 | // that can be found in the LICENSE file.
  5 | 
  6 | package models
  7 | 
  8 | import (
  9 | 	"regexp"
 10 | 	"testing"
 11 | )
 12 | 
 13 | type NewExceptionSample struct {
 14 | 	rule    string
 15 | 	object  string
 16 | 	nline   int
 17 | 	content string
 18 | 	ok      bool
 19 | }
 20 | 
 21 | func TestNewException(t *testing.T) {
 22 | 	testSamples := []NewExceptionSample{
 23 | 		{
 24 | 			rule:    "rule_1",
 25 | 			object:  "object_1",
 26 | 			nline:   0,
 27 | 			content: "content_1",
 28 | 			ok:      true,
 29 | 		},
 30 | 		{
 31 | 			rule:    "rule_2",
 32 | 			object:  "object_2",
 33 | 			nline:   0,
 34 | 			content: "*",
 35 | 			ok:      false,
 36 | 		},
 37 | 	}
 38 | 
 39 | 	for _, ts := range testSamples {
 40 | 		if !testNewExceptionSample(ts) {
 41 | 			t.Error("unexpected new exception")
 42 | 		}
 43 | 	}
 44 | }
 45 | 
 46 | func testNewExceptionSample(ts NewExceptionSample) bool {
 47 | 	x := NewException()
 48 | 
 49 | 	ok := true
 50 | 
 51 | 	err := x.SetRule(ts.rule)
 52 | 	if err != nil {
 53 | 		ok = false
 54 | 	}
 55 | 
 56 | 	err = x.SetObject(ts.object)
 57 | 	if err != nil {
 58 | 		ok = false
 59 | 	}
 60 | 
 61 | 	err = x.SetNline(ts.nline)
 62 | 	if err != nil {
 63 | 		ok = false
 64 | 	}
 65 | 
 66 | 	err = x.SetContent(ts.content)
 67 | 	if err != nil {
 68 | 		ok = false
 69 | 	}
 70 | 
 71 | 	if ok == ts.ok {
 72 | 		return true
 73 | 	} else {
 74 | 		return false
 75 | 	}
 76 | }
 77 | 
 78 | type RunExceptionSample struct {
 79 | 	secret    *Secret
 80 | 	exception *Exception
 81 | 	expResult bool
 82 | }
 83 | 
 84 | func TestRunException(t *testing.T) {
 85 | 	testSamples := []RunExceptionSample{
 86 | 		{
 87 | 			secret: &Secret{
 88 | 				Object: &Object{
 89 | 					Name: "object_1",
 90 | 				},
 91 | 				Rule: &Rule{
 92 | 					Name: "rule_1",
 93 | 				},
 94 | 				Nline: 1,
 95 | 				Line:  "secret_1",
 96 | 			},
 97 | 			exception: &Exception{
 98 | 				Rule: regexp.MustCompile("rule_1"),
 99 | 			},
100 | 			expResult: true,
101 | 		},
102 | 		{
103 | 			secret: &Secret{
104 | 				Object: &Object{
105 | 					Name: "object_2",
106 | 				},
107 | 				Rule: &Rule{
108 | 					Name: "rule_2",
109 | 				},
110 | 				Nline: 2,
111 | 				Line:  "secret_2",
112 | 			},
113 | 			exception: &Exception{
114 | 				Rule: regexp.MustCompile("rule_X"),
115 | 			},
116 | 			expResult: false,
117 | 		},
118 | 		{
119 | 			secret: &Secret{
120 | 				Object: &Object{
121 | 					Name: "object_3",
122 | 				},
123 | 				Rule: &Rule{
124 | 					Name: "rule_3",
125 | 				},
126 | 				Nline: 3,
127 | 				Line:  "secret_3",
128 | 			},
129 | 			exception: &Exception{
130 | 				Rule:   regexp.MustCompile("rule_3"),
131 | 				Object: regexp.MustCompile("object_3"),
132 | 			},
133 | 			expResult: true,
134 | 		},
135 | 		{
136 | 			secret: &Secret{
137 | 				Object: &Object{
138 | 					Name: "object_4",
139 | 				},
140 | 				Rule: &Rule{
141 | 					Name: "rule_4",
142 | 				},
143 | 				Nline: 4,
144 | 				Line:  "secret_4",
145 | 			},
146 | 			exception: &Exception{
147 | 				Rule:   regexp.MustCompile("rule_4"),
148 | 				Object: regexp.MustCompile("object_X"),
149 | 			},
150 | 			expResult: false,
151 | 		},
152 | 		{
153 | 			secret: &Secret{
154 | 				Object: &Object{
155 | 					Name: "object_5",
156 | 				},
157 | 				Rule: &Rule{
158 | 					Name: "rule_5",
159 | 				},
160 | 				Nline: 5,
161 | 				Line:  "secret_5",
162 | 			},
163 | 			exception: &Exception{
164 | 				Rule:    regexp.MustCompile("rule_5"),
165 | 				Object:  regexp.MustCompile("object_5"),
166 | 				Nline:   intPtr(5),
167 | 				Content: regexp.MustCompile("secret_5"),
168 | 			},
169 | 			expResult: true,
170 | 		},
171 | 		{
172 | 			secret: &Secret{
173 | 				Object: &Object{
174 | 					Name: "object_6",
175 | 				},
176 | 				Rule: &Rule{
177 | 					Name: "rule_6",
178 | 				},
179 | 				Nline: 6,
180 | 				Line:  "secret_6",
181 | 			},
182 | 			exception: &Exception{
183 | 				Rule:    regexp.MustCompile("rule_6"),
184 | 				Object:  regexp.MustCompile("object_X"),
185 | 				Nline:   intPtr(6),
186 | 				Content: regexp.MustCompile("secret_X"),
187 | 			},
188 | 			expResult: false,
189 | 		},
190 | 	}
191 | 
192 | 	for _, ts := range testSamples {
193 | 		if !testRunExceptionSample(ts) {
194 | 			t.Error("unexpected run exception result")
195 | 		}
196 | 	}
197 | }
198 | 
199 | func testRunExceptionSample(ts RunExceptionSample) bool {
200 | 	result := ts.exception.Run(ts.secret)
201 | 
202 | 	if result == ts.expResult {
203 | 		return true
204 | 	} else {
205 | 		return false
206 | 	}
207 | }
208 | 
209 | func intPtr(i int) *int {
210 | 	return &i
211 | }
212 | 


--------------------------------------------------------------------------------
/models/object.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2016 - Authors included on AUTHORS file.
  2 | //
  3 | // Use of this source code is governed by a Apache License
  4 | // that can be found in the LICENSE file.
  5 | 
  6 | package models
  7 | 
  8 | import (
  9 | 	"fmt"
 10 | 	"golang.org/x/crypto/blake2b"
 11 | 	"sort"
 12 | )
 13 | 
 14 | // MaxObjectContentLen contains the maximum size for the content of an object.
 15 | const MaxObjectContentLen = 1024 * 5000
 16 | 
 17 | // Contains a KeyHash or nil
 18 | type KeyHash *[]byte 
 19 | 
 20 | // Represents an object.
 21 | type Object struct {
 22 | 	Type string
 23 | 	SubType string
 24 | 
 25 | 	Name    string
 26 | 	Content []byte
 27 | 
 28 | 	Metadata       map[string]MetadataData
 29 | 	PrimaryKeyHash KeyHash
 30 | }
 31 | 
 32 | // Represents the metadata of an object.
 33 | type MetadataData struct {
 34 | 	value string
 35 | 	attr  MetadataAttributes
 36 | }
 37 | 
 38 | // Represents the attributes of metadata.
 39 | type MetadataAttributes struct {
 40 | 	// All objects with same value on this key has the same content. It's used
 41 | 	// to optimise the inspection.
 42 | 	PrimaryKey bool
 43 | }
 44 | 
 45 | // NewObject creates a new object.
 46 | func NewObject(name string, t string, st string, content []byte) *Object {
 47 | 	if len(content) > MaxObjectContentLen {
 48 | 		content = content[:MaxObjectContentLen]
 49 | 	}
 50 | 	o := &Object{
 51 | 		Type: t,
 52 | 		SubType: st,
 53 | 
 54 | 		Name:    name,
 55 | 		Content: content,
 56 | 
 57 | 		Metadata:       make(map[string]MetadataData),
 58 | 		PrimaryKeyHash: nil,
 59 | 	}
 60 | 	return o
 61 | }
 62 | 
 63 | // SetMetadata sets a metadata value for the object.
 64 | func (o *Object) SetMetadata(key string, value string, attr MetadataAttributes) error {
 65 | 	o.Metadata[key] = MetadataData{
 66 | 		value: value,
 67 | 		attr:  attr,
 68 | 	}
 69 | 
 70 | 	if attr.PrimaryKey {
 71 | 		err := o.updatePrimaryKeyHash()
 72 | 		if err != nil {
 73 | 			return err
 74 | 		}
 75 | 	}
 76 | 
 77 | 	return nil
 78 | }
 79 | 
 80 | // SetMetadata gets a metadata value from the object.
 81 | func (o *Object) GetMetadata(key string) (string, error) {
 82 | 	data, ok := o.Metadata[key]
 83 | 	if !ok {
 84 | 		return "", fmt.Errorf("%s unexistent key", key)
 85 | 	}
 86 | 
 87 | 	return data.value, nil
 88 | }
 89 | 
 90 | // GetMetadataAll gets a map that contains all metadata of the object.
 91 | func (o *Object) GetMetadataAll(attr bool) map[string]string {
 92 | 	metadataAll := make(map[string]string)
 93 | 	for k, v := range o.Metadata {
 94 | 		metadataAll[k] = v.value
 95 | 	}
 96 | 	return metadataAll
 97 | }
 98 | 
 99 | // GetPrimaryKeyHash returns the primary key hash of the object. This hash is
100 | // calculated by using the information of all metadata marked as primary key.
101 | func (o *Object) GetPrimaryKeyHash() KeyHash {
102 | 	return o.PrimaryKeyHash
103 | }
104 | 
105 | func (o *Object) updatePrimaryKeyHash() error {
106 | 	var primayKeyList []string
107 | 	for k, v := range o.Metadata {
108 | 		if v.attr.PrimaryKey {
109 | 			primayKeyList = append(primayKeyList, k)
110 | 		}
111 | 	}
112 | 	sort.Strings(primayKeyList)
113 | 
114 | 	var text string
115 | 	for _, k := range primayKeyList {
116 | 		text = text + fmt.Sprintf("{%s//%s}", k, o.Metadata[k].value)
117 | 	}
118 | 	if text == "" {
119 | 		o.PrimaryKeyHash = nil
120 | 		return nil
121 | 	}
122 | 
123 | 	h, err := blake2b.New(32, nil)
124 | 	if err != nil {
125 | 		return err
126 | 	}
127 | 	h.Write([]byte(text))
128 | 
129 | 	primayKeyHash := h.Sum(nil)
130 | 	o.PrimaryKeyHash = &primayKeyHash
131 | 
132 | 	return nil
133 | }
134 | 
135 | func GroupObjectsByMetadata(objects []Object, k string) map[string][]Object {
136 | 	objectGroups := make(map[string][]Object)
137 | 	for _, o := range objects {
138 | 		v, err := o.GetMetadata(k)
139 | 		if err != nil {
140 | 			fmt.Println(err)
141 | 		}
142 | 
143 | 		var objectList []Object
144 | 		var ok bool
145 | 
146 | 		objectList, ok = objectGroups[v]
147 | 		if !ok {
148 | 			objectList = make([]Object, 0)
149 | 		}
150 | 		objectList = append(objectList, o)
151 | 		objectGroups[v] = objectList
152 | 	}
153 | 	return objectGroups
154 | }
155 | 
156 | func GroupObjectsByPrimaryKeyHash(objects []Object) map[string][]Object {
157 | 	objectGroups := make(map[string][]Object)
158 | 	for _, o := range objects {
159 | 		var objectList []Object
160 | 		var ok bool
161 | 		var key string
162 | 
163 | 		if o.PrimaryKeyHash != nil {
164 | 			key = string(*o.PrimaryKeyHash)
165 | 		} else {
166 | 			key = o.Name
167 | 		}
168 | 
169 | 		objectList, ok = objectGroups[key]
170 | 		if !ok {
171 | 			objectList = make([]Object, 0)
172 | 		}
173 | 		objectList = append(objectList, o)
174 | 		objectGroups[key] = objectList
175 | 	}
176 | 	return objectGroups
177 | }
178 | 


--------------------------------------------------------------------------------
/cmd/seekret/main.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2016 - Authors included on AUTHORS file.
  2 | //
  3 | // Use of this source code is governed by a Apache License
  4 | // that can be found in the LICENSE file.
  5 | 
  6 | package main
  7 | 
  8 | import (
  9 | 	"fmt"
 10 | 	"github.com/apuigsech/seekret"
 11 | 	"github.com/apuigsech/seekret-source-dir"
 12 | 	"github.com/apuigsech/seekret-source-git"
 13 | 	"github.com/urfave/cli"
 14 | 	"os"
 15 | )
 16 | 
 17 | const (
 18 | 	DefaultCommitCount = 10
 19 | )
 20 | 
 21 | var s *seekret.Seekret
 22 | 
 23 | func main() {
 24 | 	s = seekret.NewSeekret()
 25 | 
 26 | 	app := cli.NewApp()
 27 | 
 28 | 	app.Name = "seekret"
 29 | 	app.Version = "0.0.1"
 30 | 	app.Usage = "seek for secrets on various sources."
 31 | 
 32 | 	app.Author = "Albert Puigsech Galicia"
 33 | 	app.Email = "albert@puigsech.com"
 34 | 
 35 | 	app.EnableBashCompletion = true
 36 | 
 37 | 	app.Flags = []cli.Flag{
 38 | 		cli.StringFlag{
 39 | 			Name:  "exception, x",
 40 | 			Usage: "load exceptions from `FILE`.",
 41 | 		},
 42 | 		cli.StringFlag{
 43 | 			Name:   "rules",
 44 | 			Usage:  "`PATH` with rules.",
 45 | 			EnvVar: "SEEKRET_RULES_PATH",
 46 | 		},
 47 | 		cli.StringFlag{
 48 | 			Name:  "format, f",
 49 | 			Usage: "specify the output format.",
 50 | 			Value: "human",
 51 | 		},
 52 | 		// TODO: To be implemented.
 53 | 		/*
 54 | 			cli.StringFlag{
 55 | 				Name: "groupby, g",
 56 | 				Usage: "Group output by specific field",
 57 | 			},
 58 | 		*/
 59 | 		cli.StringFlag{
 60 | 			Name:  "known, k",
 61 | 			Usage: "load known secrets from `FILE`.",
 62 | 		},
 63 | 		cli.IntFlag{
 64 | 			Name:  "workers, w",
 65 | 			Usage: "number of workers used for the inspection",
 66 | 			Value: 4,
 67 | 		},
 68 | 	}
 69 | 
 70 | 	app.Commands = []cli.Command{
 71 | 		{
 72 | 			Name:     "git",
 73 | 			Usage:    "seek for seecrets on a git repository",
 74 | 			Category: "seek",
 75 | 			Action:   seekretGit,
 76 | 
 77 | 			Flags: []cli.Flag{
 78 | 				cli.BoolFlag{
 79 | 					Name:  "commit-files, cf, f",
 80 | 					Usage: "inspect commited files",
 81 | 				},
 82 | 
 83 | 				cli.BoolFlag{
 84 | 					Name:  "commit-messages, cm, m",
 85 | 					Usage: "inspect commit messages",
 86 | 				},
 87 | 				cli.BoolFlag{
 88 | 					Name:  "staged-files, sf, s",
 89 | 					Usage: "inspect staged files",
 90 | 				},
 91 | 				cli.IntFlag{
 92 | 					Name:  "commit-count, cc, c",
 93 | 					Usage: "",
 94 | 					Value: DefaultCommitCount,
 95 | 				},
 96 | 			},
 97 | 		},
 98 | 		{
 99 | 			Name:     "dir",
100 | 			Usage:    "seek for seecrets on a directory",
101 | 			Category: "seek",
102 | 			Action:   seekretDir,
103 | 
104 | 			Flags: []cli.Flag{
105 | 				cli.BoolFlag{
106 | 					Name: "recursive, r",
107 | 				},
108 | 				cli.BoolFlag{
109 | 					Name: "hidden",
110 | 				},
111 | 			},
112 | 		},
113 | 	}
114 | 
115 | 	app.Before = seekretBefore
116 | 	app.After = seekretAfter
117 | 
118 | 	app.Run(os.Args)
119 | }
120 | 
121 | func seekretBefore(c *cli.Context) error {
122 | 	var err error
123 | 
124 | 	rulesPath := c.String("rules")
125 | 
126 | 	err = s.LoadRulesFromPath(rulesPath, true)
127 | 	if err != nil {
128 | 		return err
129 | 	}
130 | 
131 | 	LoadKnownFromFile(s, c.String("known"))
132 | 
133 | 	err = s.LoadExceptionsFromFile(c.String("exception"))
134 | 	if err != nil {
135 | 		return err
136 | 	}
137 | 
138 | 	return nil
139 | }
140 | 
141 | func seekretDir(c *cli.Context) error {
142 | 	source := c.Args().Get(0)
143 | 	if source == "" {
144 | 		cli.ShowSubcommandHelp(c)
145 | 		return nil
146 | 	}
147 | 
148 | 	options := map[string]interface{}{
149 | 		"hidden":    c.Bool("hidden"),
150 | 		"recursive": c.Bool("recursive"),
151 | 	}
152 | 
153 | 	err := s.LoadObjects(sourcedir.SourceTypeDir, source, options)
154 | 	if err != nil {
155 | 		return err
156 | 	}
157 | 
158 | 	return nil
159 | }
160 | 
161 | func seekretGit(c *cli.Context) error {
162 | 	source := c.Args().Get(0)
163 | 	if source == "" {
164 | 		cli.ShowSubcommandHelp(c)
165 | 		return nil
166 | 	}
167 | 
168 | 	// SourceGitLoadOptions composition:
169 | 	//   * commit-files: Include commited file content as object.
170 | 	//   * commit-messages: Include commit contect as object.
171 | 	//   * staged-files: Include stateg dile contect as object.
172 | 	//   * commit-count: Ammount of commits to analise.
173 | 	options := map[string]interface{}{
174 | 		"commit-files": false,
175 | 		"commit-messages": false,
176 | 		"staged-files": false,
177 | 		"commit-count": DefaultCommitCount,
178 | 	}
179 | 
180 | 	if c.IsSet("commit-files") {
181 | 		options["commit-files"] = true
182 | 	}
183 | 
184 | 	if c.IsSet("commit-messages") {
185 | 		options["commit-messages"] = true
186 | 	}
187 | 
188 | 	if c.IsSet("staged-files") {
189 | 		options["staged-files"] = true
190 | 	}
191 | 
192 | 	if c.IsSet("commit-count") {
193 | 		options["commit-count"] = c.Int("commit-count")
194 | 	}
195 | 
196 | 	err := s.LoadObjects(sourcegit.SourceTypeGit, source, options)
197 | 	if err != nil {
198 | 		return err
199 | 	}
200 | 
201 | 	return nil
202 | }
203 | 
204 | func seekretAfter(c *cli.Context) error {
205 | 	s.Inspect(c.Int("workers"))
206 | 
207 | 	fmt.Println(FormatOutput(s.ListSecrets(), c.String("format")))
208 | 
209 | 	return nil
210 | }
211 | 


--------------------------------------------------------------------------------
/models/object_test.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2016 - Authors included on AUTHORS file.
  2 | //
  3 | // Use of this source code is governed by a Apache License
  4 | // that can be found in the LICENSE file.
  5 | 
  6 | package models
  7 | 
  8 | import (
  9 | 	"bytes"
 10 | 	"testing"
 11 | )
 12 | 
 13 | type NewObjectSample struct {
 14 | 	objType    string
 15 | 	objSubtype string
 16 | 	name       string
 17 | 	expName    string
 18 | 	content    []byte
 19 | 	expContent []byte
 20 | 	ok         bool
 21 | }
 22 | 
 23 | func TestNewObject(t *testing.T) {
 24 | 	testSamples := []NewObjectSample{
 25 | 		{
 26 | 			objType:	"type_1",
 27 | 			objSubtype:	"subtype_1",
 28 | 			name:       "test_1",
 29 | 			expName:    "test_1",
 30 | 			content:    []byte("content_1"),
 31 | 			expContent: []byte("content_1"),
 32 | 			ok:         true,
 33 | 		},
 34 | 		{
 35 | 			objType:	"type_2",
 36 | 			objSubtype:	"subtype_2",
 37 | 			name:       "test_2",
 38 | 			expName:    "xxx",
 39 | 			content:    []byte("content_2"),
 40 | 			expContent: []byte("content_2"),
 41 | 			ok:         false,
 42 | 		},
 43 | 		{
 44 | 			objType:	"type_3",
 45 | 			objSubtype:	"subtype_3",
 46 | 			name:       "test_3",
 47 | 			expName:    "test_3",
 48 | 			content:    []byte("content_3"),
 49 | 			expContent: []byte("xxx"),
 50 | 			ok:         false,
 51 | 		},
 52 | 	}
 53 | 
 54 | 	for _, ts := range testSamples {
 55 | 		if !testNewObjectSample(ts) {
 56 | 			t.Error("unexpected new object")
 57 | 		}
 58 | 	}
 59 | }
 60 | 
 61 | func testNewObjectSample(ts NewObjectSample) bool {
 62 | 	o := NewObject(ts.name, ts.objType, ts.objSubtype, ts.content)
 63 | 
 64 | 	ok := (o.Name == ts.expName && bytes.Equal(o.Content, ts.expContent))
 65 | 
 66 | 	if ok == ts.ok {
 67 | 		return true
 68 | 	} else {
 69 | 		return false
 70 | 	}
 71 | }
 72 | 
 73 | type MetadataSample struct {
 74 | 	key        string
 75 | 	value      string
 76 | 	expValue   string
 77 | 	primaryKey bool
 78 | 	ok         bool
 79 | }
 80 | 
 81 | func TestMetadata(t *testing.T) {
 82 | 	testSamples := []MetadataSample{
 83 | 		{
 84 | 			key:        "key_1",
 85 | 			value:      "value_1",
 86 | 			expValue:   "value_1",
 87 | 			primaryKey: false,
 88 | 			ok:         true,
 89 | 		},
 90 | 		{
 91 | 			key:        "key_2",
 92 | 			value:      "value_2",
 93 | 			expValue:   "value_2",
 94 | 			primaryKey: true,
 95 | 			ok:         true,
 96 | 		},
 97 | 	}
 98 | 
 99 | 	for _, ts := range testSamples {
100 | 		if !testMetadataSample(ts) {
101 | 			t.Error("unexpected metadata")
102 | 		}
103 | 	}
104 | 
105 | }
106 | 
107 | func testMetadataSample(ts MetadataSample) bool {
108 | 	o := NewObject("test", "type", "subtype", []byte("content"))
109 | 
110 | 	o.SetMetadata(ts.key, ts.value, MetadataAttributes{PrimaryKey: ts.primaryKey})
111 | 	value, err := o.GetMetadata(ts.key)
112 | 
113 | 	ok := (err == nil && value == ts.expValue)
114 | 
115 | 	if ok == ts.ok {
116 | 		return true
117 | 	} else {
118 | 		return false
119 | 	}
120 | }
121 | 
122 | type PrimaryKeyHashSample struct {
123 | 	metadata       map[string]MetadataData
124 | 	primaryKeyHash KeyHash
125 | 	ok             bool
126 | }
127 | 
128 | func TestPrimaryKeyHash(t *testing.T) {
129 | 	testSamples := []PrimaryKeyHashSample{
130 | 		{
131 | 			metadata:       map[string]MetadataData{},
132 | 			primaryKeyHash: nil,
133 | 			ok:             true,
134 | 		},
135 | 		{
136 | 			metadata: map[string]MetadataData{
137 | 				"key_1": MetadataData{
138 | 					value: "value_1",
139 | 					attr:  MetadataAttributes{PrimaryKey: false},
140 | 				},
141 | 			},
142 | 			primaryKeyHash: nil,
143 | 			ok:             true,
144 | 		},
145 | 		{
146 | 			metadata: map[string]MetadataData{
147 | 				"key_1": MetadataData{
148 | 					value: "value_1",
149 | 					attr:  MetadataAttributes{PrimaryKey: true},
150 | 				},
151 | 			},
152 | 			primaryKeyHash: &[]byte{0xe5, 0xd7, 0x3d, 0xe2, 0x2b, 0xa, 0xb1, 0x2, 0x64, 0xbb, 0x9, 0x77, 0xae, 0xea, 0x7, 0x4f, 0xd7, 0x14, 0x5d, 0xeb, 0x93, 0x84, 0xdc, 0xe, 0xb0, 0x91, 0x37, 0x29, 0x10, 0x56, 0x3, 0x45},
153 | 			ok:             true,
154 | 		},
155 | 		{
156 | 			metadata: map[string]MetadataData{
157 | 				"key_1": MetadataData{
158 | 					value: "value_1",
159 | 					attr:  MetadataAttributes{PrimaryKey: true},
160 | 				},
161 | 				"key_2": MetadataData{
162 | 					value: "value_2",
163 | 					attr:  MetadataAttributes{PrimaryKey: false},
164 | 				},
165 | 				"key_3": MetadataData{
166 | 					value: "value_3",
167 | 					attr:  MetadataAttributes{PrimaryKey: true},
168 | 				},
169 | 			},
170 | 			primaryKeyHash: &[]byte{0xd1, 0x9c, 0xb2, 0x5e, 0x12, 0x64, 0x82, 0xfb, 0xc8, 0x96, 0xc4, 0x9a, 0x65, 0xd0, 0x4e, 0xa2, 0xb8, 0x9, 0x67, 0x41, 0xdc, 0xa1, 0xc4, 0x82, 0x3d, 0x6f, 0xa1, 0x1, 0x1d, 0xaa, 0x45, 0xbc},
171 | 			ok:             true,
172 | 		},
173 | 		{
174 | 			metadata: map[string]MetadataData{
175 | 				"key_3": MetadataData{
176 | 					value: "value_3",
177 | 					attr:  MetadataAttributes{PrimaryKey: true},
178 | 				},
179 | 				"key_2": MetadataData{
180 | 					value: "value_2",
181 | 					attr:  MetadataAttributes{PrimaryKey: false},
182 | 				},
183 | 				"key_1": MetadataData{
184 | 					value: "value_1",
185 | 					attr:  MetadataAttributes{PrimaryKey: true},
186 | 				},
187 | 			},
188 | 			primaryKeyHash: &[]byte{0xd1, 0x9c, 0xb2, 0x5e, 0x12, 0x64, 0x82, 0xfb, 0xc8, 0x96, 0xc4, 0x9a, 0x65, 0xd0, 0x4e, 0xa2, 0xb8, 0x9, 0x67, 0x41, 0xdc, 0xa1, 0xc4, 0x82, 0x3d, 0x6f, 0xa1, 0x1, 0x1d, 0xaa, 0x45, 0xbc},
189 | 			ok:             true,
190 | 		},
191 | 	}
192 | 
193 | 	for _, ts := range testSamples {
194 | 		if !testPrimaryKeyHashSample(ts) {
195 | 			t.Error("unexpected primary key hash")
196 | 		}
197 | 	}
198 | 
199 | }
200 | 
201 | func testPrimaryKeyHashSample(ts PrimaryKeyHashSample) bool {
202 | 	o := NewObject("test", "type", "subtype", []byte("content"))
203 | 
204 | 	for k, v := range ts.metadata {
205 | 		o.SetMetadata(k, v.value, v.attr)
206 | 	}
207 | 
208 | 	h := o.GetPrimaryKeyHash()
209 | 
210 | 	var ok bool
211 | 	if h == nil {
212 | 		if ts.primaryKeyHash == nil {
213 | 			ok = true
214 | 		} else {
215 | 			ok = false
216 | 		}
217 | 	} else {
218 | 		ok = (bytes.Equal(*ts.primaryKeyHash, *h))
219 | 	}
220 | 
221 | 	if ok == ts.ok {
222 | 		return true
223 | 	} else {
224 | 		return false
225 | 	}
226 | }
227 | 


--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
  1 | |Build Status| |Documentation Status|
  2 | 
  3 | =======
  4 | seekret
  5 | =======
  6 | 
  7 | Go library and command line to seek for secrets on various sources.
  8 | 
  9 | 
 10 | ************
 11 | Command Line
 12 | ************
 13 | 
 14 | Description
 15 | ===========
 16 | 
 17 | ``seekret`` inspect different sources (files into a directory or git 
 18 | repositories) to seek for secrets. It can be used to prevent that secrets are
 19 | published in exposed locations.
 20 | 
 21 | 
 22 | Installing seekret
 23 | ==================
 24 | 
 25 | ``seekret`` can be directly installed by using go get.
 26 | 
 27 | ::
 28 | 
 29 |     go get github.com/apuigsech/seekret/cmd/seekret
 30 | 
 31 | 
 32 | The requirements for a success installation are:
 33 | 
 34 |  * pkg-config
 35 |  * golang >= 1.6
 36 |  * libgit2 >= 2.23
 37 | 
 38 | 
 39 | Usage
 40 | =====
 41 | 
 42 | General Options
 43 | ~~~~~~~~~~~~~~~
 44 | 
 45 | ::
 46 | 
 47 |     NAME:
 48 |        seekret - seek for secrets on various sources.   
 49 | 
 50 |     USAGE:
 51 |        seekret [global options] command [command options] [arguments...]
 52 |        
 53 |     VERSION:
 54 |        0.0.1
 55 |        
 56 |     AUTHOR(S):
 57 |        Albert Puigsech Galicia <albert@puigsech.com> 
 58 |        
 59 |     COMMANDS:
 60 |        seek:
 61 |          git    seek for seecrets on a git repository.
 62 |          dir    seek for seecrets on a directory.   
 63 | 
 64 |     GLOBAL OPTIONS:
 65 |        --exception FILE, -x FILE    load exceptions from FILE.
 66 |        --rules PATH         PATH with rules. [$SEEKRET_RULES_PATH] 
 67 |        --format value, -f value specify the output format. (default: "human")
 68 |        --known FILE, -k FILE  load known secrets from FILE.
 69 |        --workers value, -w value  number of workers used for the inspection (default: 4)
 70 |        --help, -h           show help
 71 |        --version, -v        print the version
 72 | 
 73 | 
 74 | ``-x, --exception``
 75 | 
 76 | ``--rules``
 77 | 
 78 | ``-f, --format``
 79 | 
 80 | ``-k, --known``
 81 | 
 82 | ``-w, --workers``
 83 | 
 84 | 
 85 | Options for Git
 86 | ~~~~~~~~~~~~~~~
 87 | 
 88 | ::
 89 | 
 90 |     NAME:
 91 |        seekret git - seek for seecrets on a git repository.
 92 | 
 93 |     USAGE:
 94 |        seekret git [command options] [arguments...]
 95 | 
 96 |     CATEGORY:
 97 |        seek
 98 | 
 99 |     OPTIONS:
100 |        --count value, -c value  (default: 0)
101 |    
102 | 
103 | ``-c, --count``
104 | 
105 | 
106 | Options for Dir
107 | ~~~~~~~~~~~~~~~
108 | 
109 | ::
110 | 
111 |     NAME:
112 |        seekret dir - seek for seecrets on a directory.  
113 | 
114 |     USAGE:
115 |        seekret dir [command options] [arguments...] 
116 | 
117 |     CATEGORY:
118 |        seek 
119 | 
120 |     OPTIONS:
121 |        --recursive, -r  
122 |        --hidden
123 | 
124 | 
125 | ``-r, --recursive``
126 | 
127 | ``-h, --hidden``
128 | 
129 | 
130 | 
131 | Examples
132 | ========
133 | 
134 | Scan all files from all commits in a local repo::
135 | 
136 |     seekret git /path/to/repo
137 | 
138 | Scan all files from all commits in a remote repo::
139 | 
140 |     seekret git http://github.com/apuigsech/seekret-exposed
141 | 
142 | Scan all files from the last commit in a local repo::
143 | 
144 |     seekret git --count 1 /path/to/repo
145 | 
146 | Scan all files (including hidden) in a local folder::
147 | 
148 |     seekret dir --recursive --hidden /path/to/dir
149 | 
150 | 
151 | Hands-On
152 | ========
153 | 
154 | The repository seekret-secrets is prepare to test seekret, and can be used to
155 | perform the following hands-on examples:
156 | 
157 | 1. Inspect remote git repository::
158 | 
159 |     seekret --rules $GOPATH/src/github.com/apuigsech/seekret/rules/ git https://github.com/apuigsech/seekret-secrets.git 
160 | 
161 | 2. Inspect local got repository::
162 | 
163 |     git clone https://github.com/apuigsech/seekret-secrets.git /tmp/seekret-secrets
164 |     seekret --rules $GOPATH/src/github.com/apuigsech/seekret/rules/ git /tmp/seekret-secrets
165 | 
166 | 3. Inspect only the last 2 commits::
167 | 
168 |     seekret --rules $GOPATH/src/github.com/apuigsech/seekret/rules/ git -c 2 /tmp/seekret-secrets
169 | 
170 | 4. Inspect with exceptions::
171 | 
172 |     seekret --rules $GOPATH/src/github.com/apuigsech/seekret/rules/ -x /tmp/seekret-secrets/.exception_1 git /tmp/seekret-secrets
173 | 
174 | 
175 | *******
176 | Library
177 | *******
178 | 
179 | Importing seekret Library
180 | =========================
181 | 
182 | ::
183 | 
184 |     import seekret "github.com/apuigsech/seekret/lib"
185 | 
186 | 
187 | Init Seekret context
188 | ====================
189 | 
190 | ::
191 | 
192 |     s := seekret.NewSeekret()
193 | 
194 | 
195 | Loading Rules
196 | =============
197 | 
198 | ::
199 | 
200 |     s.LoadRulesFromPath("/path/to/main/rues:/path/to/other/rules:/path/to/more/rules")
201 | 
202 | ::
203 | 
204 |     s.LoadRulesFromDir("/path/to/rules")
205 | 
206 | 
207 | ::
208 | 
209 |     s.LoadRulesFromFile("/path/to/file.rule")
210 | 
211 | 
212 | Loading Objects
213 | ===============
214 | 
215 | ::
216 | 
217 |     opts := map[string]interface{} {
218 |       "hidden": true,
219 |       "recursive": false,
220 |     }
221 |     s.LoadObjects("dir", "/path/to/inspect", opts)
222 | 
223 | 
224 | ::
225 | 
226 |     opts := map[string]interface{} {
227 |       "count": 10,
228 |     }
229 |     s.LoadObjects("dir", "/repo/to/inspect", opts)
230 | 
231 | 
232 | Loading Exceptions
233 | ==================
234 | 
235 | ::
236 | 
237 |     s.LoadExceptionsFromFile("/path/to/exceptions/file")
238 | 
239 | 
240 | 
241 | Inspect
242 | =======
243 | 
244 | ::
245 | 
246 |     s.Inspect(5)
247 | 
248 | 
249 | 
250 | Get Inspect Results
251 | ===================
252 | 
253 | ::
254 | 
255 |     secretsList := s.ListSecrets()
256 | 
257 | 
258 | 
259 | *****
260 | Rules
261 | *****
262 | 
263 | Secret identification is performed by using a set of rules specified on the
264 | rules files. Those files, with '.rule' extension are defined by using YAML 
265 | following this format: 
266 | 
267 | ::
268 | 
269 |     rulename:
270 |       match: [regexp]
271 |       unmatch:
272 |         - [regexp]
273 |         - [regexp]
274 |         - ...
275 | 
276 | For the contents of a file is considered a secret, it must comply with the
277 | 'match' regexp and not comply ANY of the 'unmatch' reg rule and comply match 
278 | ANY of the unmatch.
279 | 
280 | 
281 | **********
282 | Exceptions
283 | **********
284 | 
285 | Exceptions determine conditions under which content should not be considered
286 | a secret. The exceptions are specified by using a YAML file that follows this
287 | format:
288 | 
289 | ::
290 |     
291 |       ...
292 |     -
293 |       rule: [rulename]
294 |       object: [regexp]
295 |       line: [linenumber]
296 |       content: [regexp]
297 |     -
298 |       ...
299 | 
300 | 
301 | The conditions are optional, so it is not necessary to specify them all, but
302 | for a content deemed exception must meet all the specified conditions.
303 | 
304 | The meaning of the various conditions explained:
305 | 
306 | ``rule``
307 | Contains the name of the rule.
308 | 
309 | ``object``
310 | Contains a regexp that should match the object name (usually the filename).
311 | 
312 | ``line``
313 | Contains the line number into the object.
314 | 
315 | ``content``
316 | Contains a regexp that should match the content.
317 | 
318 | 
319 | 
320 | .. |Build Status| image:: https://travis-ci.org/apuigsech/seekret.svg
321 |    :target: https://travis-ci.org/apuigsech/seekret
322 |    :width: 88px
323 |    :height: 20px
324 | .. |Documentation Status| image:: https://godoc.org/github.com/apuigsech/seekret?status.svg
325 |    :target: https://godoc.org/github.com/apuigsech/seekret
326 |    :width: 88px
327 |    :height: 20px
328 | 


--------------------------------------------------------------------------------
/seekret.go:
--------------------------------------------------------------------------------
  1 | // Copyright 2016 - Authors included on AUTHORS file.
  2 | //
  3 | // Use of this source code is governed by a Apache License
  4 | // that can be found in the LICENSE file.
  5 | 
  6 | package seekret
  7 | 
  8 | import (
  9 | 	"fmt"
 10 | 	"github.com/apuigsech/seekret/models"
 11 | 	"gopkg.in/yaml.v2"
 12 | 	"io/ioutil"
 13 | 	"os"
 14 | 	"path/filepath"
 15 | 	"regexp"
 16 | 	"strings"
 17 | )
 18 | 
 19 | // Seekret contains a seekret context and exposes the API to manipulate it.
 20 | type Seekret struct {
 21 | 	// List of rules loaded into the context.
 22 | 	ruleList []models.Rule
 23 | 
 24 | 	// List of objects loaded into the context.
 25 | 	objectList []models.Object
 26 | 
 27 | 	// List of exceptions loaded into the context.
 28 | 	exceptionList []models.Exception
 29 | 
 30 | 	// List of secrets detected after the inspection.
 31 | 	secretList []models.Secret
 32 | }
 33 | 
 34 | // NewSeekret returns a new seekret context.
 35 | func NewSeekret() *Seekret {
 36 | 	s := &Seekret{}
 37 | 	return s
 38 | }
 39 | 
 40 | // AddRule adds a new rule into the context.
 41 | func (s *Seekret) AddRule(rule models.Rule, enabled bool) {
 42 | 	if enabled {
 43 | 		rule.Enable()
 44 | 	}
 45 | 	s.ruleList = append(s.ruleList, rule)
 46 | }
 47 | 
 48 | type ruleYaml struct {
 49 | 	ObjectMatch string
 50 | 	Match       string
 51 | 	Unmatch     []string
 52 | }
 53 | 
 54 | // LoadRulesFromFile loads rules from a YAML file.
 55 | func (s *Seekret) LoadRulesFromFile(file string, defaulEnabled bool) error {
 56 | 	var ruleYamlMap map[string]ruleYaml
 57 | 
 58 | 	if file == "" {
 59 | 		return nil
 60 | 	}
 61 | 
 62 | 	filename, _ := filepath.Abs(file)
 63 | 
 64 | 	ruleBase := filepath.Base(filename)
 65 | 	if filepath.Ext(ruleBase) == ".rule" {
 66 | 		ruleBase = ruleBase[0 : len(ruleBase)-5]
 67 | 	}
 68 | 
 69 | 	yamlData, err := ioutil.ReadFile(filename)
 70 | 	if err != nil {
 71 | 		return err
 72 | 	}
 73 | 
 74 | 	err = yaml.Unmarshal(yamlData, &ruleYamlMap)
 75 | 	if err != nil {
 76 | 		return err
 77 | 	}
 78 | 
 79 | 	for k, v := range ruleYamlMap {
 80 | 		rule, err := models.NewRule(ruleBase+"."+k, v.Match)
 81 | 		if err != nil {
 82 | 			return err
 83 | 		}
 84 | 
 85 | 		for _, e := range v.Unmatch {
 86 | 			rule.AddUnmatch(e)
 87 | 		}
 88 | 		s.AddRule(*rule, defaulEnabled)
 89 | 	}
 90 | 
 91 | 	return nil
 92 | }
 93 | 
 94 | // LoadRulesFromFile loads rules from all YAML files inside a directory.
 95 | func (s *Seekret) LoadRulesFromDir(dir string, defaulEnabled bool) error {
 96 | 	fi, err := os.Stat(dir)
 97 | 	if err != nil {
 98 | 		return err
 99 | 	}
100 | 
101 | 	if !fi.IsDir() {
102 | 		err := fmt.Errorf("%s is not a directory", dir)
103 | 		return err
104 | 	}
105 | 
106 | 	fileList, err := filepath.Glob(dir + "/*")
107 | 	if err != nil {
108 | 		return err
109 | 	}
110 | 	for _, file := range fileList {
111 | 		if strings.HasSuffix(file, ".rule") {
112 | 			err := s.LoadRulesFromFile(file, defaulEnabled)
113 | 			if err != nil {
114 | 				return err
115 | 			}
116 | 		}
117 | 	}
118 | 	return nil
119 | }
120 | 
121 | // LoadRulesFromFile loads rules from all YAML files inside different
122 | // directories separated by ':'.
123 | func (s *Seekret) LoadRulesFromPath(path string, defaulEnabled bool) error {
124 | 	if path == "" {
125 | 		path = DefaultRulesPath()
126 | 	}
127 | 	dirList := strings.Split(path, ":")
128 | 	for _, dir := range dirList {
129 | 		err := s.LoadRulesFromDir(dir, defaulEnabled)
130 | 		if err != nil {
131 | 			return err
132 | 		}
133 | 	}
134 | 	return nil
135 | }
136 | 
137 | const defaultRulesDir = "$GOPATH/src/github.com/apuigsech/seekret/rules"
138 | 
139 | // DefaultRulesPath return the default PATH that contains rules.
140 | func DefaultRulesPath() string {
141 | 	rulesPath := os.Getenv("SEEKRET_RULES_PATH")
142 | 	if rulesPath == "" {
143 | 		rulesPath = os.ExpandEnv(defaultRulesDir)
144 | 	}
145 | 	return rulesPath
146 | }
147 | 
148 | // ListRules return an array with all loaded rules.
149 | func (s *Seekret) ListRules() []models.Rule {
150 | 	return s.ruleList
151 | }
152 | 
153 | // EnableRule enables specific rule.
154 | func (s *Seekret) EnableRule(name string) error {
155 | 	return setRuleEnabled(s.ruleList, name, true)
156 | }
157 | 
158 | // DisableRule disables specific rule.
159 | func (s *Seekret) DisableRule(name string) error {
160 | 	return setRuleEnabled(s.ruleList, name, false)
161 | }
162 | 
163 | // EnableRule enables rules that match with a regular expression.
164 | func (s *Seekret) EnableRuleByRegexp(name string) int {
165 | 	return setRuleEnabledByRegexp(s.ruleList, name, true)
166 | }
167 | 
168 | // DisableRule disables rules that match with a regular expression.
169 | func (s *Seekret) DisableRuleByRegexp(name string) int {
170 | 	return setRuleEnabledByRegexp(s.ruleList, name, false)
171 | }
172 | 
173 | 
174 | func setRuleEnabled(ruleList []models.Rule, name string, enabled bool) error {
175 | 	found := false
176 | 	for i,r := range ruleList {
177 | 		if r.Name == name {
178 | 			found = true
179 | 			ruleList[i].Enabled = enabled
180 | 		}
181 | 	}
182 | 	if !found {
183 | 		err := fmt.Errorf("%s rule not found", name)
184 | 		return err
185 | 	}
186 | 
187 | 	return nil
188 | }
189 | 
190 | func setRuleEnabledByRegexp(ruleList []models.Rule, nameRegexp string, enabled bool) int {
191 | 	count := 0
192 | 	nameMatch,err := regexp.Compile("(?i)" + nameRegexp)
193 | 	if err != nil {
194 | 		return 0
195 | 	}
196 | 	for i,r := range ruleList {
197 | 		if nameMatch.Match([]byte(r.Name)) {
198 | 			count = count + 1
199 | 			ruleList[i].Enabled = enabled
200 | 		}
201 | 	}
202 | 	return count
203 | }
204 | 
205 | // LoadObjects loads objects form an specific source. It can load objects from
206 | // different source types, that are implemented following the SourceType
207 | // interface.
208 | func (s *Seekret) LoadObjects(st SourceType, source string, opt LoadOptions) error {
209 | 	objectList, err := st.LoadObjects(source, opt)
210 | 	if err != nil {
211 | 		return err
212 | 	}
213 | 	s.objectList = append(s.objectList, objectList...)
214 | 	return nil
215 | }
216 | 
217 | // GroupObjectsByMetadata returns a map with all objects grouped by specific
218 | // metadata key.
219 | func (s *Seekret) GroupObjectsByMetadata(k string) map[string][]models.Object {
220 | 	return models.GroupObjectsByMetadata(s.objectList, k)
221 | }
222 | 
223 | // GroupObjectsByPrimaryKeyHash returns a map with all objects grouped by
224 | // the primary key hash, that is calculated from all metadata keys with the
225 | // primary attribute.
226 | // All returned objects could have the same content, even if are not the same.
227 | func (s *Seekret) GroupObjectsByPrimaryKeyHash() map[string][]models.Object {
228 | 	return models.GroupObjectsByPrimaryKeyHash(s.objectList)
229 | }
230 | 
231 | type exceptionYaml struct {
232 | 	Rule    *string
233 | 	Object  *string
234 | 	Line    *int
235 | 	Content *string
236 | }
237 | 
238 | // AddException adds a new exception into the context.
239 | func (s *Seekret) AddException(exception models.Exception) {
240 | 	s.exceptionList = append(s.exceptionList, exception)
241 | }
242 | 
243 | // LoadExceptionsFromFile loads exceptions from a YAML file.
244 | func (s *Seekret) LoadExceptionsFromFile(file string) error {
245 | 	var exceptionYamlList []exceptionYaml
246 | 
247 | 	if file == "" {
248 | 		return nil
249 | 	}
250 | 
251 | 	filename, _ := filepath.Abs(file)
252 | 	yamlData, err := ioutil.ReadFile(filename)
253 | 	if err != nil {
254 | 		return err
255 | 	}
256 | 
257 | 	err = yaml.Unmarshal(yamlData, &exceptionYamlList)
258 | 	if err != nil {
259 | 		return err
260 | 	}
261 | 
262 | 	for _, v := range exceptionYamlList {
263 | 		x := models.NewException()
264 | 
265 | 		if v.Rule != nil {
266 | 			err := x.SetRule(*v.Rule)
267 | 			if err != nil {
268 | 				return err
269 | 			}
270 | 		}
271 | 
272 | 		if v.Object != nil {
273 | 			err := x.SetObject(*v.Object)
274 | 			if err != nil {
275 | 				return err
276 | 			}
277 | 		}
278 | 
279 | 		if v.Line != nil {
280 | 			err := x.SetNline(*v.Line)
281 | 			if err != nil {
282 | 				return err
283 | 			}
284 | 		}
285 | 
286 | 		if v.Content != nil {
287 | 			err := x.SetContent(*v.Content)
288 | 			if err != nil {
289 | 				return err
290 | 			}
291 | 		}
292 | 
293 | 		s.AddException(*x)
294 | 	}
295 | 
296 | 	return nil
297 | }
298 | 
299 | // ListSecrets return an array with all found secrets after the inspection.
300 | func (s *Seekret) ListSecrets() []models.Secret {
301 | 	return s.secretList
302 | }
303 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                               Apache License
  2 |                         Version 2.0, January 2004
  3 |                      http://www.apache.org/licenses/
  4 | 
  5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 | 1. Definitions.
  8 | 
  9 |    "License" shall mean the terms and conditions for use, reproduction,
 10 |    and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |    "Licensor" shall mean the copyright owner or entity authorized by
 13 |    the copyright owner that is granting the License.
 14 | 
 15 |    "Legal Entity" shall mean the union of the acting entity and all
 16 |    other entities that control, are controlled by, or are under common
 17 |    control with that entity. For the purposes of this definition,
 18 |    "control" means (i) the power, direct or indirect, to cause the
 19 |    direction or management of such entity, whether by contract or
 20 |    otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |    outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |    "You" (or "Your") shall mean an individual or Legal Entity
 24 |    exercising permissions granted by this License.
 25 | 
 26 |    "Source" form shall mean the preferred form for making modifications,
 27 |    including but not limited to software source code, documentation
 28 |    source, and configuration files.
 29 | 
 30 |    "Object" form shall mean any form resulting from mechanical
 31 |    transformation or translation of a Source form, including but
 32 |    not limited to compiled object code, generated documentation,
 33 |    and conversions to other media types.
 34 | 
 35 |    "Work" shall mean the work of authorship, whether in Source or
 36 |    Object form, made available under the License, as indicated by a
 37 |    copyright notice that is included in or attached to the work
 38 |    (an example is provided in the Appendix below).
 39 | 
 40 |    "Derivative Works" shall mean any work, whether in Source or Object
 41 |    form, that is based on (or derived from) the Work and for which the
 42 |    editorial revisions, annotations, elaborations, or other modifications
 43 |    represent, as a whole, an original work of authorship. For the purposes
 44 |    of this License, Derivative Works shall not include works that remain
 45 |    separable from, or merely link (or bind by name) to the interfaces of,
 46 |    the Work and Derivative Works thereof.
 47 | 
 48 |    "Contribution" shall mean any work of authorship, including
 49 |    the original version of the Work and any modifications or additions
 50 |    to that Work or Derivative Works thereof, that is intentionally
 51 |    submitted to Licensor for inclusion in the Work by the copyright owner
 52 |    or by an individual or Legal Entity authorized to submit on behalf of
 53 |    the copyright owner. For the purposes of this definition, "submitted"
 54 |    means any form of electronic, verbal, or written communication sent
 55 |    to the Licensor or its representatives, including but not limited to
 56 |    communication on electronic mailing lists, source code control systems,
 57 |    and issue tracking systems that are managed by, or on behalf of, the
 58 |    Licensor for the purpose of discussing and improving the Work, but
 59 |    excluding communication that is conspicuously marked or otherwise
 60 |    designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |    "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |    on behalf of whom a Contribution has been received by Licensor and
 64 |    subsequently incorporated within the Work.
 65 | 
 66 | 2. Grant of Copyright License. Subject to the terms and conditions of
 67 |    this License, each Contributor hereby grants to You a perpetual,
 68 |    worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |    copyright license to reproduce, prepare Derivative Works of,
 70 |    publicly display, publicly perform, sublicense, and distribute the
 71 |    Work and such Derivative Works in Source or Object form.
 72 | 
 73 | 3. Grant of Patent License. Subject to the terms and conditions of
 74 |    this License, each Contributor hereby grants to You a perpetual,
 75 |    worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |    (except as stated in this section) patent license to make, have made,
 77 |    use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |    where such license applies only to those patent claims licensable
 79 |    by such Contributor that are necessarily infringed by their
 80 |    Contribution(s) alone or by combination of their Contribution(s)
 81 |    with the Work to which such Contribution(s) was submitted. If You
 82 |    institute patent litigation against any entity (including a
 83 |    cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |    or a Contribution incorporated within the Work constitutes direct
 85 |    or contributory patent infringement, then any patent licenses
 86 |    granted to You under this License for that Work shall terminate
 87 |    as of the date such litigation is filed.
 88 | 
 89 | 4. Redistribution. You may reproduce and distribute copies of the
 90 |    Work or Derivative Works thereof in any medium, with or without
 91 |    modifications, and in Source or Object form, provided that You
 92 |    meet the following conditions:
 93 | 
 94 |    (a) You must give any other recipients of the Work or
 95 |        Derivative Works a copy of this License; and
 96 | 
 97 |    (b) You must cause any modified files to carry prominent notices
 98 |        stating that You changed the files; and
 99 | 
100 |    (c) You must retain, in the Source form of any Derivative Works
101 |        that You distribute, all copyright, patent, trademark, and
102 |        attribution notices from the Source form of the Work,
103 |        excluding those notices that do not pertain to any part of
104 |        the Derivative Works; and
105 | 
106 |    (d) If the Work includes a "NOTICE" text file as part of its
107 |        distribution, then any Derivative Works that You distribute must
108 |        include a readable copy of the attribution notices contained
109 |        within such NOTICE file, excluding those notices that do not
110 |        pertain to any part of the Derivative Works, in at least one
111 |        of the following places: within a NOTICE text file distributed
112 |        as part of the Derivative Works; within the Source form or
113 |        documentation, if provided along with the Derivative Works; or,
114 |        within a display generated by the Derivative Works, if and
115 |        wherever such third-party notices normally appear. The contents
116 |        of the NOTICE file are for informational purposes only and
117 |        do not modify the License. You may add Your own attribution
118 |        notices within Derivative Works that You distribute, alongside
119 |        or as an addendum to the NOTICE text from the Work, provided
120 |        that such additional attribution notices cannot be construed
121 |        as modifying the License.
122 | 
123 |    You may add Your own copyright statement to Your modifications and
124 |    may provide additional or different license terms and conditions
125 |    for use, reproduction, or distribution of Your modifications, or
126 |    for any such Derivative Works as a whole, provided Your use,
127 |    reproduction, and distribution of the Work otherwise complies with
128 |    the conditions stated in this License.
129 | 
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 |    any Contribution intentionally submitted for inclusion in the Work
132 |    by You to the Licensor shall be under the terms and conditions of
133 |    this License, without any additional terms or conditions.
134 |    Notwithstanding the above, nothing herein shall supersede or modify
135 |    the terms of any separate license agreement you may have executed
136 |    with Licensor regarding such Contributions.
137 | 
138 | 6. Trademarks. This License does not grant permission to use the trade
139 |    names, trademarks, service marks, or product names of the Licensor,
140 |    except as required for reasonable and customary use in describing the
141 |    origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 |    agreed to in writing, Licensor provides the Work (and each
145 |    Contributor provides its Contributions) on an "AS IS" BASIS,
146 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |    implied, including, without limitation, any warranties or conditions
148 |    of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |    PARTICULAR PURPOSE. You are solely responsible for determining the
150 |    appropriateness of using or redistributing the Work and assume any
151 |    risks associated with Your exercise of permissions under this License.
152 | 
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 |    whether in tort (including negligence), contract, or otherwise,
155 |    unless required by applicable law (such as deliberate and grossly
156 |    negligent acts) or agreed to in writing, shall any Contributor be
157 |    liable to You for damages, including any direct, indirect, special,
158 |    incidental, or consequential damages of any character arising as a
159 |    result of this License or out of the use or inability to use the
160 |    Work (including but not limited to damages for loss of goodwill,
161 |    work stoppage, computer failure or malfunction, or any and all
162 |    other commercial damages or losses), even if such Contributor
163 |    has been advised of the possibility of such damages.
164 | 
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 |    the Work or Derivative Works thereof, You may choose to offer,
167 |    and charge a fee for, acceptance of support, warranty, indemnity,
168 |    or other liability obligations and/or rights consistent with this
169 |    License. However, in accepting such obligations, You may act only
170 |    on Your own behalf and on Your sole responsibility, not on behalf
171 |    of any other Contributor, and only if You agree to indemnify,
172 |    defend, and hold each Contributor harmless for any liability
173 |    incurred by, or claims asserted against, such Contributor by reason
174 |    of your accepting any such warranty or additional liability.
175 | 
176 | END OF TERMS AND CONDITIONS
177 | 
178 | APPENDIX: How to apply the Apache License to your work.
179 | 
180 |    To apply the Apache License to your work, attach the following
181 |    boilerplate notice, with the fields enclosed by brackets "[]"
182 |    replaced with your own identifying information. (Don't include
183 |    the brackets!)  The text should be enclosed in the appropriate
184 |    comment syntax for the file format. We also recommend that a
185 |    file or class name and description of purpose be included on the
186 |    same "printed page" as the copyright notice for easier
187 |    identification within third-party archives.
188 | 
189 | Copyright 2016 - Albert Puigsech Galicia <albert@puigsech.com> 
190 | 
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 | 
195 |     http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 | 


--------------------------------------------------------------------------------