├── hercules ├── hercules_test.go └── repo.go ├── github ├── github_test.go └── github.go ├── scanmanager ├── scanmanager_test.go ├── dryrun.go ├── github.go ├── path.go └── scanmanager.go ├── pkg ├── filters │ ├── skip.yml │ ├── c_sharp.yml │ ├── js.yml │ └── skip-deps.yml ├── patterns │ ├── strong │ │ ├── private-keys.yml │ │ ├── security-scan.yml │ │ └── github-dorks.yml │ └── weak │ │ └── mypatterns.yml ├── postinst └── hungryfox.service ├── .gitignore ├── api └── api.go ├── state ├── filestate │ ├── format.go │ └── filestate.go └── dbstate │ └── state.go ├── senders ├── file │ └── file.go ├── webhook │ └── sender.go └── email │ ├── server.go │ ├── sender.go │ └── template.go ├── .github └── FUNDING.yml ├── .travis.yml ├── LICENSE ├── helpers ├── helpers_test.go └── helpers.go ├── repolist ├── repolist.go └── repolist_test.go ├── hungryfox.go ├── searcher ├── searcher_test.go └── searcher.go ├── Makefile ├── router └── router.go ├── go.mod ├── config └── config.go ├── README.md └── cmd └── hungryfox └── main.go /hercules/hercules_test.go: -------------------------------------------------------------------------------- 1 | package repo -------------------------------------------------------------------------------- /github/github_test.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | -------------------------------------------------------------------------------- /scanmanager/scanmanager_test.go: -------------------------------------------------------------------------------- 1 | package scanmanager 2 | -------------------------------------------------------------------------------- /pkg/filters/skip.yml: -------------------------------------------------------------------------------- 1 | - name: these files never has secrets 2 | file: \.(css|html)$ 3 | -------------------------------------------------------------------------------- /pkg/filters/c_sharp.yml: -------------------------------------------------------------------------------- 1 | - file: \.config$ 2 | content: publicKeyToken=.+ culture="neutral" 3 | 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .vscode 3 | *.exe 4 | *.exe~ 5 | build 6 | coverage.txt 7 | 8 | state.yml 9 | leaks.json 10 | go.sum 11 | 12 | -------------------------------------------------------------------------------- /pkg/filters/js.yml: -------------------------------------------------------------------------------- 1 | - file: \.min\.js$ 2 | - file: \.js\.map$ 3 | - file: dll.vendor.js$ 4 | - file: bluebird.min.js$ 5 | - file: /package-lock.json$ 6 | -------------------------------------------------------------------------------- /pkg/filters/skip-deps.yml: -------------------------------------------------------------------------------- 1 | - file: Godeps/_workspace/src/ 2 | 3 | - file: node_modules/ 4 | 5 | - file: vendor/ 6 | 7 | - file: \.config$ 8 | content: publicKeyToken=.+ culture="neutral" 9 | 10 | - file: /package-lock.json$ 11 | 12 | - content: '"(get|mini)pass": "\^?[\d.]+"' 13 | -------------------------------------------------------------------------------- /pkg/patterns/strong/private-keys.yml: -------------------------------------------------------------------------------- 1 | - name: Private Key 2 | content: (?i)^[\s-]*BEGIN (PGP |RSA |DSA |ES |OPENSSH )?PRIVATE KEY( BLOCK)?[\s-]*$ 3 | 4 | - name: Putty Private Key 5 | content: (?i)^\s*PuTTY-User-Key-File\s*$ 6 | 7 | - name: certificate 8 | content: (?i)^[\s-]*BEGIN CERTIFICATE[\s-]* 9 | 10 | - name: gnupgp 11 | file: /\.gpg$ 12 | -------------------------------------------------------------------------------- /api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | type Server struct { 4 | 5 | } 6 | 7 | func (s *Server) Start() error { 8 | return nil 9 | } 10 | 11 | func AddFilter() error { 12 | return nil 13 | } 14 | 15 | func AddPattern() error { 16 | return nil 17 | } 18 | 19 | func MarkLeakAsResolved() error { 20 | return nil 21 | } 22 | 23 | func Status() { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /pkg/patterns/strong/security-scan.yml: -------------------------------------------------------------------------------- 1 | # https://github.com/onetwopunch/security-scan/blob/master/matchers.json 2 | - name: AWS Access Key ID 3 | content: AKIA[0-9A-Z]{16} 4 | 5 | - name: URL with Password 6 | content: (?i)\w+://[^:]*:[^@]+@[\w\d-.]+ 7 | 8 | - name: URL Basic auth 9 | content: (?i)https?://[\w\d_-]+?:[\d\w_.-]+@.+ 10 | 11 | - name: Google Access Token 12 | content: ya29.[\w\d_\\-]{68} 13 | 14 | - name: Google API 15 | content: AIzaSy[\w\d_\\-]{33} 16 | -------------------------------------------------------------------------------- /pkg/postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | 4 | # Initial installation: $1 == 1 5 | # Upgrade: $1 == 2, and configured to restart on upgrade 6 | 7 | if ! getent group "hungryfox" > /dev/null 2>&1 ; then 8 | groupadd -r "hungryfox" 9 | fi 10 | if ! getent passwd "hungryfox" > /dev/null 2>&1 ; then 11 | useradd -r -g hungryfox -d /usr/share/hungryfox -m -s /sbin/nologin -c "hungryfox" hungryfox 12 | fi 13 | 14 | if [ -x /bin/systemctl ] ; then 15 | /bin/systemctl daemon-reload 16 | fi 17 | -------------------------------------------------------------------------------- /state/filestate/format.go: -------------------------------------------------------------------------------- 1 | package filestate 2 | 3 | import "time" 4 | 5 | type RepoJSON struct { 6 | RepoURL string `yaml:"url"` 7 | CloneURL string `yaml:"clone_url"` 8 | RepoPath string `yaml:"repo_path"` 9 | DataPath string `yaml:"data_path"` 10 | Refs []string `yaml:"refs"` 11 | ScanStatus ScanJSON `yaml:"scan_status"` 12 | } 13 | 14 | type ScanJSON struct { 15 | StartTime time.Time `yaml:"start_time"` 16 | EndTime time.Time `yaml:"end_time"` 17 | Success bool `yaml:"success"` 18 | } 19 | -------------------------------------------------------------------------------- /pkg/patterns/weak/mypatterns.yml: -------------------------------------------------------------------------------- 1 | - content: secret[:] 2 | 3 | - name: password apikey or secret 4 | content: (?i)(pass|api[a-z0-9_-]{0,10}key|secret)[a-z0-9_.-]{0,10}['"]?\s{0,10}[=:,]\s{0,10}["'].+['"] 5 | 6 | - name: password apikey or secret in YAML 7 | file: (?i)\.(yaml|yml)$ 8 | content: (?i)(pass|api[\w\d_-]{0,10}key|secret)[\w\d_.-]{0,10}['"]?\s{0,10}:\s{0,10}["']?[^\s]+ 9 | 10 | - name: password apikey or secret in xml 11 | content: (?i)<(pass|api[a-z0-9_-]{0,10}key|secret)[a-z0-9_.-]{0,10}>.+<\/(pass|api[a-z0-9_-]{0,10}key|secret)[a-z0-9_.-]{0,10}> 12 | -------------------------------------------------------------------------------- /pkg/hungryfox.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=HungryFox is a Git Security Tool 3 | Wants=network-online.target 4 | After=network-online.target 5 | 6 | [Service] 7 | ExecStart=/usr/bin/hungryfox -config=/etc/hungryfox/config.yml 8 | User=hungryfox 9 | Group=hungryfox 10 | StandardError=journal 11 | Restart=always 12 | ExecReload=/bin/kill -HUP $MAINPID 13 | TimeoutStopSec=30s 14 | LimitMEMLOCK=infinity 15 | LimitNOFILE=4096 16 | Nice=19 17 | IOSchedulingClass=3 18 | IOSchedulingPriority=7 19 | StandardOutput=journal 20 | StandardError=journal 21 | 22 | [Install] 23 | WantedBy=multi-user.target 24 | -------------------------------------------------------------------------------- /pkg/patterns/strong/github-dorks.yml: -------------------------------------------------------------------------------- 1 | # https://github.com/techgaun/github-dorks/blob/master/github-dorks.txt 2 | - file: /\.npmrc$ 3 | content: _auth\s{0,10}=.+ 4 | 5 | - file: /\.dockercfg$ 6 | content: \"auth\":\s?\".+\" 7 | 8 | - name: slack token 9 | content: xox[pboa]-[\d\w-]+ 10 | 11 | - name: slack webhook url 12 | content: (?i)https?://hooks\.slack\.com/services/[\w\d/]+ 13 | 14 | - file: /\.s3cfg$ 15 | 16 | - file: /\.htpasswd$ 17 | 18 | - file: /\.git-credentials$ 19 | 20 | - file: /[._]netrc$ 21 | content: password 22 | 23 | - file: /filezilla.xml$ 24 | content: Pass 25 | 26 | - file: /\.pgpass$ 27 | -------------------------------------------------------------------------------- /senders/file/file.go: -------------------------------------------------------------------------------- 1 | package file 2 | 3 | import ( 4 | "encoding/json" 5 | "os" 6 | 7 | "github.com/AlexAkulov/hungryfox" 8 | ) 9 | 10 | type File struct { 11 | LeaksFile string 12 | } 13 | 14 | func (self *File) Start() error { 15 | return nil 16 | } 17 | 18 | func (self *File) Stop() error { 19 | return nil 20 | } 21 | 22 | func (self *File) Send(leak hungryfox.Leak) error { 23 | f, err := os.OpenFile(self.LeaksFile, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644) 24 | if err != nil { 25 | return err 26 | } 27 | defer f.Close() 28 | line, _ := json.Marshal(leak) 29 | f.Write(line) 30 | f.WriteString("\n") 31 | return nil 32 | } 33 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | # github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | # patreon: # Replace with a single Patreon username 5 | # open_collective: # Replace with a single Open Collective username 6 | ko_fi: alexakulov 7 | # tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | # community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | # liberapay: # Replace with a single Liberapay username 10 | # issuehunt: alexakulov 11 | # otechie: # Replace with a single Otechie username 12 | # custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | sudo: false 3 | go: 4 | - 1.11.4 5 | env: 6 | - GO111MODULE=on 7 | addons: 8 | apt: 9 | packages: 10 | rpm 11 | install: 12 | - gem install fpm 13 | - npm install -g snyk 14 | script: 15 | - make travis 16 | after_success: 17 | - bash <(curl -s https://codecov.io/bash) 18 | - snyk test 19 | deploy: 20 | - provider: packagecloud 21 | username: AlexAkulov 22 | repository: hungryfox-unstable 23 | token: $PACKAGECLOUD_TOKEN 24 | package_glob: build/hungryfox-*.rpm 25 | dist: el/7 26 | skip_cleanup: true 27 | - provider: packagecloud 28 | username: AlexAkulov 29 | repository: hungryfox-unstable 30 | token: $PACKAGECLOUD_TOKEN 31 | package_glob: build/hungryfox_*.deb 32 | dist: debian/buster 33 | skip_cleanup: true 34 | -------------------------------------------------------------------------------- /senders/webhook/sender.go: -------------------------------------------------------------------------------- 1 | package webhook 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "io/ioutil" 7 | "net/http" 8 | 9 | "github.com/AlexAkulov/hungryfox" 10 | ) 11 | 12 | type Sender struct { 13 | Method string 14 | URL string 15 | Headers map[string]string 16 | } 17 | 18 | func (self *Sender) Start() error { 19 | return nil 20 | } 21 | 22 | func (self *Sender) Stop() error { 23 | return nil 24 | } 25 | 26 | func (self *Sender) Send(leak hungryfox.Leak) error { 27 | line, _ := json.Marshal(leak) 28 | 29 | req, err := http.NewRequest(self.Method, self.URL, bytes.NewBuffer(line)) 30 | for k, v := range self.Headers { 31 | req.Header.Set(k, v) 32 | } 33 | req.Header.Set("Content-Type", "application/json") 34 | 35 | client := &http.Client{} 36 | resp, err := client.Do(req) 37 | if err != nil { 38 | return err 39 | } 40 | defer resp.Body.Close() 41 | _, err = ioutil.ReadAll(resp.Body) 42 | return err 43 | } 44 | -------------------------------------------------------------------------------- /state/dbstate/state.go: -------------------------------------------------------------------------------- 1 | package dbstate 2 | 3 | import ( 4 | "github.com/AlexAkulov/hungryfox" 5 | 6 | "upper.io/db.v3/lib/sqlbuilder" 7 | "upper.io/db.v3/ql" 8 | ) 9 | 10 | type StateManager struct { 11 | Location string 12 | db sqlbuilder.Database 13 | } 14 | 15 | type State struct { 16 | RepoID string `db:"repoid"` 17 | Refs string `db:"refs"` 18 | Status string `db:"status"` 19 | } 20 | 21 | func (s *StateManager) Start() error { 22 | settings := ql.ConnectionURL{Database: s.Location} 23 | var err error 24 | if s.db, err = ql.Open(settings); err != nil { 25 | return err 26 | } 27 | return nil 28 | } 29 | 30 | func (s *StateManager) Stop() error { 31 | return s.db.Close() 32 | } 33 | 34 | func (s StateManager) Save(r hungryfox.Repo) { 35 | } 36 | 37 | func (s StateManager) Load(id string) (hungryfox.RepoState, hungryfox.ScanStatus) { 38 | return hungryfox.RepoState{}, hungryfox.ScanStatus{} 39 | } 40 | 41 | func (s *StateManager) setup() error { 42 | // table := `CREATE TABLE state ( 43 | // repoid string, 44 | // refs string, 45 | // status string, 46 | // )` 47 | return nil 48 | } 49 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Alexander Akulov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /helpers/helpers_test.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | . "github.com/smartystreets/goconvey/convey" 8 | ) 9 | 10 | func TestParseDuration(t *testing.T) { 11 | Convey("1s", t, func() { 12 | result, err := ParseDuration("1s") 13 | So(err, ShouldBeNil) 14 | So(result, ShouldEqual, time.Duration(time.Second)) 15 | }) 16 | Convey("22m", t, func() { 17 | result, err := ParseDuration("22m") 18 | So(err, ShouldBeNil) 19 | So(result, ShouldEqual, time.Duration(time.Minute*22)) 20 | }) 21 | Convey("333h", t, func() { 22 | result, err := ParseDuration("333h") 23 | So(err, ShouldBeNil) 24 | So(result, ShouldEqual, time.Duration(time.Hour*333)) 25 | }) 26 | Convey("4444d", t, func() { 27 | result, err := ParseDuration("4444d") 28 | So(err, ShouldBeNil) 29 | So(result, ShouldEqual, time.Duration(time.Hour*24*4444)) 30 | }) 31 | Convey("5y", t, func() { 32 | result, err := ParseDuration("5y") 33 | So(err, ShouldBeNil) 34 | So(result, ShouldEqual, time.Duration(time.Hour*24*365*5)) 35 | }) 36 | Convey("3h2m1s", t, func() { 37 | result, err := ParseDuration("3h2m1s") 38 | So(err, ShouldBeNil) 39 | So(result, ShouldEqual, time.Duration(time.Hour*3+time.Minute*2+time.Second)) 40 | }) 41 | } 42 | -------------------------------------------------------------------------------- /repolist/repolist.go: -------------------------------------------------------------------------------- 1 | package repolist 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/AlexAkulov/hungryfox" 7 | ) 8 | 9 | type RepoList struct { 10 | list []hungryfox.Repo 11 | State hungryfox.IStateManager 12 | } 13 | 14 | func (l *RepoList) Clear() { 15 | l.list = nil 16 | } 17 | 18 | func (l *RepoList) addRepo(r hungryfox.Repo) { 19 | if l.list == nil { 20 | l.list = make([]hungryfox.Repo, 0) 21 | } 22 | for i := range l.list { 23 | if l.list[i].Location.URL == r.Location.URL { 24 | l.list[i] = r 25 | return 26 | } 27 | } 28 | l.list = append(l.list, r) 29 | } 30 | 31 | func (l *RepoList) AddRepo(r hungryfox.Repo) { 32 | r.State, r.Scan = l.State.Load(r.Location.URL) 33 | l.addRepo(r) 34 | } 35 | 36 | func (l *RepoList) UpdateRepo(r hungryfox.Repo) { 37 | l.addRepo(r) 38 | l.State.Save(r) 39 | } 40 | 41 | func (l *RepoList) GetRepoByIndex(i int) *hungryfox.Repo { 42 | if i > len(l.list)-1 || i < 0 { 43 | return nil 44 | } 45 | r := l.list[i] 46 | return &r 47 | } 48 | 49 | func (l *RepoList) GetRepoForScan() int { 50 | rID := -1 51 | lastScan := time.Now().UTC() 52 | for i, r := range l.list { 53 | if r.Scan.StartTime.IsZero() { 54 | return i 55 | } 56 | if r.Scan.EndTime.Before(lastScan) { 57 | rID = i 58 | lastScan = r.Scan.EndTime 59 | } 60 | } 61 | return rID 62 | } 63 | 64 | func (l *RepoList) GetTotalRepos() int { 65 | return len(l.list) 66 | } 67 | -------------------------------------------------------------------------------- /helpers/helpers.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "regexp" 5 | "strconv" 6 | "strings" 7 | "time" 8 | ) 9 | 10 | func PrettyDuration(d time.Duration) string { 11 | s := d.Round(time.Second).String() 12 | if strings.HasSuffix(s, "m0s") { 13 | s = s[:len(s)-2] 14 | } 15 | if strings.HasSuffix(s, "h0m") { 16 | s = s[:len(s)-2] 17 | } 18 | return s 19 | } 20 | 21 | func ParseDuration(str string) (time.Duration, error) { 22 | // durationRegex := regexp.MustCompile(`(?P\d+y(ears?)?)?(?P\d+m(onths?))?(?P\d+d(ays?)?)?(P?\d+h(ours?)?)?(?P\d+m(in(ute)?s?)?)?(?P\d+s(ec(ond)?s?)?)?`) 23 | durationRegex := regexp.MustCompile(`(?P\d+y)?(?P\d+d)?(?P\d+h)?(?P\d+m)?(?P\d+s)?`) 24 | matches := durationRegex.FindStringSubmatch(str) 25 | years := ParseInt64(matches[1]) 26 | days := ParseInt64(matches[2]) 27 | hours := ParseInt64(matches[3]) 28 | minutes := ParseInt64(matches[4]) 29 | seconds := ParseInt64(matches[5]) 30 | 31 | hour := int64(time.Hour) 32 | minute := int64(time.Minute) 33 | second := int64(time.Second) 34 | duration := time.Duration(years*24*365*hour + days*24*hour + hours*hour + minutes*minute + seconds*second) 35 | return duration, nil 36 | } 37 | 38 | func ParseInt64(value string) int64 { 39 | if len(value) == 0 { 40 | return 0 41 | } 42 | parsed, err := strconv.Atoi(value[:len(value)-1]) 43 | if err != nil { 44 | return 0 45 | } 46 | return int64(parsed) 47 | } 48 | -------------------------------------------------------------------------------- /scanmanager/dryrun.go: -------------------------------------------------------------------------------- 1 | package scanmanager 2 | 3 | import ( 4 | "github.com/AlexAkulov/hungryfox" 5 | "github.com/AlexAkulov/hungryfox/hercules" 6 | ) 7 | 8 | func (sm *ScanManager) DryRun() { 9 | total := sm.repoList.GetTotalRepos() 10 | for i := 0; i < total; i++ { 11 | r := sm.repoList.GetRepoByIndex(i) 12 | if r == nil { 13 | panic("bad index") 14 | return 15 | } 16 | if err := sm.getState(r); err != nil { 17 | sm.Log.Error().Str("error", err.Error()). 18 | Str("data_path", r.Location.DataPath). 19 | Str("repo_path", r.Location.RepoPath). 20 | Str("clone_url", r.Location.CloneURL). 21 | Bool("AllowUpdate", r.Options.AllowUpdate). 22 | Msg("can't open repo") 23 | continue 24 | } 25 | sm.Log.Debug().Int("index", i+1).Int("total", total).Str("data_path", r.Location.DataPath).Str("repo_path", r.Location.RepoPath).Msg("ok") 26 | } 27 | } 28 | 29 | func (sm *ScanManager) getState(r *hungryfox.Repo) error { 30 | r.Repo = &repo.Repo{ 31 | DiffChannel: sm.DiffChannel, 32 | HistoryPastLimit: sm.config.Common.HistoryPastLimit, 33 | DataPath: r.Location.DataPath, 34 | RepoPath: r.Location.RepoPath, 35 | URL: r.Location.URL, 36 | CloneURL: r.Location.CloneURL, 37 | AllowUpdate: r.Options.AllowUpdate, 38 | } 39 | if err := r.Repo.Open(); err != nil { 40 | return err 41 | } 42 | defer r.Repo.Close() 43 | r.State.Refs = r.Repo.GetRefs() 44 | sm.repoList.UpdateRepo(*r) 45 | return nil 46 | } 47 | -------------------------------------------------------------------------------- /hungryfox.go: -------------------------------------------------------------------------------- 1 | package hungryfox 2 | 3 | import "time" 4 | 5 | type Diff struct { 6 | CommitHash string 7 | RepoURL string 8 | RepoPath string 9 | FilePath string 10 | LineBegin int 11 | Content string 12 | AuthorEmail string 13 | Author string 14 | TimeStamp time.Time 15 | } 16 | 17 | type RepoOptions struct { 18 | AllowUpdate bool 19 | } 20 | 21 | type RepoLocation struct { 22 | CloneURL string 23 | URL string 24 | DataPath string 25 | RepoPath string 26 | } 27 | 28 | type RepoState struct { 29 | Refs []string 30 | } 31 | 32 | type ScanStatus struct { 33 | StartTime time.Time 34 | EndTime time.Time 35 | Success bool 36 | } 37 | 38 | type Repo struct { 39 | Options RepoOptions 40 | Location RepoLocation 41 | State RepoState 42 | Scan ScanStatus 43 | Repo IRepo 44 | } 45 | 46 | type IMessageSender interface { 47 | Start() error 48 | Send(Leak) error 49 | Stop() error 50 | } 51 | 52 | type ILeakSearcher interface { 53 | Start() error 54 | SetConfig() error 55 | Search(Diff) 56 | Stop() error 57 | } 58 | 59 | type IRepo interface { 60 | Open() error 61 | Close() error 62 | Scan() error 63 | GetProgress() int 64 | GetRefs() []string 65 | SetRefs([]string) 66 | } 67 | 68 | type IStateManager interface { 69 | Load(string) (RepoState, ScanStatus) 70 | Save(Repo) 71 | } 72 | 73 | type Leak struct { 74 | PatternName string `json:"pattern_name"` 75 | Regexp string `json:"pattern"` 76 | FilePath string `json:"filepath"` 77 | RepoPath string `json:"repo_path"` 78 | LeakString string `json:"leak"` 79 | RepoURL string `json:"repo_url"` 80 | CommitHash string `json:"commit"` 81 | TimeStamp time.Time `json:"ts"` 82 | Line int `json:"line"` 83 | CommitAuthor string `json:"author"` 84 | CommitEmail string `json:"email"` 85 | } 86 | -------------------------------------------------------------------------------- /senders/email/server.go: -------------------------------------------------------------------------------- 1 | package email 2 | 3 | import ( 4 | "crypto/tls" 5 | "fmt" 6 | "html/template" 7 | "net/smtp" 8 | "time" 9 | 10 | "github.com/AlexAkulov/hungryfox" 11 | 12 | "github.com/facebookgo/muster" 13 | "github.com/rs/zerolog" 14 | ) 15 | 16 | // Config - SMTP settings 17 | type Config struct { 18 | From string 19 | SMTPHost string 20 | SMTPPort int 21 | InsecureTLS bool 22 | Username string 23 | Password string 24 | Delay time.Duration 25 | } 26 | 27 | // Sender - send email 28 | type Sender struct { 29 | AuditorEmail string 30 | Config *Config 31 | Log zerolog.Logger 32 | template *template.Template 33 | muster *muster.Client 34 | } 35 | 36 | // Start - start sender 37 | func (s *Sender) Start() error { 38 | t, err := smtp.Dial(fmt.Sprintf("%s:%d", s.Config.SMTPHost, s.Config.SMTPPort)) 39 | if err != nil { 40 | return err 41 | } 42 | defer t.Close() 43 | // Test TLS handshake 44 | if err := t.StartTLS(&tls.Config{ 45 | InsecureSkipVerify: s.Config.InsecureTLS, 46 | ServerName: s.Config.SMTPHost, 47 | }); err != nil { 48 | return err 49 | } 50 | // Test authentication 51 | if s.Config.Password != "" { 52 | if err := t.Auth(smtp.PlainAuth( 53 | "", 54 | s.Config.Username, 55 | s.Config.Password, 56 | s.Config.SMTPHost, 57 | )); err != nil { 58 | return err 59 | } 60 | } 61 | if s.template, err = template.New("mail").Parse(defaultTemplate); err != nil { 62 | return err 63 | } 64 | s.muster = &muster.Client{ 65 | MaxBatchSize: 100, 66 | MaxConcurrentBatches: 1, 67 | BatchTimeout: s.Config.Delay, 68 | BatchMaker: s.batchMaker, 69 | } 70 | return s.muster.Start() 71 | } 72 | 73 | // Stop - stop sender 74 | func (s *Sender) Stop() error { 75 | return s.muster.Stop() 76 | } 77 | 78 | // Send - send leaks 79 | func (s *Sender) Send(leak hungryfox.Leak) error { 80 | s.muster.Work <- leak 81 | return nil 82 | } 83 | -------------------------------------------------------------------------------- /searcher/searcher_test.go: -------------------------------------------------------------------------------- 1 | package searcher 2 | 3 | import ( 4 | "regexp" 5 | "testing" 6 | 7 | "github.com/AlexAkulov/hungryfox" 8 | "github.com/rs/zerolog" 9 | 10 | . "github.com/smartystreets/goconvey/convey" 11 | ) 12 | 13 | func TestGetLeaks(t *testing.T) { 14 | Convey("Test GetLeaks", t, func() { 15 | obj := Searcher{ 16 | Log: zerolog.Nop(), 17 | patterns: []patternType{ 18 | patternType{ 19 | Name: "pattern1", 20 | ContentRe: regexp.MustCompile("secret"), 21 | FileRe: regexp.MustCompile("secret"), 22 | }, 23 | patternType{ 24 | Name: "pattern2", 25 | ContentRe: regexp.MustCompile("Password="), 26 | FileRe: matchAllRegex, 27 | }, 28 | }, 29 | } 30 | testData := hungryfox.Diff{ 31 | CommitHash: "hash123", 32 | RepoURL: "http://github.com", 33 | RepoPath: "my/repo", 34 | FilePath: "no_secret_here.txt", 35 | Content: ` 36 | line 1 37 | line 2 38 | secret1 39 | line 3 40 | secret2 41 | password="qwerty123" 42 | `, 43 | AuthorEmail: "alexakulov86@gmail.com", 44 | Author: "AA", 45 | } 46 | expectedData := []hungryfox.Leak{ 47 | hungryfox.Leak{ 48 | PatternName: "pattern1", 49 | Regexp: "secret", 50 | FilePath: "no_secret_here.txt", 51 | RepoPath: "my/repo", 52 | RepoURL: "http://github.com", 53 | CommitHash: "hash123", 54 | CommitAuthor: "AA", 55 | CommitEmail: "alexakulov86@gmail.com", 56 | LeakString: "\t\t\tsecret1", 57 | }, 58 | hungryfox.Leak{ 59 | PatternName: "pattern1", 60 | Regexp: "secret", 61 | FilePath: "no_secret_here.txt", 62 | RepoPath: "my/repo", 63 | RepoURL: "http://github.com", 64 | CommitHash: "hash123", 65 | CommitAuthor: "AA", 66 | CommitEmail: "alexakulov86@gmail.com", 67 | LeakString: "\t\t\tsecret2", 68 | }, 69 | } 70 | So(obj.GetLeaks(testData), ShouldResemble, expectedData) 71 | }) 72 | } 73 | -------------------------------------------------------------------------------- /scanmanager/github.go: -------------------------------------------------------------------------------- 1 | package scanmanager 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/AlexAkulov/hungryfox" 7 | "github.com/AlexAkulov/hungryfox/config" 8 | "github.com/AlexAkulov/hungryfox/github" 9 | ) 10 | 11 | func getGitHubRepoURL(repoPath string) string { 12 | return fmt.Sprintf("https://github.com/%s", repoPath) 13 | } 14 | func getGitHubCloneURL(repoPath string) string { 15 | return fmt.Sprintf("https://github.com/%s.git", repoPath) 16 | } 17 | func getGitHubRepoPath(repoPath string) (string, error) { 18 | return "", nil 19 | } 20 | 21 | func (sm *ScanManager) inspectGithub(inspect config.Inspect) error { 22 | githubClient := github.Client{ 23 | Token: inspect.Token, 24 | WorkDir: inspect.WorkDir, 25 | } 26 | repoLocations := map[hungryfox.RepoLocation]struct{}{} 27 | 28 | for _, org := range inspect.Orgs { 29 | sm.Log.Debug().Str("organisation", org).Msg("get repos from github.com") 30 | repoList, err := githubClient.FetchOrgRepos(org) 31 | if err != nil { 32 | sm.Log.Error().Str("error", err.Error()).Str("organisation", org).Msg("can't fetch repos from github") 33 | continue 34 | } 35 | for _, repoLocation := range repoList { 36 | repoLocations[repoLocation] = struct{}{} 37 | } 38 | } 39 | for _, user := range inspect.Users { 40 | sm.Log.Debug().Str("user", user).Msg("get repos from github.com") 41 | repoList, err := githubClient.FetchUserRepos(user) 42 | if err != nil { 43 | sm.Log.Error().Str("error", err.Error()).Str("user", user).Msg("can't fetch repos from github") 44 | continue 45 | } 46 | for _, repoLocation := range repoList { 47 | repoLocations[repoLocation] = struct{}{} 48 | } 49 | } 50 | for _, repo := range inspect.Repos { 51 | repoLocation := hungryfox.RepoLocation{ 52 | URL: getGitHubRepoURL(repo), 53 | CloneURL: getGitHubCloneURL(repo), 54 | RepoPath: repo, 55 | DataPath: inspect.WorkDir, 56 | } 57 | repoLocations[repoLocation] = struct{}{} 58 | } 59 | 60 | for repoLocation := range repoLocations { 61 | sm.repoList.AddRepo(hungryfox.Repo{ 62 | Location: repoLocation, 63 | Options: hungryfox.RepoOptions{AllowUpdate: true}, 64 | }) 65 | } 66 | 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | NAME := hungryfox 2 | GIT_TAG := $(shell git describe --always --tags --abbrev=0 | tail -c +2) 3 | GIT_COMMIT := $(shell git rev-list v${GIT_TAG}..HEAD --count) 4 | GO_VERSION := $(shell go version | cut -d' ' -f3) 5 | VERSION := ${GIT_TAG}.${GIT_COMMIT} 6 | RELEASE := 1 7 | GO_VERSION := $(shell go version | cut -d' ' -f3) 8 | BUILD_DATE := $(shell date --iso-8601=second) 9 | LDFLAGS := -ldflags "-X main.version=${VERSION}-${RELEASE} -X main.goVersion=${GO_VERSION} -X main.buildDate=${BUILD_DATE}" 10 | 11 | .PHONY: default clean prepare test test_codecov build rpm travis 12 | 13 | default: clean test build 14 | 15 | clean: 16 | rm -rf build 17 | 18 | test: 19 | go test ./... 20 | 21 | test_codecov: 22 | go test -race -coverprofile="coverage.txt" ./... 23 | 24 | build: 25 | mkdir -p build/root/usr/bin 26 | go build ${LDFLAGS} -o build/root/usr/bin/${NAME} ./cmd/hungryfox 27 | 28 | tar: 29 | mkdir -p build/root/usr/lib/systemd/system 30 | cp pkg/${NAME}.service build/root/usr/lib/systemd/system/${NAME}.service 31 | mkdir -p build/root/etc/${NAME} 32 | build/root/usr/bin/${NAME} -default-config > build/root/etc/${NAME}/config.yml 33 | cp -r pkg/patterns build/root/etc/${NAME}/ 34 | cp -r pkg/filters build/root/etc/${NAME}/ 35 | tar -czvPf build/${NAME}-${VERSION}-${RELEASE}.tar.gz -C build/root . 36 | 37 | rpm: 38 | fpm -t rpm \ 39 | -s "tar" \ 40 | --description "HungryFox" \ 41 | --vendor "Alexander Akulov" \ 42 | --url "https://github.com/AlexAkulov/hungryfox" \ 43 | --license "MIT" \ 44 | --name "${NAME}" \ 45 | --version "${VERSION}" \ 46 | --iteration "${RELEASE}" \ 47 | --config-files "/etc/${NAME}" \ 48 | --after-install "./pkg/postinst" \ 49 | -p build \ 50 | build/${NAME}-${VERSION}-${RELEASE}.tar.gz 51 | 52 | deb: 53 | fpm -t deb \ 54 | -s "tar" \ 55 | --description "HungryFox" \ 56 | --vendor "Alexander Akulov" \ 57 | --url "https://github.com/AlexAkulov/hungryfox" \ 58 | --license "MIT" \ 59 | --name "${NAME}" \ 60 | --version "${VERSION}" \ 61 | --iteration "${RELEASE}" \ 62 | --after-install "./pkg/postinst" \ 63 | -p build \ 64 | build/${NAME}-${VERSION}-${RELEASE}.tar.gz 65 | 66 | travis: test_codecov build tar rpm deb 67 | -------------------------------------------------------------------------------- /scanmanager/path.go: -------------------------------------------------------------------------------- 1 | package scanmanager 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | 9 | "github.com/AlexAkulov/hungryfox" 10 | "github.com/AlexAkulov/hungryfox/config" 11 | ) 12 | 13 | func expandGlob(inspect config.Inspect) (map[string]struct{}, error) { 14 | excludePaths := make(map[string]struct{}) 15 | for _, pattern := range inspect.Paths { 16 | if !strings.HasPrefix(pattern, "!") { 17 | continue 18 | } 19 | pattern = strings.TrimPrefix(pattern, "!") 20 | paths, err := filepath.Glob(pattern) 21 | if err != nil { 22 | return nil, err 23 | } 24 | for _, path := range paths { 25 | excludePaths[path] = struct{}{} 26 | } 27 | } 28 | scanPaths := make(map[string]struct{}) 29 | for _, pattern := range inspect.Paths { 30 | if strings.HasPrefix(pattern, "!") { 31 | continue 32 | } 33 | paths, err := filepath.Glob(pattern) 34 | if err != nil { 35 | return nil, err 36 | } 37 | for _, path := range paths { 38 | if _, ok := excludePaths[path]; ok { 39 | continue 40 | } 41 | if f, _ := os.Stat(path); f.IsDir() { 42 | scanPaths[path] = struct{}{} 43 | } 44 | } 45 | } 46 | return scanPaths, nil 47 | } 48 | 49 | func (sm *ScanManager) inspectRepoPath(inspectObject config.Inspect) error { 50 | scanPathList, err := expandGlob(inspectObject) 51 | if err != nil { 52 | sm.Log.Error().Str("error", err.Error()).Msg("can't expand glob") 53 | return err 54 | } 55 | for path := range scanPathList { 56 | location := getRepoLocation(path, inspectObject) 57 | sm.repoList.AddRepo(hungryfox.Repo{ 58 | Options: hungryfox.RepoOptions{AllowUpdate: false}, 59 | Location: location, 60 | }) 61 | } 62 | return nil 63 | } 64 | 65 | func getRepoLocation(path string, inspectObject config.Inspect) hungryfox.RepoLocation { 66 | prefix := strings.Replace(inspectObject.TrimPrefix, "\\", "/", -1) 67 | prefix = strings.TrimSuffix(prefix, "/") 68 | path = strings.Replace(path, "\\", "/", -1) 69 | path = strings.TrimPrefix(path, prefix) 70 | path = strings.Trim(path, "/") 71 | url := strings.TrimSuffix(inspectObject.URL, "/") 72 | url = fmt.Sprintf("%s/%s", url, strings.TrimSuffix(path, ".git")) 73 | 74 | return hungryfox.RepoLocation{ 75 | DataPath: prefix, 76 | RepoPath: path, 77 | URL: url, 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /repolist/repolist_test.go: -------------------------------------------------------------------------------- 1 | package repolist 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/AlexAkulov/hungryfox" 8 | . "github.com/smartystreets/goconvey/convey" 9 | ) 10 | 11 | type FakeStateManager struct { 12 | data []hungryfox.Repo 13 | } 14 | 15 | func (f FakeStateManager) Load(url string) (hungryfox.RepoState, hungryfox.ScanStatus) { 16 | for i := range f.data { 17 | if f.data[i].Location.URL == url { 18 | return f.data[i].State, f.data[i].Scan 19 | } 20 | } 21 | return hungryfox.RepoState{}, hungryfox.ScanStatus{} 22 | } 23 | 24 | func (f FakeStateManager) Save(r hungryfox.Repo) { 25 | for i := range f.data { 26 | if f.data[i].Location.URL == r.Location.URL { 27 | f.data[i] = r 28 | return 29 | } 30 | } 31 | f.data = append(f.data, r) 32 | } 33 | 34 | func TestGetRepoForScan(t *testing.T) { 35 | Convey("addRepoToScan", t, func() { 36 | now := time.Now().UTC() 37 | testData := []hungryfox.Repo{ 38 | hungryfox.Repo{ 39 | Location: hungryfox.RepoLocation{URL: "minute"}, 40 | Scan: hungryfox.ScanStatus{ 41 | StartTime: now.Add(-time.Minute), 42 | EndTime: now.Add(-time.Minute), 43 | }, 44 | }, 45 | hungryfox.Repo{ 46 | Location: hungryfox.RepoLocation{URL: "day"}, 47 | Scan: hungryfox.ScanStatus{ 48 | StartTime: now.Add(-time.Hour * 24), 49 | EndTime: now.Add(-time.Hour * 24), 50 | }, 51 | }, 52 | hungryfox.Repo{ 53 | Location: hungryfox.RepoLocation{URL: "no scan"}, 54 | Scan: hungryfox.ScanStatus{ 55 | EndTime: now, 56 | }, 57 | }, 58 | hungryfox.Repo{ 59 | Location: hungryfox.RepoLocation{URL: "hour"}, 60 | Scan: hungryfox.ScanStatus{ 61 | StartTime: now.Add(-time.Hour), 62 | EndTime: now.Add(-time.Hour), 63 | }, 64 | }, 65 | } 66 | stateManager := FakeStateManager{data: testData} 67 | rl := RepoList{State: stateManager} 68 | for _, r := range testData { 69 | rl.AddRepo(r) 70 | } 71 | 72 | So(len(rl.list), ShouldEqual, 4) 73 | for _, expectedID := range []int{2, 1, 3, 0} { 74 | id := rl.GetRepoForScan() 75 | So(id, ShouldEqual, expectedID) 76 | r := *rl.GetRepoByIndex(id) 77 | So(r, ShouldResemble, testData[expectedID]) 78 | r.Scan.StartTime = now 79 | r.Scan.EndTime = now 80 | rl.UpdateRepo(r) 81 | } 82 | }) 83 | } 84 | -------------------------------------------------------------------------------- /github/github.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | 7 | "github.com/AlexAkulov/hungryfox" 8 | 9 | "github.com/google/go-github/github" 10 | "golang.org/x/oauth2" 11 | ) 12 | 13 | type Client struct { 14 | Token string 15 | WorkDir string 16 | client *github.Client 17 | } 18 | 19 | func (c *Client) connect() { 20 | if c.client == nil { 21 | c.client = github.NewClient(c.getTokenClient()) 22 | } 23 | } 24 | 25 | func (c *Client) FetchOrgRepos(orgName string) ([]hungryfox.RepoLocation, error) { 26 | opts := &github.RepositoryListByOrgOptions{ 27 | ListOptions: github.ListOptions{PerPage: 10}, 28 | } 29 | c.connect() 30 | ctx := context.Background() 31 | var repoList []hungryfox.RepoLocation 32 | 33 | for { 34 | repo, resp, err := c.client.Repositories.ListByOrg(ctx, orgName, opts) 35 | if err != nil { 36 | return repoList, err 37 | } 38 | if resp.NextPage == 0 { 39 | break 40 | } 41 | repoList = append(repoList, c.convertRepoList(repo)...) 42 | opts.Page = resp.NextPage 43 | } 44 | return repoList, nil 45 | } 46 | 47 | func (c *Client) FetchUserRepos(userName string) ([]hungryfox.RepoLocation, error) { 48 | opts := &github.RepositoryListOptions{ 49 | ListOptions: github.ListOptions{PerPage: 10}, 50 | } 51 | c.connect() 52 | ctx := context.Background() 53 | var repoList []hungryfox.RepoLocation 54 | for { 55 | repo, resp, err := c.client.Repositories.List(ctx, userName, opts) 56 | if err != nil { 57 | return repoList, err 58 | } 59 | if resp.NextPage == 0 { 60 | break 61 | } 62 | repoList = append(repoList, c.convertRepoList(repo)...) 63 | opts.Page = resp.NextPage 64 | } 65 | return repoList, nil 66 | } 67 | 68 | func (c *Client) convertRepoList(list []*github.Repository) (hfRepoList []hungryfox.RepoLocation) { 69 | for _, repo := range list { 70 | hfRepoList = append(hfRepoList, hungryfox.RepoLocation{ 71 | URL: *repo.HTMLURL, 72 | CloneURL: *repo.CloneURL, 73 | DataPath: c.WorkDir, 74 | RepoPath: *repo.FullName, 75 | }) 76 | } 77 | return 78 | } 79 | 80 | func (c *Client) getTokenClient() *http.Client { 81 | if c.Token == "" { 82 | return nil 83 | } 84 | return oauth2.NewClient( 85 | context.Background(), 86 | oauth2.StaticTokenSource(&oauth2.Token{AccessToken: c.Token}), 87 | ) 88 | } 89 | -------------------------------------------------------------------------------- /router/router.go: -------------------------------------------------------------------------------- 1 | package router 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/AlexAkulov/hungryfox" 7 | "github.com/AlexAkulov/hungryfox/config" 8 | "github.com/AlexAkulov/hungryfox/helpers" 9 | "github.com/AlexAkulov/hungryfox/senders/email" 10 | "github.com/AlexAkulov/hungryfox/senders/file" 11 | "github.com/AlexAkulov/hungryfox/senders/webhook" 12 | 13 | "github.com/rs/zerolog" 14 | "gopkg.in/tomb.v2" 15 | ) 16 | 17 | type LeaksRouter struct { 18 | LeakChannel <-chan *hungryfox.Leak 19 | Config *config.Config 20 | Log zerolog.Logger 21 | 22 | senders map[string]hungryfox.IMessageSender 23 | tomb tomb.Tomb 24 | } 25 | 26 | func (r *LeaksRouter) Start() error { 27 | delay, err := helpers.ParseDuration(r.Config.SMTP.Delay) 28 | if err != nil { 29 | return fmt.Errorf("can't parse delay with: %v", err) 30 | } 31 | r.senders = map[string]hungryfox.IMessageSender{} 32 | if r.Config.SMTP.Enable { 33 | r.senders["email"] = &email.Sender{ 34 | AuditorEmail: r.Config.SMTP.Recipient, 35 | Config: &email.Config{ 36 | From: r.Config.SMTP.From, 37 | SMTPHost: r.Config.SMTP.Host, 38 | SMTPPort: r.Config.SMTP.Port, 39 | InsecureTLS: !r.Config.SMTP.TLS, 40 | Username: r.Config.SMTP.Username, 41 | Password: r.Config.SMTP.Password, 42 | Delay: delay, 43 | }, 44 | Log: r.Log, 45 | } 46 | } 47 | 48 | if r.Config.WebHook.Enable { 49 | r.senders["webhook"] = &webhook.Sender{ 50 | Method: r.Config.WebHook.Method, 51 | URL: r.Config.WebHook.URL, 52 | Headers: r.Config.WebHook.Headers, 53 | } 54 | } 55 | 56 | r.senders["file"] = &file.File{ 57 | LeaksFile: r.Config.Common.LeaksFile, 58 | } 59 | 60 | for senderName, sender := range r.senders { 61 | if err := sender.Start(); err != nil { 62 | return err 63 | } 64 | r.Log.Debug().Str("service", senderName).Msg("strated") 65 | } 66 | 67 | r.tomb.Go(func() error { 68 | for { 69 | select { 70 | case <-r.tomb.Dying(): // Stop 71 | return nil 72 | case leak := <-r.LeakChannel: 73 | for _, sender := range r.senders { 74 | sender.Send(*leak) 75 | } 76 | } 77 | } 78 | }) 79 | return nil 80 | } 81 | 82 | func (r *LeaksRouter) Stop() error { 83 | r.tomb.Kill(nil) 84 | r.tomb.Wait() 85 | for _, sender := range r.senders { 86 | sender.Stop() 87 | } 88 | return nil 89 | } 90 | -------------------------------------------------------------------------------- /senders/email/sender.go: -------------------------------------------------------------------------------- 1 | package email 2 | 3 | import ( 4 | "crypto/tls" 5 | "fmt" 6 | "io" 7 | "net/smtp" 8 | "strings" 9 | 10 | "github.com/AlexAkulov/hungryfox" 11 | 12 | "github.com/facebookgo/muster" 13 | "gopkg.in/gomail.v2" 14 | ) 15 | 16 | type mailTemplateStruct struct { 17 | LeaksCount int 18 | FilesCount int 19 | Repos []*mailTemplateRepoStruct 20 | } 21 | 22 | type mailTemplateRepoStruct struct { 23 | RepoURL string 24 | Items []hungryfox.Leak 25 | } 26 | 27 | func (s *Sender) batchMaker() muster.Batch { 28 | return &batch{ 29 | Sender: s, 30 | Repos: map[string]*mailTemplateRepoStruct{}, 31 | Files: map[string]struct{}{}, 32 | } 33 | } 34 | 35 | type batch struct { 36 | LeaksCount int 37 | Repos map[string]*mailTemplateRepoStruct 38 | Files map[string]struct{} 39 | Sender *Sender 40 | } 41 | 42 | func (b *batch) Fire(notifier muster.Notifier) { 43 | defer notifier.Done() 44 | if b.LeaksCount < 1 { 45 | return 46 | } 47 | messageData := &mailTemplateStruct{ 48 | FilesCount: len(b.Files), 49 | LeaksCount: b.LeaksCount, 50 | } 51 | for _, repo := range b.Repos { 52 | messageData.Repos = append(messageData.Repos, repo) 53 | } 54 | err := b.Sender.sendMessage(b.Sender.AuditorEmail, messageData) 55 | if err != nil { 56 | b.Sender.Log.Error().Str("error", err.Error()).Msg("can't send email") 57 | } 58 | } 59 | 60 | func (b *batch) Add(item interface{}) { 61 | leak := item.(hungryfox.Leak) 62 | leak.LeakString = strings.TrimSpace(leak.LeakString) 63 | if len(leak.LeakString) > 512 { 64 | leak.LeakString = "too long" 65 | } 66 | if b.Repos[leak.RepoURL] == nil { 67 | b.Repos[leak.RepoURL] = &mailTemplateRepoStruct{ 68 | RepoURL: leak.RepoURL, 69 | Items: []hungryfox.Leak{}, 70 | } 71 | } 72 | b.Repos[leak.RepoURL].Items = append(b.Repos[leak.RepoURL].Items, leak) 73 | b.Files[fmt.Sprintf("%s/%s", leak.RepoURL, leak.FilePath)] = struct{}{} 74 | b.LeaksCount++ 75 | } 76 | 77 | func (s *Sender) sendMessage(recipient string, messageData *mailTemplateStruct) error { 78 | d := gomail.Dialer{ 79 | Host: s.Config.SMTPHost, 80 | Port: s.Config.SMTPPort, 81 | TLSConfig: &tls.Config{ 82 | InsecureSkipVerify: s.Config.InsecureTLS, 83 | ServerName: s.Config.SMTPHost, 84 | }, 85 | } 86 | if s.Config.Password != "" { 87 | d.Auth = smtp.PlainAuth( 88 | "", 89 | s.Config.Username, 90 | s.Config.Password, 91 | s.Config.SMTPHost) 92 | } 93 | 94 | m := gomail.NewMessage() 95 | m.SetHeader("From", s.Config.From) 96 | m.SetHeader("To", strings.Split(recipient, ",")...) 97 | 98 | var subject string 99 | if len(messageData.Repos) == 1 { 100 | subject = fmt.Sprintf("Found %d leaks in %s", messageData.LeaksCount, messageData.Repos[0].RepoURL) 101 | } else { 102 | subject = fmt.Sprintf("Found %d leaks in %d repos", messageData.LeaksCount, len(messageData.Repos)) 103 | } 104 | m.SetHeader("Subject", subject) 105 | m.AddAlternativeWriter("text/html", func(w io.Writer) error { 106 | return s.template.Execute(w, messageData) 107 | }) 108 | return d.DialAndSend(m) 109 | } 110 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/AlexAkulov/hungryfox 2 | 3 | require ( 4 | github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52 5 | github.com/google/go-github v17.0.0+incompatible 6 | github.com/rs/zerolog v1.14.3 7 | github.com/sasha-s/go-deadlock v0.2.0 8 | github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a 9 | golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a 10 | gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df 11 | gopkg.in/src-d/go-git.v4 v4.11.0 12 | gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 13 | gopkg.in/yaml.v2 v2.2.2 14 | upper.io/db.v3 v3.5.7+incompatible 15 | ) 16 | 17 | require ( 18 | github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d // indirect 19 | github.com/cznic/fileutil v0.0.0-20181122101858-4d67cfea8c87 // indirect 20 | github.com/cznic/golex v0.0.0-20181122101858-9c343928389c // indirect 21 | github.com/cznic/internal v0.0.0-20181122101858-3279554c546e // indirect 22 | github.com/cznic/lldb v1.1.0 // indirect 23 | github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect 24 | github.com/cznic/ql v1.2.0 // indirect 25 | github.com/cznic/sortutil v0.0.0-20181122101858-f5f958428db8 // indirect 26 | github.com/cznic/strutil v0.0.0-20181122101858-275e90344537 // indirect 27 | github.com/cznic/zappy v0.0.0-20181122101859-ca47d358d4b1 // indirect 28 | github.com/edsrzf/mmap-go v1.0.0 // indirect 29 | github.com/emirpasic/gods v1.12.0 // indirect 30 | github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect 31 | github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect 32 | github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01 // indirect 33 | github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect 34 | github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect 35 | github.com/gliderlabs/ssh v0.1.4 // indirect 36 | github.com/golang/protobuf v1.3.1 // indirect 37 | github.com/golang/snappy v0.0.1 // indirect 38 | github.com/google/go-querystring v1.0.0 // indirect 39 | github.com/gopherjs/gopherjs v0.0.0-20190411002643-bd77b112433e // indirect 40 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect 41 | github.com/jtolds/gls v4.20.0+incompatible // indirect 42 | github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e // indirect 43 | github.com/mitchellh/go-homedir v1.1.0 // indirect 44 | github.com/pelletier/go-buffruneio v0.2.0 // indirect 45 | github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect 46 | github.com/remyoudompheng/bigfft v0.0.0-20190321074620-2f0d2b0e0001 // indirect 47 | github.com/sergi/go-diff v1.0.0 // indirect 48 | github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect 49 | github.com/src-d/gcfg v1.4.0 // indirect 50 | github.com/stretchr/testify v1.3.0 // indirect 51 | github.com/xanzy/ssh-agent v0.2.1 // indirect 52 | golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 // indirect 53 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect 54 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect 55 | golang.org/x/text v0.3.8 // indirect 56 | google.golang.org/appengine v1.5.0 // indirect 57 | gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect 58 | gopkg.in/src-d/go-billy.v4 v4.3.0 // indirect 59 | gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 // indirect 60 | gopkg.in/warnings.v0 v0.1.2 // indirect 61 | ) 62 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "time" 7 | 8 | "github.com/AlexAkulov/hungryfox/helpers" 9 | 10 | "gopkg.in/yaml.v2" 11 | ) 12 | 13 | type WebHook struct { 14 | Enable bool `yaml:"enable"` 15 | Method string `yaml:"method"` 16 | URL string `yaml:"url"` 17 | Headers map[string]string `yaml:"headers"` 18 | } 19 | 20 | type SMTP struct { 21 | Enable bool `yaml:"enable"` 22 | From string `yaml:"mail_from"` 23 | Host string `yaml:"host"` 24 | Port int `yaml:"port"` 25 | TLS bool `yaml:"tls"` 26 | Username string `yaml:"username"` 27 | Password string `yaml:"password"` 28 | Recipient string `yaml:"recipient"` 29 | SentToAuthor bool `yaml:"sent_to_autor"` 30 | Delay string `yaml:"delay"` 31 | } 32 | 33 | type Config struct { 34 | Common *Common `yaml:"common"` 35 | Inspect []Inspect `yaml:"inspect"` 36 | Patterns []Pattern `yaml:"patterns"` 37 | Filters []Pattern `yaml:"filters"` 38 | SMTP *SMTP `yaml:"smtp"` 39 | WebHook *WebHook `yaml:"webhook"` 40 | } 41 | 42 | type Inspect struct { 43 | Type string `yaml:"type"` 44 | Paths []string `yaml:"paths"` 45 | URL string `yaml:"url"` 46 | Token string `yaml:"token"` 47 | TrimPrefix string `yaml:"trim_prefix"` 48 | TrimSuffix string `yaml:"trim_suffix"` 49 | WorkDir string `yaml:"work_dir"` 50 | Users []string `yaml:"users"` 51 | Repos []string `yaml:"repos"` 52 | Orgs []string `yaml:"orgs"` 53 | } 54 | 55 | type Common struct { 56 | StateFile string `yaml:"state_file"` 57 | HistoryPastLimitString string `yaml:"history_limit"` 58 | LogLevel string `yaml:"log_level"` 59 | LeaksFile string `yaml:"leaks_file"` 60 | ScanIntervalString string `yaml:"scan_interval"` 61 | PatternsPath string `yaml:"patterns_path"` 62 | FiltresPath string `yaml:"filters_path"` 63 | Workers int `yaml:"workers"` 64 | HistoryPastLimit time.Time 65 | ScanInterval time.Duration 66 | } 67 | 68 | type Pattern struct { 69 | Name string `yaml:"name"` 70 | File string `yaml:"file"` 71 | Content string `yaml:"content"` 72 | } 73 | 74 | func defaultConfig() *Config { 75 | return &Config{ 76 | SMTP: &SMTP{ 77 | Delay: "5m", 78 | }, 79 | } 80 | } 81 | 82 | func LoadConfig(configLocation string) (*Config, error) { 83 | config := defaultConfig() 84 | configYaml, err := ioutil.ReadFile(configLocation) 85 | if err != nil { 86 | return nil, fmt.Errorf("can't read with: %v", err) 87 | } 88 | err = yaml.Unmarshal(configYaml, &config) 89 | if err != nil { 90 | return nil, fmt.Errorf("can't parse with: %v", err) 91 | } 92 | pastLimit, err := helpers.ParseDuration(config.Common.HistoryPastLimitString) 93 | if err != nil { 94 | return nil, err 95 | } 96 | config.Common.HistoryPastLimit = time.Now().Add(-pastLimit) 97 | config.Common.ScanInterval, err = helpers.ParseDuration(config.Common.ScanIntervalString) 98 | if err != nil { 99 | return nil, err 100 | } 101 | if config.Common.ScanInterval < time.Second { 102 | return nil, fmt.Errorf("scan_interval so small") 103 | } 104 | return config, nil 105 | } 106 | 107 | func PrintDefaultConfig() { 108 | c := defaultConfig() 109 | d, _ := yaml.Marshal(&c) 110 | fmt.Print(string(d)) 111 | } 112 | -------------------------------------------------------------------------------- /state/filestate/filestate.go: -------------------------------------------------------------------------------- 1 | package filestate 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "time" 8 | 9 | "github.com/AlexAkulov/hungryfox" 10 | 11 | "gopkg.in/tomb.v2" 12 | "gopkg.in/yaml.v2" 13 | ) 14 | 15 | type StateManager struct { 16 | Location string 17 | state map[string]hungryfox.Repo 18 | tomb tomb.Tomb 19 | saveRepoChan chan hungryfox.Repo 20 | loadRepoChan chan hungryfox.Repo 21 | loadRepoChanRequest chan string 22 | } 23 | 24 | func (s *StateManager) Start() error { 25 | if err := s.load(); err != nil { 26 | return err 27 | } 28 | s.saveRepoChan = make(chan hungryfox.Repo) 29 | s.loadRepoChan = make(chan hungryfox.Repo) 30 | s.loadRepoChanRequest = make(chan string) 31 | 32 | s.tomb.Go(func() error { 33 | saveTicker := time.NewTicker(time.Minute) 34 | for { 35 | select { 36 | case <-s.tomb.Dying(): 37 | return s.saveToFile() 38 | case <-saveTicker.C: 39 | if err := s.saveToFile(); err != nil { 40 | fmt.Printf("can't save state with err: %v\n", err) 41 | } 42 | case r := <-s.saveRepoChan: 43 | s.state[r.Location.URL] = r 44 | case url := <-s.loadRepoChanRequest: 45 | if r, ok := s.state[url]; ok { 46 | s.loadRepoChan <- r 47 | continue 48 | } 49 | s.loadRepoChan <- hungryfox.Repo{} 50 | } 51 | } 52 | }) 53 | return nil 54 | } 55 | 56 | func (s *StateManager) Stop() error { 57 | s.tomb.Kill(nil) 58 | return s.tomb.Wait() 59 | } 60 | 61 | func (s *StateManager) load() error { 62 | if _, err := os.Stat(s.Location); os.IsNotExist(err) { 63 | if _, err := os.Create(s.Location); err != nil { 64 | return fmt.Errorf("can't create with: %v", err) 65 | } 66 | } 67 | 68 | stateRaw, err := ioutil.ReadFile(s.Location) 69 | if err != nil { 70 | return fmt.Errorf("can't open, %v", err) 71 | } 72 | if s.state, err = converFromRawData(stateRaw); err != nil { 73 | return fmt.Errorf("can't parse, %v", err) 74 | } 75 | return nil 76 | } 77 | 78 | func convertToRawData(stateStruct map[string]hungryfox.Repo) ([]byte, error) { 79 | fileStruct := []RepoJSON{} 80 | for _, r := range stateStruct { 81 | fileStruct = append(fileStruct, RepoJSON{ 82 | RepoURL: r.Location.URL, 83 | CloneURL: r.Location.CloneURL, 84 | RepoPath: r.Location.RepoPath, 85 | DataPath: r.Location.DataPath, 86 | Refs: r.State.Refs, 87 | ScanStatus: ScanJSON{ 88 | StartTime: r.Scan.StartTime, 89 | EndTime: r.Scan.EndTime, 90 | Success: r.Scan.Success, 91 | }, 92 | }) 93 | } 94 | return yaml.Marshal(&fileStruct) 95 | } 96 | 97 | func converFromRawData(rawData []byte) (map[string]hungryfox.Repo, error) { 98 | stateJSON := []RepoJSON{} 99 | if err := yaml.Unmarshal(rawData, &stateJSON); err != nil { 100 | return nil, err 101 | } 102 | result := map[string]hungryfox.Repo{} 103 | for _, r := range stateJSON { 104 | result[r.RepoURL] = hungryfox.Repo{ 105 | Location: hungryfox.RepoLocation{ 106 | URL: r.RepoURL, 107 | CloneURL: r.CloneURL, 108 | DataPath: r.DataPath, 109 | RepoPath: r.RepoPath, 110 | }, 111 | State: hungryfox.RepoState{ 112 | Refs: r.Refs, 113 | }, 114 | Scan: hungryfox.ScanStatus{ 115 | StartTime: r.ScanStatus.StartTime, 116 | EndTime: r.ScanStatus.EndTime, 117 | Success: r.ScanStatus.Success, 118 | }, 119 | } 120 | } 121 | return result, nil 122 | } 123 | 124 | func (s *StateManager) saveToFile() error { 125 | if _, err := os.Stat(s.Location); os.IsNotExist(err) { 126 | if _, err := os.Create(s.Location); err != nil { 127 | return fmt.Errorf("can't create, %v", err) 128 | } 129 | } 130 | rawData, err := convertToRawData(s.state) 131 | if err != nil { 132 | return err 133 | } 134 | if err := ioutil.WriteFile(s.Location, rawData, 0644); err != nil { 135 | return fmt.Errorf("can't save, %v", err) 136 | } 137 | return nil 138 | } 139 | 140 | func (s StateManager) Save(r hungryfox.Repo) { 141 | s.saveRepoChan <- r 142 | } 143 | 144 | func (s StateManager) Load(url string) (hungryfox.RepoState, hungryfox.ScanStatus) { 145 | s.loadRepoChanRequest <- url 146 | r := <-s.loadRepoChan 147 | return r.State, r.Scan 148 | } 149 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HungryFox 2 | 3 | [![Build Status](https://travis-ci.org/AlexAkulov/hungryfox.svg?branch=master)](https://travis-ci.org/AlexAkulov/hungryfox) 4 | [![codecov](https://codecov.io/gh/AlexAkulov/hungryfox/branch/master/graph/badge.svg)](https://codecov.io/gh/AlexAkulov/hungryfox) 5 | 6 | 7 | **State: In development now! You probably will get many bugs!** 8 | 9 | HungryFox is a software for continuous search for leaks of sensitive information like passwords, api-keys, private certificates and etc in your repositories. 10 | 11 | HungryFox differs from other solutions as it can work as a daemon and efficiently scans each new commit in repo and sends notification about found leaks. 12 | 13 | HungryFor works on regex-patterns only and does not use analyze by entropy because in my opinion this way generates a lot of false positive events. Maybe analyse by entropy will be added in future. 14 | 15 | It is hard to write a good enough regex-pattern that could simultaneously find all leaks and not to generate a lot of false positive events so HungryFox in addition with regex-patterns has regex-filters. You can write 16 | weak regex-pattern for search leaks and skip known false positive with the help of regex-filters. 17 | 18 | 19 | ## Features 20 | - [x] Patterns and filters 21 | - [x] State support 22 | - [x] Notifications by email 23 | - [x] History limit by time 24 | - [x] GitHub-support 25 | - [ ] Written on pure go and no requirement of external git ([wait](https://github.com/src-d/go-git/issues/757)) 26 | - [ ] Line number of leak ([wait](https://github.com/src-d/go-git/issues/806)) 27 | - [ ] GitHook support 28 | - [ ] HTTP Api 29 | - [ ] WebUI 30 | - [ ] Tests 31 | - [ ] Integration with Hashicorp Vault 32 | 33 | ## Installation 34 | 35 | ### From Sources 36 | 37 | ``` 38 | go get github.com/AlexAkulov/hungryfox/cmd/hungryfox 39 | ``` 40 | 41 | ### From [packagecloud.io](https://packagecloud.io/AlexAkulov/hungryfox-unstable) 42 | 43 | [![](https://img.shields.io/badge/deb-packagecloud.io-844fec.svg)](https://packagecloud.io/AlexAkulov/hungryfox-unstable/install#bash-deb) 44 | [![](https://img.shields.io/badge/rpm-packagecloud.io-844fec.svg)](https://packagecloud.io/AlexAkulov/hungryfox-unstable/install#bash-rpm) 45 | 46 | 47 | ## Configuation 48 | ``` 49 | common: 50 | state_file: /var/lib/hungryfox/state.yml 51 | history_limit: 1y 52 | scan_interval: 30m 53 | log_level: debug 54 | leaks_file: /var/lib/hungryfox/leaks.json 55 | 56 | smtp: 57 | enable: true 58 | host: smtp.kontur 59 | port: 25 60 | mail_from: hungryfox@example.com 61 | disable_tls: true 62 | recipient: security@example.com 63 | sent_to_author: false 64 | 65 | webhook: 66 | enable: true 67 | method: POST 68 | url: https://example.com/webhook 69 | headers: 70 | x-sample-header: value 71 | 72 | inspect: 73 | # Inspects for leaks in your local repositories without clone or fetch. It is suitable for running on git-server 74 | - type: path 75 | trim_prefix: "/var/volume/repositories" 76 | trim_suffix: ".git" 77 | url: https://gitlab.example.com 78 | paths: 79 | - "/data/gitlab/repositories/*/*.git" 80 | - "/data/gitlab/repositories/*/*/*.git" 81 | - "!/data/gitlab/repositories/excluded/repo.git" 82 | # Inspects for leaks on GitHub. HungryFox will clone the repositories into work_dir and fetch them before scannig 83 | - type: github 84 | token: # is required for scanning private repositories 85 | work_dir: "/var/hungryfox/github" 86 | users: 87 | - AlexAkulov 88 | repos: 89 | - moira-alert/moira 90 | orgs: 91 | - skbkontur 92 | 93 | patterns: 94 | - name: secret in my code # not required 95 | file: \.go$ # .+ by default 96 | content: (?i)secret = ".+" # .+ by default 97 | 98 | filters: 99 | - name: skip any leaks in tests # not required 100 | file: /IntegrationTests/.+_test\.go$ # .+ by default 101 | # content: # .+ by default 102 | ``` 103 | ## Performance 104 | We use HungryFox for scanning ~3,5K repositories on our GitLab server and about one hundred repositories on GitHub 105 | 106 | ## Alternatives 107 | - [Gitrob by michenriksen](https://github.com/michenriksen/gitrob) 108 | - [Gitleaks by zricethezav](https://github.com/zricethezav/gitleaks) 109 | - [git-secrets by AWSLabs](https://github.com/awslabs/git-secrets) 110 | - [Truffle Hog by dxa4481](https://github.com/dxa4481/truffleHog) 111 | - [repo-scraper by dssg](https://github.com/dssg/repo-scraper) 112 | - [Security Scan by onetwopunch](https://github.com/onetwopunch/security-scan) 113 | - [repo-security-scanner by UKHomeOffice](https://github.com/UKHomeOffice/repo-security-scanner) 114 | - [detect-secrets by Yelp](https://github.com/Yelp/detect-secrets) 115 | - [Github Dorks by techgaun](https://github.com/techgaun/github-dorks) 116 | - [Repo Supervisor by Auth0](https://github.com/auth0/repo-supervisor) 117 | - [git-all-secrets by anshumanbh](https://github.com/anshumanbh/git-all-secrets) 118 | -------------------------------------------------------------------------------- /scanmanager/scanmanager.go: -------------------------------------------------------------------------------- 1 | package scanmanager 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/AlexAkulov/hungryfox" 7 | "github.com/AlexAkulov/hungryfox/config" 8 | "github.com/AlexAkulov/hungryfox/helpers" 9 | "github.com/AlexAkulov/hungryfox/hercules" 10 | "github.com/AlexAkulov/hungryfox/repolist" 11 | 12 | "github.com/rs/zerolog" 13 | "gopkg.in/tomb.v2" 14 | ) 15 | 16 | // ScanManager - 17 | type ScanManager struct { 18 | DiffChannel chan<- *hungryfox.Diff 19 | Log zerolog.Logger 20 | StateManager hungryfox.IStateManager 21 | 22 | config *config.Config 23 | tomb tomb.Tomb 24 | currentRepo int 25 | repoList *repolist.RepoList 26 | } 27 | 28 | // SetConfig - update configuration 29 | func (sm *ScanManager) SetConfig(config *config.Config) { 30 | sm.config = config 31 | sm.Log.Debug().Str("service", "scan manager").Msg("config reloaded") 32 | sm.updateScanList() 33 | } 34 | 35 | // Status - get status for current repo 36 | func (sm *ScanManager) Status() *hungryfox.Repo { 37 | return sm.repoList.GetRepoByIndex(sm.currentRepo) 38 | } 39 | 40 | func (sm *ScanManager) updateScanList() { 41 | sm.Log.Debug().Str("status", "start").Msg("update scan list") 42 | if sm.repoList == nil { 43 | sm.repoList = &repolist.RepoList{State: sm.StateManager} 44 | } 45 | sm.repoList.Clear() 46 | for _, inspectObject := range sm.config.Inspect { 47 | switch inspectObject.Type { 48 | case "path": 49 | sm.inspectRepoPath(inspectObject) 50 | case "github": 51 | sm.inspectGithub(inspectObject) 52 | default: 53 | sm.Log.Error().Str("type", inspectObject.Type).Msg("unsupported type") 54 | } 55 | } 56 | sm.Log.Debug().Str("status", "complete").Msg("update scan list") 57 | } 58 | 59 | // Start - start ScanManager instance 60 | func (sm *ScanManager) Start(config *config.Config) error { 61 | sm.config = config 62 | sm.currentRepo = -1 63 | sm.updateScanList() 64 | 65 | sm.tomb.Go(func() error { 66 | updateTicker := time.NewTicker(time.Minute * 30) 67 | scanTimer := time.NewTimer(time.Second) 68 | for { 69 | select { 70 | case <-sm.tomb.Dying(): 71 | return nil 72 | case <-updateTicker.C: 73 | sm.updateScanList() 74 | case <-scanTimer.C: 75 | scanTimer = sm.scanNext() 76 | } 77 | } 78 | }) 79 | return nil 80 | } 81 | 82 | func (sm *ScanManager) Stop() error { 83 | sm.tomb.Kill(nil) 84 | if err := sm.tomb.Wait(); err != nil { 85 | sm.Log.Error().Str("error", err.Error()).Msg("stop") 86 | } 87 | return nil 88 | } 89 | 90 | func (sm *ScanManager) scanNext() *time.Timer { 91 | rID := sm.repoList.GetRepoForScan() 92 | if rID < 0 { 93 | waitTime := time.Duration(time.Minute) 94 | sm.Log.Debug().Str("wait", helpers.PrettyDuration(waitTime)).Msg("no repo for scan") 95 | return time.NewTimer(waitTime) 96 | } 97 | sm.currentRepo = rID 98 | defer func() { 99 | sm.currentRepo = -1 100 | }() 101 | r := sm.repoList.GetRepoByIndex(rID) 102 | elapsedTime := time.Since(r.Scan.EndTime) 103 | if elapsedTime > sm.config.Common.ScanInterval { 104 | sm.Log.Info().Str("data_path", r.Location.DataPath).Str("repo_path", r.Location.RepoPath).Msg("start scan") 105 | sm.ScanRepo(rID) 106 | return time.NewTimer(0) 107 | } 108 | waitTime := sm.config.Common.ScanInterval - elapsedTime 109 | sm.Log.Info().Str("wait", helpers.PrettyDuration(waitTime)).Msg("wait repo for scan") 110 | return time.NewTimer(waitTime) 111 | } 112 | 113 | // ScanRepo - open exist git repository and fing leaks 114 | func (sm *ScanManager) ScanRepo(index int) { 115 | r := sm.repoList.GetRepoByIndex(index) 116 | if r == nil { 117 | panic("bad index") 118 | } 119 | sm.Log.Debug().Str("repo_url", r.Location.URL).Int("refs", len(r.State.Refs)).Msg("state loaded") 120 | r.Repo = &repo.Repo{ 121 | DiffChannel: sm.DiffChannel, 122 | HistoryPastLimit: sm.config.Common.HistoryPastLimit, 123 | DataPath: r.Location.DataPath, 124 | RepoPath: r.Location.RepoPath, 125 | URL: r.Location.URL, 126 | CloneURL: r.Location.CloneURL, 127 | AllowUpdate: r.Options.AllowUpdate, 128 | } 129 | r.Repo.SetRefs(r.State.Refs) 130 | startScan := time.Now().UTC() 131 | r.Scan.StartTime = startScan 132 | sm.repoList.UpdateRepo(*r) 133 | 134 | err := openScanClose(*r) 135 | r.State.Refs = r.Repo.GetRefs() 136 | newR := hungryfox.Repo{ 137 | Location: r.Location, 138 | Options: r.Options, 139 | State: hungryfox.RepoState{Refs: r.Repo.GetRefs()}, 140 | Scan: hungryfox.ScanStatus{ 141 | StartTime: startScan, 142 | EndTime: time.Now().UTC(), 143 | Success: err == nil, 144 | }, 145 | } 146 | sm.repoList.UpdateRepo(newR) 147 | 148 | if err != nil { 149 | sm.Log.Error().Str("data_path", newR.Location.DataPath).Str("repo_path", newR.Location.RepoPath).Str("error", err.Error()).Msg("scan failed") 150 | } else { 151 | sm.Log.Info().Str("data_path", newR.Location.DataPath).Str("repo_path", newR.Location.RepoPath).Str("duration", helpers.PrettyDuration(time.Since(newR.Scan.StartTime))).Msg("scan completed") 152 | } 153 | return 154 | } 155 | 156 | func openScanClose(r hungryfox.Repo) error { 157 | if err := r.Repo.Open(); err != nil { 158 | return err 159 | } 160 | defer r.Repo.Close() 161 | return r.Repo.Scan() 162 | } 163 | -------------------------------------------------------------------------------- /senders/email/template.go: -------------------------------------------------------------------------------- 1 | package email 2 | 3 | const defaultTemplate = ` 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 85 | 86 | 87 | 88 | 89 | 90 | 109 | 110 | 111 | 112 | 166 | 167 | 168 | 169 | 173 | 174 |
91 | 96 | 97 | 98 | 101 | 102 |
99 |

GitLab Security Alert

100 |
103 | 108 |
113 | 118 | 119 | 120 | 121 | 127 | 128 | 129 | {{ range .Repos }} 130 | 131 | 135 | 136 | 137 | {{ range .Items }} 138 | 139 | 148 | 149 | {{ end }} 150 | 151 | 152 | 153 | {{ end }} 154 | 155 | 158 | 159 |
122 |

Кажется, мы нашли что-то похожее на пароль, токен или ключ.

123 |

Как удалить пароль из репозитория написано тут.

124 |

Если это ошибка, ответь на это письмо чтобы мы добавили это в исключения.

125 |

Найдено {{ .LeaksCount }} утечек в {{ .FilesCount }} файлах. 126 |

132 | {{ .RepoURL }} 133 |
134 |
140 |

{{ .FilePath }} 141 |

142 |

{{ .LeakString }}

143 |

Commit 144 | {{ .CommitHash }} by 145 | {{ .CommitAuthor }} ({{ .TimeStamp.Format "15:04:05 02.01.2006" }})

146 | 147 |
156 |

Отдел безопасности веб-сервисов

157 |
160 | 165 |
170 |

Не хочу больше получать такие письма, 171 | отписаться.

172 |
175 | 176 | 177 | ` 178 | -------------------------------------------------------------------------------- /cmd/hungryfox/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "net/http" 7 | _ "net/http/pprof" 8 | "os" 9 | "os/signal" 10 | "runtime" 11 | "syscall" 12 | "time" 13 | 14 | "github.com/AlexAkulov/hungryfox" 15 | "github.com/AlexAkulov/hungryfox/config" 16 | "github.com/AlexAkulov/hungryfox/helpers" 17 | "github.com/AlexAkulov/hungryfox/router" 18 | "github.com/AlexAkulov/hungryfox/scanmanager" 19 | "github.com/AlexAkulov/hungryfox/searcher" 20 | "github.com/AlexAkulov/hungryfox/state/filestate" 21 | 22 | "github.com/rs/zerolog" 23 | ) 24 | 25 | var ( 26 | version = "unknown" 27 | skipScan = flag.Bool("skip-scan", false, "Update state for all repo") 28 | configFlag = flag.String("config", "config.yml", "config file location") 29 | pprofFlag = flag.Bool("pprof", false, "Enable listen pprof on :6060") 30 | printConfigFlag = flag.Bool("default-config", false, "Print default config to stdout and exit") 31 | ) 32 | 33 | func main() { 34 | flag.Parse() 35 | 36 | if *printConfigFlag { 37 | config.PrintDefaultConfig() 38 | os.Exit(0) 39 | } 40 | 41 | conf, err := config.LoadConfig(*configFlag) 42 | if err != nil { 43 | fmt.Fprintf(os.Stderr, "failed to open config %s: %v\n", "config.yml", err) 44 | os.Exit(1) 45 | } 46 | 47 | var lvl zerolog.Level 48 | switch conf.Common.LogLevel { 49 | case "debug": 50 | lvl = zerolog.DebugLevel 51 | case "info": 52 | lvl = zerolog.InfoLevel 53 | case "warn": 54 | lvl = zerolog.WarnLevel 55 | case "error": 56 | lvl = zerolog.ErrorLevel 57 | default: 58 | fmt.Fprintf(os.Stderr, "Unknown log_level '%s'", conf.Common.LogLevel) 59 | os.Exit(1) 60 | } 61 | // logger := zerolog.New(os.Stdout).Level(lvl).With().Timestamp().Logger() 62 | logger := zerolog.New(os.Stdout).Level(lvl).With().Timestamp().Logger().Output(zerolog.ConsoleWriter{Out: os.Stdout}) 63 | 64 | diffChannel := make(chan *hungryfox.Diff, 100) 65 | leakChannel := make(chan *hungryfox.Leak, 1) 66 | 67 | if *skipScan { 68 | stateManager := &filestate.StateManager{ 69 | Location: conf.Common.StateFile, 70 | } 71 | if err := stateManager.Start(); err != nil { 72 | logger.Error().Str("service", "state manager").Str("error", err.Error()).Msg("fail") 73 | os.Exit(1) 74 | } 75 | logger.Debug().Str("service", "state manager").Msg("started") 76 | 77 | logger.Debug().Str("service", "scan manager").Msg("start") 78 | scanManager := &scanmanager.ScanManager{ 79 | DiffChannel: diffChannel, 80 | Log: logger, 81 | StateManager: stateManager, 82 | } 83 | scanManager.SetConfig(conf) 84 | scanManager.DryRun() 85 | stateManager.Stop() 86 | os.Exit(0) 87 | } 88 | 89 | logger.Debug().Str("service", "leaks router").Msg("start") 90 | leakRouter := &router.LeaksRouter{ 91 | LeakChannel: leakChannel, 92 | Config: conf, 93 | Log: logger, 94 | } 95 | if err := leakRouter.Start(); err != nil { 96 | logger.Error().Str("service", "leaks router").Str("error", err.Error()).Msg("fail") 97 | os.Exit(1) 98 | } 99 | logger.Debug().Str("service", "leaks router").Msg("strated") 100 | 101 | logger.Debug().Str("service", "leaks searcher").Msg("start") 102 | 103 | numCPUs := runtime.NumCPU() - 1 104 | if numCPUs < 1 { 105 | numCPUs = 1 106 | } 107 | if conf.Common.Workers > 0 { 108 | numCPUs = conf.Common.Workers 109 | } 110 | leakSearcher := &searcher.Searcher{ 111 | Workers: numCPUs, 112 | DiffChannel: diffChannel, 113 | LeakChannel: leakChannel, 114 | Log: logger, 115 | } 116 | if err := leakSearcher.Start(conf); err != nil { 117 | logger.Error().Str("service", "leaks searcher").Str("error", err.Error()).Msg("fail") 118 | os.Exit(1) 119 | } 120 | logger.Debug().Str("service", "leaks searcher").Int("workers", numCPUs).Msg("started") 121 | 122 | logger.Debug().Str("service", "state manager").Msg("start") 123 | stateManager := &filestate.StateManager{ 124 | Location: conf.Common.StateFile, 125 | } 126 | if err := stateManager.Start(); err != nil { 127 | logger.Error().Str("service", "state manager").Str("error", err.Error()).Msg("fail") 128 | os.Exit(1) 129 | } 130 | logger.Debug().Str("service", "state manager").Msg("started") 131 | 132 | logger.Debug().Str("service", "scan manager").Msg("start") 133 | scanManager := &scanmanager.ScanManager{ 134 | DiffChannel: diffChannel, 135 | Log: logger, 136 | StateManager: stateManager, 137 | } 138 | if err := scanManager.Start(conf); err != nil { 139 | logger.Error().Str("service", "scan manager").Str("error", err.Error()).Msg("fail") 140 | os.Exit(1) 141 | } 142 | logger.Debug().Str("service", "scan manager").Msg("started") 143 | 144 | statusTicker := time.NewTicker(time.Second * 10) 145 | defer statusTicker.Stop() 146 | go func() { 147 | for range statusTicker.C { 148 | r := scanManager.Status() 149 | if r != nil { 150 | l := leakSearcher.Status(r.Location.URL) 151 | logger.Info().Int("leaks", l.LeaksFound).Int("leaks_filtred", l.LeaksFiltred).Str("duration", helpers.PrettyDuration(time.Since(r.Scan.StartTime))).Str("repo", r.Location.URL).Msg("scan") 152 | continue 153 | } 154 | } 155 | }() 156 | if *pprofFlag { 157 | go func() { 158 | if err := http.ListenAndServe(":6060", nil); err != nil { 159 | logger.Error().Str("error", err.Error()).Msg("can't start pprof") 160 | } 161 | }() 162 | } 163 | 164 | logger.Info().Str("version", version).Msg("started") 165 | 166 | signalChannel := make(chan os.Signal, 1) 167 | signal.Notify(signalChannel, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) 168 | 169 | for { 170 | s := <-signalChannel 171 | logger.Info().Str("signal", s.String()).Msg("received signal") 172 | if s != syscall.SIGHUP { 173 | break 174 | } 175 | 176 | newConf, err := config.LoadConfig("config.yml") 177 | if err != nil { 178 | logger.Error().Str("error", err.Error()).Msg("can't update config") 179 | continue 180 | } 181 | leakSearcher.Update(newConf) 182 | scanManager.SetConfig(newConf) 183 | logger.Info().Msg("settings reloaded") 184 | } 185 | 186 | if err := scanManager.Stop(); err != nil { 187 | logger.Error().Str("error", err.Error()).Str("service", "scan manager").Msg("can't stop") 188 | } 189 | logger.Debug().Str("service", "scan manager").Msg("stopped") 190 | 191 | if err := leakSearcher.Stop(); err != nil { 192 | logger.Error().Str("error", err.Error()).Str("service", "leak searcher").Msg("can't stop") 193 | } 194 | logger.Debug().Str("service", "leak searcher").Msg("stopped") 195 | 196 | if err := leakRouter.Stop(); err != nil { 197 | logger.Error().Str("error", err.Error()).Str("service", "leaks router").Msg("can't stop") 198 | } 199 | logger.Debug().Str("service", "leaks router").Msg("stopped") 200 | 201 | logger.Debug().Str("service", "state manager").Msg("stop") 202 | if err := stateManager.Stop(); err != nil { 203 | logger.Error().Str("error", err.Error()).Str("service", "state manager").Msg("can't stop") 204 | } 205 | 206 | logger.Info().Str("version", version).Msg("stopped") 207 | } 208 | -------------------------------------------------------------------------------- /searcher/searcher.go: -------------------------------------------------------------------------------- 1 | package searcher 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "path/filepath" 7 | "regexp" 8 | "strings" 9 | 10 | sync "github.com/sasha-s/go-deadlock" 11 | 12 | "github.com/AlexAkulov/hungryfox" 13 | "github.com/AlexAkulov/hungryfox/config" 14 | 15 | "github.com/rs/zerolog" 16 | "gopkg.in/tomb.v2" 17 | yaml "gopkg.in/yaml.v2" 18 | ) 19 | 20 | var matchAllRegex = regexp.MustCompile(".+") 21 | 22 | type patternType struct { 23 | Name string 24 | ContentRe *regexp.Regexp 25 | FileRe *regexp.Regexp 26 | } 27 | 28 | type RepoStats struct { 29 | LeaksFound int `json:"leaks_found"` 30 | LeaksFiltred int `json:"leaks_filtred"` 31 | } 32 | 33 | type Searcher struct { 34 | Workers int 35 | DiffChannel <-chan *hungryfox.Diff 36 | LeakChannel chan<- *hungryfox.Leak 37 | Log zerolog.Logger 38 | 39 | config *config.Config 40 | stats map[string]RepoStats 41 | statsMutex sync.RWMutex 42 | tomb tomb.Tomb 43 | patterns []patternType 44 | filters []patternType 45 | updateConfigChan chan *config.Config 46 | } 47 | 48 | func compilePatterns(configPatterns []config.Pattern) ([]patternType, error) { 49 | result := make([]patternType, 0) 50 | for _, configPattern := range configPatterns { 51 | p := patternType{ 52 | Name: configPattern.Name, 53 | FileRe: matchAllRegex, 54 | ContentRe: matchAllRegex, 55 | } 56 | if configPattern.File != "*" && configPattern.File != "" { 57 | var err error 58 | if p.FileRe, err = regexp.Compile(configPattern.File); err != nil { 59 | return nil, fmt.Errorf("can't compile pattern file regexp '%s' with: %v", configPattern.File, err) 60 | } 61 | } 62 | if configPattern.Content != "*" && configPattern.Content != "" { 63 | var err error 64 | if p.ContentRe, err = regexp.Compile(configPattern.Content); err != nil { 65 | return nil, fmt.Errorf("can't compile pattern content regexp '%s' with: %v", configPattern.Content, err) 66 | } 67 | } 68 | result = append(result, p) 69 | } 70 | return result, nil 71 | } 72 | 73 | func (s *Searcher) Update(conf *config.Config) { 74 | s.updateConfigChan <- conf 75 | } 76 | 77 | func (s *Searcher) Start(conf *config.Config) error { 78 | if err := s.updateConfig(conf); err != nil { 79 | return err 80 | } 81 | s.updateConfigChan = make(chan *config.Config, 1) 82 | 83 | if s.Workers < 1 { 84 | return fmt.Errorf("workers count can't be less 1") 85 | } 86 | 87 | s.stats = map[string]RepoStats{} 88 | for i := 0; i < s.Workers; i++ { 89 | s.tomb.Go(s.worker) 90 | } 91 | return nil 92 | } 93 | 94 | func (s *Searcher) worker() error { 95 | for { 96 | select { 97 | case newConf := <-s.updateConfigChan: 98 | err := s.updateConfig(newConf) 99 | if err != nil { 100 | s.Log.Error().Str("error", err.Error()).Msg("can't update patterns and filtres") 101 | } 102 | case <-s.tomb.Dying(): 103 | return nil 104 | case diff := <-s.DiffChannel: 105 | leaks := s.GetLeaks(*diff) 106 | filtredLeaks := 0 107 | for i := range leaks { 108 | if s.filterLeak(leaks[i]) { 109 | filtredLeaks++ 110 | continue 111 | } 112 | s.LeakChannel <- &leaks[i] 113 | } 114 | leaksCount := len(leaks) - filtredLeaks 115 | if leaksCount > 0 || filtredLeaks > 0 { 116 | s.statsMutex.Lock() 117 | repoStats, _ := s.stats[diff.RepoURL] 118 | repoStats.LeaksFiltred += filtredLeaks 119 | repoStats.LeaksFound += leaksCount 120 | s.stats[diff.RepoURL] = repoStats 121 | s.statsMutex.Unlock() 122 | } 123 | } 124 | } 125 | } 126 | 127 | func (s *Searcher) Stop() error { 128 | s.tomb.Kill(nil) 129 | return s.tomb.Wait() 130 | } 131 | 132 | func loadPatternsFromFile(file string) ([]patternType, error) { 133 | rawPatterns := []config.Pattern{} 134 | rawData, err := ioutil.ReadFile(file) 135 | if err != nil { 136 | return nil, fmt.Errorf("can't read file '%s' with: %v", file, err) 137 | } 138 | if err := yaml.Unmarshal(rawData, &rawPatterns); err != nil { 139 | return nil, fmt.Errorf("can't parse file '%s' with: %v", file, err) 140 | } 141 | result, err := compilePatterns(rawPatterns) 142 | if err != nil { 143 | return nil, fmt.Errorf("can't compile file '%s' with: %v", file, err) 144 | } 145 | return result, nil 146 | } 147 | 148 | func loadPatternsFromPath(path string) ([]patternType, error) { 149 | result := []patternType{} 150 | files, err := filepath.Glob(path) 151 | if err != nil { 152 | return nil, err 153 | } 154 | for _, file := range files { 155 | patterns, err := loadPatternsFromFile(file) 156 | if err != nil { 157 | return nil, err 158 | } 159 | result = append(result, patterns...) 160 | } 161 | return result, nil 162 | } 163 | 164 | func (s *Searcher) updateConfig(conf *config.Config) error { 165 | newCompiledPatterns, err := compilePatterns(conf.Patterns) 166 | if err != nil { 167 | return err 168 | } 169 | newCompiledFiltres, err := compilePatterns(conf.Filters) 170 | if err != nil { 171 | return err 172 | } 173 | 174 | if conf.Common.PatternsPath != "" { 175 | newFilePatterns, err := loadPatternsFromPath(conf.Common.PatternsPath) 176 | if err != nil { 177 | return err 178 | } 179 | newCompiledPatterns = append(newCompiledPatterns, newFilePatterns...) 180 | } 181 | 182 | if conf.Common.FiltresPath != "" { 183 | newFileFilters, err := loadPatternsFromPath(conf.Common.FiltresPath) 184 | if err != nil { 185 | return err 186 | } 187 | newCompiledFiltres = append(newCompiledFiltres, newFileFilters...) 188 | } 189 | s.patterns, s.filters = newCompiledPatterns, newCompiledFiltres 190 | s.Log.Info().Int("patterns", len(newCompiledPatterns)).Int("filters", len(newCompiledFiltres)).Msg("loaded") 191 | s.config = conf 192 | return nil 193 | } 194 | 195 | func (s *Searcher) Status(repoURL string) RepoStats { 196 | s.statsMutex.RLock() 197 | defer s.statsMutex.RUnlock() 198 | if repoStats, ok := s.stats[repoURL]; ok { 199 | return repoStats 200 | } 201 | return RepoStats{} 202 | } 203 | 204 | func (s *Searcher) GetLeaks(diff hungryfox.Diff) []hungryfox.Leak { 205 | leaks := make([]hungryfox.Leak, 0) 206 | lines := strings.Split(diff.Content, "\n") 207 | for _, line := range lines { 208 | for _, pattern := range s.patterns { 209 | repoFilePath := fmt.Sprintf("%s/%s", diff.RepoURL, diff.FilePath) 210 | if !pattern.FileRe.MatchString(repoFilePath) { 211 | continue 212 | } 213 | if pattern.ContentRe.MatchString(line) { 214 | if len(line) > 1024 { 215 | line = line[:1024] 216 | } 217 | leaks = append(leaks, hungryfox.Leak{ 218 | RepoPath: diff.RepoPath, 219 | FilePath: diff.FilePath, 220 | PatternName: pattern.Name, 221 | Regexp: pattern.ContentRe.String(), 222 | LeakString: line, 223 | CommitHash: diff.CommitHash, 224 | TimeStamp: diff.TimeStamp, 225 | CommitAuthor: diff.Author, 226 | CommitEmail: diff.AuthorEmail, 227 | RepoURL: diff.RepoURL, 228 | }) 229 | } 230 | } 231 | } 232 | return leaks 233 | } 234 | 235 | func (s *Searcher) filterLeak(leak hungryfox.Leak) bool { 236 | for _, filter := range s.filters { 237 | if filter.FileRe.MatchString(fmt.Sprintf("%s/%s", leak.RepoURL, leak.FilePath)) && filter.ContentRe.MatchString(leak.LeakString) { 238 | return true 239 | } 240 | } 241 | return false 242 | } 243 | -------------------------------------------------------------------------------- /hercules/repo.go: -------------------------------------------------------------------------------- 1 | package repo 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "path/filepath" 8 | "runtime" 9 | "strings" 10 | "time" 11 | 12 | "github.com/AlexAkulov/hungryfox" 13 | 14 | "gopkg.in/src-d/go-git.v4" 15 | "gopkg.in/src-d/go-git.v4/plumbing" 16 | "gopkg.in/src-d/go-git.v4/plumbing/format/diff" 17 | "gopkg.in/src-d/go-git.v4/plumbing/object" 18 | ) 19 | 20 | type Repo struct { 21 | DiffChannel chan<- *hungryfox.Diff 22 | HistoryPastLimit time.Time 23 | DataPath string 24 | RepoPath string 25 | CloneURL string 26 | URL string 27 | AllowUpdate bool 28 | repository *git.Repository 29 | scannedHash map[string]struct{} 30 | commitsTotal int 31 | commitsScanned int 32 | } 33 | 34 | func (r *Repo) GetProgress() int { 35 | if r.commitsTotal > 0 { 36 | return (r.commitsScanned / r.commitsTotal) * 1000 37 | } 38 | return -1 39 | } 40 | 41 | func (r *Repo) Close() error { 42 | r.repository = nil // ??? 43 | runtime.GC() // ??? 44 | return nil 45 | } 46 | 47 | func (r *Repo) SetRefs(refs []string) { 48 | r.scannedHash = map[string]struct{}{} 49 | for _, hash := range refs { 50 | r.scannedHash[hash] = struct{}{} 51 | } 52 | } 53 | 54 | func (r *Repo) GetRefs() (refsMap []string) { 55 | refsMap = []string{} 56 | if err := r.open(); err != nil { 57 | return 58 | } 59 | 60 | refs, err := r.repository.References() 61 | if err != nil { 62 | return 63 | } 64 | refs.ForEach(func(ref *plumbing.Reference) error { 65 | if ref.Hash().IsZero() { 66 | return nil 67 | } 68 | if strings.HasPrefix(ref.Name().String(), "refs/keep-around/") { 69 | return nil 70 | } 71 | refsMap = append(refsMap, ref.Hash().String()) 72 | return nil 73 | }) 74 | lastCommit := r.getLastCommit() 75 | if lastCommit != "" { 76 | refsMap = append(refsMap, lastCommit) 77 | } 78 | return 79 | } 80 | 81 | func (r *Repo) isChecked(commitHash string) bool { 82 | _, ok := r.scannedHash[commitHash] 83 | return ok 84 | } 85 | 86 | func (r *Repo) getLastCommit() string { 87 | oldWD, err := os.Getwd() 88 | if err != nil { 89 | return "" 90 | } 91 | if err := os.Chdir(r.fullRepoPath()); err != nil { 92 | return "" 93 | } 94 | // --topo-order??? 95 | out, err := exec.Command("git", "rev-list", "--all", "--remotes", "--date-order", "--max-count=1").Output() 96 | os.Chdir(oldWD) 97 | if err != nil { 98 | return "" 99 | } 100 | commits := strings.Split(string(out), "\n") 101 | if len(commits) > 0 { 102 | return commits[0] 103 | } 104 | return "" 105 | } 106 | 107 | func (r *Repo) getRevList() (result []*object.Commit, err error) { 108 | oldWD, err := os.Getwd() 109 | if err != nil { 110 | return nil, fmt.Errorf("error on get working dir: %v", err) 111 | } 112 | if err := os.Chdir(r.fullRepoPath()); err != nil { 113 | return nil, fmt.Errorf("error on change dir to %s: %v", r.fullRepoPath(), err) 114 | } 115 | // --topo-order??? 116 | out, err := exec.Command("git", "rev-list", "--all", "--remotes", "--date-order").Output() 117 | os.Chdir(oldWD) 118 | if err != nil { 119 | return nil, err 120 | } 121 | 122 | hashList := strings.Split(string(out), "\n") 123 | for _, commitHash := range hashList { 124 | commitHash = strings.TrimSpace(commitHash) 125 | if r.isChecked(commitHash) { 126 | break 127 | } 128 | commit, err := r.repository.CommitObject(plumbing.NewHash(commitHash)) 129 | if err != nil { 130 | continue 131 | } 132 | if commit.NumParents() > 1 { 133 | // ignore merge commit 134 | continue 135 | } 136 | result = append(result, commit) 137 | } 138 | 139 | r.commitsTotal = len(result) 140 | return result, nil 141 | } 142 | 143 | func (r *Repo) open() error { 144 | var err error 145 | if r.repository == nil { 146 | r.repository, err = git.PlainOpen(r.fullRepoPath()) 147 | } 148 | return err 149 | } 150 | 151 | // Scan - rt 152 | func (r *Repo) Scan() error { 153 | commits, err := r.getRevList() 154 | if err != nil { 155 | return err 156 | } 157 | for i, commit := range commits { 158 | r.commitsScanned = i + 1 159 | if commit.Committer.When.Before(r.HistoryPastLimit) { 160 | r.getAllChanges(commit, false) 161 | break 162 | } 163 | r.getCommitChanges(commit) 164 | } 165 | return nil 166 | } 167 | 168 | func (r *Repo) getAllChanges(commit *object.Commit, initCommit bool) error { 169 | tree, err := commit.Tree() 170 | if err != nil { 171 | return err 172 | } 173 | changes, err := object.DiffTree(nil, tree) 174 | if err != nil { 175 | return err 176 | } 177 | patch, err := changes.Patch() 178 | if err != nil { 179 | return err 180 | } 181 | for _, p := range patch.FilePatches() { 182 | _, f := p.Files() 183 | if f == nil || p.IsBinary() { 184 | continue 185 | } 186 | for _, chunk := range p.Chunks() { 187 | if chunk.Type() != diff.Add { 188 | continue 189 | } 190 | // TODO: Use blame for this 191 | author := "unknown" 192 | authorEmail := "unknown" 193 | 194 | if initCommit { 195 | author = commit.Author.Name 196 | authorEmail = commit.Author.Email 197 | } 198 | r.DiffChannel <- &hungryfox.Diff{ 199 | CommitHash: commit.Hash.String(), 200 | RepoURL: r.URL, 201 | RepoPath: r.RepoPath, 202 | FilePath: f.Path(), 203 | LineBegin: 0, // TODO: await https://github.com/src-d/go-git/issues/806 204 | Content: chunk.Content(), 205 | Author: author, 206 | AuthorEmail: authorEmail, 207 | TimeStamp: commit.Author.When, 208 | } 209 | } 210 | } 211 | return nil 212 | } 213 | 214 | func (r *Repo) getCommitChanges(commit *object.Commit) error { 215 | if commit == nil { 216 | return nil 217 | } 218 | defer func(){ 219 | if r := recover(); r != nil { 220 | fmt.Println("Recovered\n", r) 221 | } 222 | }() 223 | parrentCommit, err := commit.Parent(0) 224 | if err != nil { 225 | return r.getAllChanges(commit, true) 226 | } 227 | patch, err := parrentCommit.Patch(commit) 228 | if err != nil { 229 | return err 230 | } 231 | for _, p := range patch.FilePatches() { 232 | _, f := p.Files() 233 | if f == nil || p.IsBinary() { 234 | continue 235 | } 236 | for _, chunk := range p.Chunks() { 237 | if chunk.Type() != diff.Add { 238 | continue 239 | } 240 | r.DiffChannel <- &hungryfox.Diff{ 241 | CommitHash: commit.Hash.String(), 242 | RepoURL: r.URL, 243 | RepoPath: r.RepoPath, 244 | FilePath: f.Path(), 245 | LineBegin: 0, // TODO: await https://github.com/src-d/go-git/issues/806 246 | Content: chunk.Content(), 247 | Author: commit.Author.Name, 248 | AuthorEmail: commit.Author.Email, 249 | TimeStamp: commit.Author.When, 250 | } 251 | } 252 | } 253 | return nil 254 | } 255 | 256 | func (r *Repo) fullRepoPath() string { 257 | return filepath.Join(r.DataPath, r.RepoPath) 258 | } 259 | 260 | func (r *Repo) Open() error { 261 | if !r.AllowUpdate { 262 | return r.open() 263 | } 264 | if _, err := os.Stat(r.fullRepoPath()); os.IsNotExist(err) { 265 | if err := os.MkdirAll(r.fullRepoPath(), 0755); err != nil { 266 | return err 267 | } 268 | cloneOptions := &git.CloneOptions{ 269 | URL: r.CloneURL, 270 | NoCheckout: true, 271 | } 272 | repository, err := git.PlainClone(r.fullRepoPath(), false, cloneOptions) 273 | 274 | if err != nil { 275 | return err 276 | } 277 | r.repository = repository 278 | return nil 279 | } 280 | 281 | if err := r.open(); err != nil { 282 | return err 283 | } 284 | 285 | if err := r.repository.Fetch(&git.FetchOptions{Force: true}); err != nil && err != git.NoErrAlreadyUpToDate { 286 | return err 287 | } 288 | 289 | return nil 290 | } 291 | --------------------------------------------------------------------------------