├── .github ├── CODEOWNERS ├── secret_scanning.yml ├── scripts │ └── pr-issue-info │ │ ├── title-fail.md │ │ ├── issue-fail.md │ │ └── get_title_types.py ├── workflows │ ├── new-rules.yml │ ├── pr-title.yml │ ├── pr-labels.yml │ ├── cx-one-scan.yaml │ ├── validate-readme.yml │ ├── codecov.yaml │ ├── trivy-vulnerability-scan.yaml │ ├── security.yml │ ├── trivy-cache.yaml │ ├── pr-validation.yml │ ├── ci-projects.yaml │ ├── cesar.yaml │ └── run-projects.yaml ├── pull_request_template.md └── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── trivy.yaml ├── tools.go ├── internal └── resources │ └── scanner.go ├── tests └── testData │ ├── input │ ├── secret_at_end.txt │ ├── secret_at_end_with_newline.txt │ └── multi_line_secret.txt │ └── expectedReport │ ├── secret_at_end_report.json │ ├── secret_at_end_with_newline_report.json │ └── multi_line_secret_report.json ├── lib ├── config │ └── config.go ├── utils │ ├── channels.go │ ├── test_utils.go │ ├── logger.go │ ├── http.go │ ├── flags.go │ └── http_test.go ├── reporting │ ├── json.go │ ├── sarif_test.go │ ├── yaml.go │ ├── report.go │ └── sarif.go └── secrets │ ├── secret_test.go │ └── secret.go ├── pkg ├── testData │ ├── secrets │ │ ├── github-pat.txt │ │ └── jwt.txt │ ├── expectedReportWithIgnoredRule.json │ ├── expectedReport.json │ ├── expectedReportWithIgnoredResults.json │ └── expectedReportWithValidation.json ├── scanner.go └── scan.go ├── ignore.openvex ├── trivy-whitelist.openvex ├── engine ├── rules │ ├── rule.go │ ├── github.go │ ├── clojars.go │ ├── plaid.go │ ├── privateKey.go │ ├── authenticated_url.go │ ├── vault.go │ ├── atlassian.go │ ├── aws.go │ ├── gitlab.go │ ├── sumologic.go │ ├── hardcodedPassword.go │ ├── 1password.go │ ├── utils.go │ └── generic-key.go ├── extra │ ├── mutex.go │ ├── extra.go │ └── extra_test.go ├── validation │ ├── client.go │ ├── github.go │ ├── gitlab.go │ ├── validator.go │ ├── pairs.go │ ├── gcp.go │ └── alibaba.go ├── linecontent │ ├── linecontent.go │ └── linecontent_test.go ├── semaphore │ ├── semaphore_mock.go │ ├── semaphore_test.go │ └── semaphore.go ├── config.go ├── chunk │ ├── chunk_mock.go │ └── chunk_test.go ├── plugins_mock_test.go └── score │ └── score.go ├── .pre-commit-config.yaml ├── .gitignore ├── main.go ├── cmd ├── enum_flags_test.go ├── enum_flags.go ├── exit_handler.go ├── main_test.go ├── plugins_mock_test.go ├── exit_handler_test.go ├── config.go └── main.go ├── CONTRIBUTING.md ├── .ci ├── update-readme.sh └── check_new_rules.go ├── Dockerfile ├── plugins ├── plugins.go ├── filesystem.go ├── paligo_test.go └── filesystem_test.go ├── benches ├── README.md └── test_data.go ├── .golangci.yml ├── Makefile └── go.mod /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @Checkmarx/2ms-dev 2 | -------------------------------------------------------------------------------- /.github/secret_scanning.yml: -------------------------------------------------------------------------------- 1 | paths-ignore: 2 | - "**/*_test.go" -------------------------------------------------------------------------------- /trivy.yaml: -------------------------------------------------------------------------------- 1 | vulnerability: 2 | vex: 3 | - ignore.openvex 4 | -------------------------------------------------------------------------------- /tools.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | _ "golang.org/x/net/html" 5 | ) 6 | -------------------------------------------------------------------------------- /internal/resources/scanner.go: -------------------------------------------------------------------------------- 1 | package resources 2 | 3 | type ScanConfig struct { 4 | IgnoreResultIds []string 5 | IgnoreRules []string 6 | WithValidation bool 7 | PluginName string 8 | } 9 | -------------------------------------------------------------------------------- /tests/testData/input/secret_at_end.txt: -------------------------------------------------------------------------------- 1 | `"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`, 2 | `"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",` -------------------------------------------------------------------------------- /tests/testData/input/secret_at_end_with_newline.txt: -------------------------------------------------------------------------------- 1 | `"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`, 2 | `"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",` 3 | -------------------------------------------------------------------------------- /lib/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type Config struct { 4 | Name string 5 | Version string 6 | } 7 | 8 | func LoadConfig(name, version string) *Config { 9 | return &Config{Name: name, Version: version} 10 | } 11 | -------------------------------------------------------------------------------- /.github/scripts/pr-issue-info/title-fail.md: -------------------------------------------------------------------------------- 1 | # Pull Request Title Guideline 2 | 3 | Please, follow the guideline for a pull request title: 4 | 5 | `(): ` 6 | 7 | Thank you! 8 | *2ms Team* 9 | -------------------------------------------------------------------------------- /pkg/testData/secrets/github-pat.txt: -------------------------------------------------------------------------------- 1 | TextExampleghp_1234567890abcdefghijklmnopqrstuvwxyzTextExampleghp_abcdefghijklmnopqrstuvwxyz1234567890TextExample 2 | Text_Example = ghp_9876543210zyxwvutsrqponmlkjihgfedcba 3 | -------------------------------------------------------------------------------- /lib/utils/channels.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "sync" 4 | 5 | func BindChannels[T any](source <-chan T, dest chan<- T, wg *sync.WaitGroup) { 6 | if wg != nil { 7 | defer wg.Done() 8 | } 9 | for item := range source { 10 | dest <- item 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /ignore.openvex: -------------------------------------------------------------------------------- 1 | { 2 | "@context": "https://openvex.dev/ns/v0.2.0", 3 | "@id": "https://openvex.dev/docs/public/vex-d906fd067a1c6d14702845312a6bddd365153fe7c9eb651e6d7b89dad2bc9d22", 4 | "author": "Monica Casanova", 5 | "timestamp": "2024-04-11T16:02:39.0223474+01:00", 6 | "version": 1, 7 | "statements": [ 8 | ] 9 | } -------------------------------------------------------------------------------- /trivy-whitelist.openvex: -------------------------------------------------------------------------------- 1 | { 2 | "@context": "https://openvex.dev/ns", 3 | "@id": "https://openvex.dev/docs/public/vex-2e67563e128250cbcb3e98930df948dd053e43271d70dc50cfa22d57e03fe96f", 4 | "timestamp": "2024-05-08T16:00:16.853479631-06:00", 5 | "version": 1, 6 | "author":"Omer fainshtein", 7 | "statements": [ 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /engine/rules/rule.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "github.com/zricethezav/gitleaks/v8/config" 5 | ) 6 | 7 | type ScoreParameters struct { 8 | Category RuleCategory 9 | RuleType uint8 10 | } 11 | 12 | type Rule struct { 13 | Rule config.Rule 14 | Tags []string 15 | ScoreParameters ScoreParameters 16 | } 17 | -------------------------------------------------------------------------------- /.github/scripts/pr-issue-info/issue-fail.md: -------------------------------------------------------------------------------- 1 | # Issue Title Guidelines 2 | 3 | Please, follow the guideline for an issue title: 4 | 5 | For **bug**: 6 | 7 | `bug(<scope>): <title starting with lowercase letter>` 8 | 9 | For **feature request**: 10 | 11 | `feat(<scope>): <title starting with lowercase letter>` 12 | 13 | Thank you! 14 | *2ms Team* 15 | -------------------------------------------------------------------------------- /lib/reporting/json.go: -------------------------------------------------------------------------------- 1 | package reporting 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | ) 7 | 8 | func writeJson(report *Report) (string, error) { 9 | jsonReport, err := json.MarshalIndent(report, "", " ") 10 | if err != nil { 11 | return "", fmt.Errorf("failed to create Json report with error: %v", err) 12 | } 13 | 14 | return string(jsonReport), nil 15 | } 16 | -------------------------------------------------------------------------------- /.github/scripts/pr-issue-info/get_title_types.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | import os 3 | 4 | def yaml_to_regex(yaml_file): 5 | with open(yaml_file, 'r') as f: 6 | data = yaml.safe_load(f) 7 | regex = '|'.join(data) 8 | print(f"^({regex})\([a-z]+\): [a-z]") 9 | 10 | 11 | if __name__ == "__main__": 12 | file_path = os.environ['FILE_PATH'] 13 | yaml_to_regex(file_path) -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: local 3 | hooks: 4 | - id: make-check 5 | name: make check 6 | description: Run project checks, gofmt, golangci-lint, tests and coverage 7 | entry: make check 8 | language: system 9 | pass_filenames: false 10 | types: [ go ] # Only run when Go files change 11 | stages: [ pre-push ] # Explicitly run at push time 12 | -------------------------------------------------------------------------------- /engine/extra/mutex.go: -------------------------------------------------------------------------------- 1 | package extra 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | type NamedMutex struct { 8 | mutexes sync.Map 9 | } 10 | 11 | func (n *NamedMutex) Lock(key string) { 12 | mu, _ := n.mutexes.LoadOrStore(key, &sync.Mutex{}) 13 | mu.(*sync.Mutex).Lock() 14 | } 15 | 16 | func (n *NamedMutex) Unlock(key string) { 17 | mu, ok := n.mutexes.Load(key) 18 | if ok { 19 | mu.(*sync.Mutex).Unlock() 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /pkg/testData/secrets/jwt.txt: -------------------------------------------------------------------------------- 1 | TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1 TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 TextExample 2 | Text_Example = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | .idea 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | vendor/ 16 | 17 | # IDE directories and files 18 | .vscode/ 19 | 20 | dist 21 | 2ms 22 | 23 | cover.out.tmp -------------------------------------------------------------------------------- /engine/rules/github.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/zricethezav/gitleaks/v8/config" 7 | ) 8 | 9 | func GitHubApp() *config.Rule { 10 | return &config.Rule{ 11 | Description: "Identified a GitHub App Token, which may compromise GitHub application integrations and source code security.", 12 | RuleID: "github-app-token", 13 | Regex: regexp.MustCompile(`(?:ghu|ghs)_[0-9a-zA-Z]{36}`), 14 | Keywords: []string{"ghu_", "ghs_"}, 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /engine/rules/clojars.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/zricethezav/gitleaks/v8/config" 7 | ) 8 | 9 | func Clojars() *config.Rule { 10 | return &config.Rule{ 11 | Description: "Uncovered a possible Clojars API token, risking unauthorized access to Clojure libraries and potential code manipulation.", 12 | RuleID: "clojars-api-token", 13 | Regex: regexp.MustCompile(`(?i)CLOJARS_[a-z0-9]{60}`), 14 | Keywords: []string{"clojars"}, 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /engine/rules/plaid.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "github.com/zricethezav/gitleaks/v8/config" 5 | ) 6 | 7 | func PlaidAccessID() *config.Rule { 8 | return &config.Rule{ 9 | RuleID: "plaid-client-id", 10 | Description: "Uncovered a Plaid Client ID, which could lead to unauthorized financial service integrations and data breaches.", 11 | Regex: generateSemiGenericRegex([]string{"plaid"}, alphaNumeric("24"), true), 12 | 13 | Entropy: 3.0, 14 | Keywords: []string{ 15 | "plaid", 16 | }, 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /.github/workflows/new-rules.yml: -------------------------------------------------------------------------------- 1 | name: New Rules from Gitleaks 2 | 3 | on: 4 | workflow_dispatch: 5 | schedule: 6 | - cron: "0 2 * * 6" # At 02:00 on Saturday 7 | 8 | jobs: 9 | update_secrets: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 13 | - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 14 | with: 15 | go-version-file: go.mod 16 | - name: Check Gitleaks new rules 17 | run: go run .ci/check_new_rules.go 18 | -------------------------------------------------------------------------------- /engine/rules/privateKey.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/zricethezav/gitleaks/v8/config" 7 | ) 8 | 9 | func PrivateKey() *config.Rule { 10 | return &config.Rule{ 11 | RuleID: "private-key", 12 | Description: "Identified a Private Key, which may compromise cryptographic security and sensitive data encryption.", 13 | Regex: regexp.MustCompile(`(?i)-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----[\s\S-]{64,}?KEY(?: BLOCK)?-----`), //nolint:gocritic,lll 14 | Keywords: []string{"-----BEGIN"}, 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /engine/rules/authenticated_url.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/zricethezav/gitleaks/v8/config" 7 | ) 8 | 9 | func AuthenticatedURL() *config.Rule { 10 | regex := regexp.MustCompile(`://(\w+:\w\S+)@\S+\.\S+`) 11 | return &config.Rule{ 12 | Description: "Identify username:password inside URLS", 13 | RuleID: "authenticated-url", 14 | Regex: regex, 15 | Keywords: []string{"://"}, 16 | SecretGroup: 1, 17 | Allowlists: []*config.Allowlist{ 18 | { 19 | StopWords: []string{"password", "pass"}, 20 | }, 21 | }, 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | <!-- 2 | Thanks for contributing to 2ms by offering a pull request. 3 | --> 4 | 5 | Closes # 6 | 7 | **Proposed Changes** 8 | 9 | <!-- 10 | Please describe the big picture of your changes here. If it fixes a bug or resolves a feature request, be sure to link to that issue. 11 | --> 12 | 13 | **Checklist** 14 | 15 | - [ ] I covered my changes with tests. 16 | - [ ] I Updated the documentation that is affected by my changes: 17 | - [ ] Change in the CLI arguments 18 | - [ ] Change in the configuration file 19 | 20 | I submit this contribution under the Apache-2.0 license. 21 | -------------------------------------------------------------------------------- /engine/validation/client.go: -------------------------------------------------------------------------------- 1 | package validation 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | ) 7 | 8 | func sendValidationRequest(endpoint, authorization string) (*http.Response, error) { 9 | req, err := http.NewRequestWithContext(context.Background(), "GET", endpoint, http.NoBody) 10 | if err != nil { 11 | return nil, err 12 | } 13 | req.Header.Set("Authorization", authorization) 14 | 15 | // TODO: do not recreate this client for each request 16 | client := &http.Client{} 17 | resp, err := client.Do(req) 18 | if err != nil { 19 | return nil, err 20 | } 21 | 22 | return resp, nil 23 | } 24 | -------------------------------------------------------------------------------- /.github/workflows/pr-title.yml: -------------------------------------------------------------------------------- 1 | name: Validate Conventional Commit title 2 | 3 | on: 4 | pull_request: 5 | types: [opened, edited, synchronize, reopened] 6 | 7 | jobs: 8 | validate: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: install commitlint 12 | run: npm install -g @commitlint/cli @commitlint/config-conventional 13 | - name: config commitlint 14 | run: | 15 | echo "module.exports = {extends: ['@commitlint/config-conventional']}" > commitlint.config.js 16 | - name: validate PR title 17 | run: | 18 | echo ${{ github.event.pull_request.title }} | commitlint 19 | -------------------------------------------------------------------------------- /engine/validation/github.go: -------------------------------------------------------------------------------- 1 | package validation 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | 7 | "github.com/checkmarx/2ms/v4/lib/secrets" 8 | "github.com/rs/zerolog/log" 9 | ) 10 | 11 | func validateGithub(s *secrets.Secret) (secrets.ValidationResult, string) { 12 | const githubURL = "https://api.github.com/" 13 | 14 | resp, err := sendValidationRequest(githubURL, fmt.Sprintf("token %s", s.Value)) 15 | 16 | if err != nil { 17 | log.Warn().Err(err).Msg("Failed to validate secret") 18 | return secrets.UnknownResult, "" 19 | } 20 | defer resp.Body.Close() 21 | 22 | if resp.StatusCode == http.StatusOK { 23 | return secrets.ValidResult, "" 24 | } 25 | return secrets.InvalidResult, "" 26 | } 27 | -------------------------------------------------------------------------------- /tests/testData/input/multi_line_secret.txt: -------------------------------------------------------------------------------- 1 | `"client_id" : "0afae57f3ccfd9d7f5767067bc48b30f719e271ba470488056e37ab35d4b6506"`, 2 | `"client_secret" : "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde",` 3 | -----BEGIN RSA PRIVATE KEY----- MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu KUpRKfFLfRYC9AIKjbJTWit+Cq 4 | vjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp79mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs /5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00 -----END RSA PRIVATE KEY----- 5 | -------------------------------------------------------------------------------- /.github/workflows/pr-labels.yml: -------------------------------------------------------------------------------- 1 | name: PR Labels 2 | 3 | on: 4 | pull_request_target: 5 | types: [opened] 6 | 7 | jobs: 8 | mark_as_community: 9 | runs-on: ubuntu-latest 10 | permissions: 11 | pull-requests: write 12 | steps: 13 | - name: Mark as Community if PR is from a fork 14 | if: github.event.pull_request.head.repo.full_name != github.repository 15 | uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 16 | with: 17 | script: | 18 | github.rest.issues.addLabels({ 19 | issue_number: context.issue.number, 20 | owner: context.repo.owner, 21 | repo: context.repo.repo, 22 | labels: ['Community'] 23 | }) 24 | -------------------------------------------------------------------------------- /lib/utils/test_utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | // normalizeReportData recursively traverses the report data and removes any carriage return characters. 10 | func NormalizeReportData(data interface{}) (interface{}, error) { 11 | bytes, err := json.Marshal(data) 12 | if err != nil { 13 | return nil, fmt.Errorf("failed to marshal data: %w", err) 14 | } 15 | 16 | jsonStr := string(bytes) 17 | jsonStr = strings.ReplaceAll(jsonStr, "\\r", "") 18 | 19 | // Unmarshal back to a Go data structure 20 | var result interface{} 21 | err = json.Unmarshal([]byte(jsonStr), &result) 22 | if err != nil { 23 | return nil, fmt.Errorf("failed to unmarshal data: %w", err) 24 | } 25 | 26 | return result, nil 27 | } 28 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea/feature for this project 4 | title: 'feat(<scope>): <title starting with lowercase letter>' 5 | labels: community, feature request 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when \[...\]) 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "os/signal" 6 | 7 | "github.com/checkmarx/2ms/v4/cmd" 8 | "github.com/checkmarx/2ms/v4/lib/utils" 9 | "github.com/rs/zerolog" 10 | "github.com/rs/zerolog/log" 11 | ) 12 | 13 | func main() { 14 | zerolog.SetGlobalLevel(zerolog.InfoLevel) 15 | log.Logger = utils.CreateLogger(zerolog.InfoLevel) 16 | 17 | // this block sets up a go routine to listen for an interrupt signal 18 | // which will immediately exit gitleaks 19 | stopChan := make(chan os.Signal, 1) 20 | signal.Notify(stopChan, os.Interrupt) 21 | go listenForInterrupt(stopChan) 22 | 23 | cmd.Exit(cmd.Execute()) 24 | } 25 | 26 | func listenForInterrupt(stopScan chan os.Signal) { 27 | <-stopScan 28 | log.Error().Msg("Interrupt signal received. Exiting...") 29 | os.Exit(1) 30 | } 31 | -------------------------------------------------------------------------------- /cmd/enum_flags_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func TestIgnoreOnExitSet(t *testing.T) { 10 | tests := []struct { 11 | input string 12 | expected ignoreOnExit 13 | err bool 14 | }{ 15 | {"none", ignoreOnExitNone, false}, 16 | {"all", ignoreOnExitAll, false}, 17 | {"results", ignoreOnExitResults, false}, 18 | {"errors", ignoreOnExitErrors, false}, 19 | {"invalid", "", true}, 20 | } 21 | 22 | for _, tt := range tests { 23 | t.Run(fmt.Sprintf("Set(%s)", tt.input), func(t *testing.T) { 24 | var i ignoreOnExit 25 | err := i.Set(tt.input) 26 | if tt.err { 27 | assert.Error(t, err) 28 | } else { 29 | assert.NoError(t, err) 30 | assert.Equal(t, tt.expected, i) 31 | } 32 | }) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /tests/testData/expectedReport/secret_at_end_report.json: -------------------------------------------------------------------------------- 1 | { 2 | "totalItemsScanned": 1, 3 | "totalSecretsFound": 1, 4 | "results": { 5 | "80fa9bfa31b3488b04b07983e636651b38bb1e11": [ 6 | { 7 | "id": "80fa9bfa31b3488b04b07983e636651b38bb1e11", 8 | "source": "testData/input/secret_at_end.txt", 9 | "ruleId": "generic-api-key", 10 | "startLine": 2, 11 | "endLine": 2, 12 | "lineContent": "\t\t`\"client_secret\" : \"6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde\",`", 13 | "startColumn": 5, 14 | "endColumn": 87, 15 | "value": "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde", 16 | "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", 17 | "cvssScore": 8.2 18 | } 19 | ] 20 | } 21 | } -------------------------------------------------------------------------------- /engine/rules/vault.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "github.com/zricethezav/gitleaks/v8/config" 5 | ) 6 | 7 | // Using this local version because newer versions of gitleaks have an entropy value, which was set as too high 8 | // It's here as prevention in case a newer version of gitleaks starts getting used and causes issues on this rule 9 | // If gitleaks is updated on 2ms and the new version of this rule has entropy, set it to 3.0 10 | func VaultServiceToken() *config.Rule { 11 | // define rule 12 | return &config.Rule{ 13 | Description: "Identified a Vault Service Token, potentially compromising infrastructure security and access to sensitive credentials.", 14 | RuleID: "vault-service-token", 15 | Regex: generateUniqueTokenRegex(`hvs\.[a-z0-9_-]{90,100}`, true), 16 | Keywords: []string{"hvs"}, 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /tests/testData/expectedReport/secret_at_end_with_newline_report.json: -------------------------------------------------------------------------------- 1 | { 2 | "totalItemsScanned": 1, 3 | "totalSecretsFound": 1, 4 | "results": { 5 | "c7510bd9bcfa7887912dd28bd57aab89be736acd": [ 6 | { 7 | "id": "c7510bd9bcfa7887912dd28bd57aab89be736acd", 8 | "source": "testData/input/secret_at_end_with_newline.txt", 9 | "ruleId": "generic-api-key", 10 | "startLine": 2, 11 | "endLine": 2, 12 | "lineContent": "\t\t`\"client_secret\" : \"6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde\",`", 13 | "startColumn": 5, 14 | "endColumn": 87, 15 | "value": "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde", 16 | "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", 17 | "cvssScore": 8.2 18 | } 19 | ] 20 | } 21 | } -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Welcome to the 2ms club! 2 | 3 | > [!NOTE] 4 | > This is the first version of the document, we will rewrite it on the fly. 5 | 6 | ## Test 7 | 8 | Along with the regular unit tests, we also have a set of other tests: 9 | 10 | - `tests/cli` - e2e tests that build the CLI, run it, and check the output. 11 | To skip these tests, run `go test -short ./...`. 12 | - `tests/lint` - linter, to verify we are not using our forbidden functions (for example, using `fmt.Print` instead of `log.Info`) 13 | - `.ci/check_new_rules.go` - compares the list of rules in the [latest _gitleaks_ release](https://github.com/gitleaks/gitleaks/releases/latest) with our list of rules, and fails if there are rules in the release that are not in our list. 14 | - `.ci/update-readme.sh` - auto update the `help` message in the [README.md](README.md#command-line-interface) file. 15 | -------------------------------------------------------------------------------- /engine/rules/atlassian.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils" 5 | "github.com/zricethezav/gitleaks/v8/config" 6 | ) 7 | 8 | func Atlassian() *config.Rule { 9 | return &config.Rule{ 10 | Description: `Detected an Atlassian API token, 11 | posing a threat to project management and 12 | collaboration tool security and data confidentiality.`, 13 | RuleID: "atlassian-api-token", 14 | Regex: utils.MergeRegexps( 15 | utils.GenerateSemiGenericRegex( 16 | []string{"(?-i:ATLASSIAN|[Aa]tlassian)", "(?-i:CONFLUENCE|[Cc]onfluence)", "(?-i:JIRA|[Jj]ira)"}, 17 | `[a-z0-9]{20}[a-f0-9]{4}`, // The last 4 characters are an MD5 hash. 18 | true, 19 | ), 20 | utils.GenerateUniqueTokenRegex(`ATATT3[A-Za-z0-9_\-=]{186}`, false), 21 | ), 22 | Entropy: 3.5, 23 | Keywords: []string{"atlassian", "confluence", "jira", "atatt3"}, 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /cmd/enum_flags.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | ) 7 | 8 | type ignoreOnExit string 9 | 10 | const ( 11 | ignoreOnExitNone ignoreOnExit = "none" 12 | ignoreOnExitAll ignoreOnExit = "all" 13 | ignoreOnExitResults ignoreOnExit = "results" 14 | ignoreOnExitErrors ignoreOnExit = "errors" 15 | ) 16 | 17 | // verify that ignoreOnExit implements flag.Value interface 18 | // https://github.com/uber-go/guide/blob/master/style.md#verify-interface-compliance 19 | var _ flag.Value = (*ignoreOnExit)(nil) 20 | 21 | func (i *ignoreOnExit) String() string { 22 | return string(*i) 23 | } 24 | 25 | func (i *ignoreOnExit) Set(value string) error { 26 | switch value { 27 | case "none", "all", "results", "errors": 28 | *i = ignoreOnExit(value) 29 | return nil 30 | default: 31 | return fmt.Errorf("invalid value %s", value) 32 | } 33 | } 34 | 35 | func (i *ignoreOnExit) Type() string { 36 | return "ignoreOnExit" 37 | } 38 | -------------------------------------------------------------------------------- /.ci/update-readme.sh: -------------------------------------------------------------------------------- 1 | update_readme() { 2 | output_file=$1 3 | placeholder_name=$2 4 | target_file=$3 5 | 6 | sed -i "/<!-- $placeholder_name:start -->/,/<!-- $placeholder_name:end -->/{ 7 | /<!-- $placeholder_name:start -->/{ 8 | p 9 | r $output_file 10 | } 11 | /<!-- $placeholder_name:end -->/!d 12 | }" $target_file 13 | } 14 | 15 | # Update the README with the help message 16 | help_message=$(go run .) 17 | 18 | echo "" >output.txt 19 | echo '```text' >>output.txt 20 | echo "$help_message" >>output.txt 21 | echo '```' >>output.txt 22 | echo "" >>output.txt 23 | update_readme "output.txt" "command-line" "README.md" 24 | rm output.txt 25 | 26 | go run . rules | awk 'BEGIN{FS = " *"}{print "| " $1 " | " $2 " | " $3 " | " $4 " |";}' >output.txt 27 | update_readme "output.txt" "table" "./docs/list-of-rules.md" 28 | rm output.txt 29 | 30 | git --no-pager diff README.md ./docs/list-of-rules.md 31 | -------------------------------------------------------------------------------- /engine/rules/aws.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/zricethezav/gitleaks/v8/config" 7 | ) 8 | 9 | func AWS() *config.Rule { 10 | return &config.Rule{ 11 | RuleID: "aws-access-token", 12 | Description: "Identified a pattern that may indicate AWS credentials, risking unauthorized cloud resource access and data breaches on AWS platforms.", //nolint:lll 13 | Regex: regexp.MustCompile(`\b((?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z2-7]{16})\b`), 14 | Entropy: 3, 15 | Keywords: []string{ 16 | // https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids 17 | "A3T", // todo: might not be a valid AWS token 18 | "AKIA", // Access key 19 | "ASIA", // Temporary (AWS STS) access key 20 | "ABIA", // AWS STS service bearer token 21 | "ACCA", // Context-specific credential 22 | }, 23 | Allowlists: []*config.Allowlist{ 24 | { 25 | Regexes: []*regexp.Regexp{ 26 | regexp.MustCompile(`.+EXAMPLE$`), 27 | }, 28 | }, 29 | }, 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /engine/rules/gitlab.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/zricethezav/gitleaks/v8/config" 7 | ) 8 | 9 | func GitlabPatRoutable() *config.Rule { 10 | return &config.Rule{ 11 | RuleID: "gitlab-pat-routable", 12 | Description: "Identified a GitLab Personal Access Token (routable), risking unauthorized access to GitLab repositories and codebase exposure.", //nolint:lll 13 | Regex: regexp.MustCompile(`\bglpat-[0-9a-zA-Z_-]{27,300}\.[0-9a-z]{2}[0-9a-z]{7}\b`), 14 | Entropy: 4, 15 | Keywords: []string{"glpat-"}, 16 | } 17 | } 18 | 19 | func GitlabRunnerAuthenticationTokenRoutable() *config.Rule { 20 | return &config.Rule{ 21 | RuleID: "gitlab-runner-authentication-token-routable", 22 | Description: "Discovered a GitLab Runner Authentication Token (Routable), posing a risk to CI/CD pipeline integrity and unauthorized access.", //nolint:lll 23 | Regex: regexp.MustCompile(`\bglrt-t\d_[0-9a-zA-Z_\-]{27,300}\.[0-9a-z]{2}[0-9a-z]{7}\b`), 24 | Entropy: 4, 25 | Keywords: []string{"glrt-"}, 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /engine/rules/sumologic.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "github.com/zricethezav/gitleaks/v8/cmd/generate/config/utils" 5 | "github.com/zricethezav/gitleaks/v8/config" 6 | ) 7 | 8 | func SumoLogicAccessID() *config.Rule { 9 | return &config.Rule{ 10 | RuleID: "sumologic-access-id", 11 | Description: "Discovered a SumoLogic Access ID, potentially compromising log management services and data analytics integrity.", 12 | Regex: utils.GenerateSemiGenericRegex([]string{"(?-i:[Ss]umo|SUMO)"}, "su[a-zA-Z0-9]{12}", false), 13 | Entropy: 3, 14 | Keywords: []string{ 15 | "sumo", 16 | }, 17 | } 18 | } 19 | 20 | func SumoLogicAccessToken() *config.Rule { 21 | return &config.Rule{ 22 | RuleID: "sumologic-access-token", 23 | Description: "Uncovered a SumoLogic Access Token, which could lead to unauthorized access to log data and analytics insights.", 24 | Regex: utils.GenerateSemiGenericRegex([]string{"(?-i:[Ss]umo|SUMO)"}, utils.AlphaNumeric("64"), true), 25 | Entropy: 3, 26 | Keywords: []string{ 27 | "sumo", 28 | }, 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /.github/workflows/cx-one-scan.yaml: -------------------------------------------------------------------------------- 1 | name: cx-one-scan 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | push: 7 | branches: 8 | - master 9 | schedule: 10 | - cron: '00 7 * * *' 11 | 12 | jobs: 13 | cx-one-scan: 14 | name: cx-one-scan 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout 18 | uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 19 | - name: Checkmarx One CLI Action 20 | uses: checkmarx/ast-github-action@86e9ae570a811f9a1fb85903647a307aa3bf6253 # 2.0.44 21 | with: 22 | base_uri: ${{ secrets.AST_RND_SCANS_BASE_URI }} 23 | cx_tenant: ${{ secrets.AST_RND_SCANS_TENANT }} 24 | cx_client_id: ${{ secrets.AST_RND_SCANS_CLIENT_ID }} 25 | cx_client_secret: ${{ secrets.AST_RND_SCANS_CLIENT_SECRET }} 26 | additional_params: --tags scs --threshold "sast-critical=1; sast-high=1; sast-medium=1; sast-low=1; sca-critical=1; sca-high=1; sca-medium=1; sca-low=1; iac-security-critical=1; iac-security-high=1; iac-security-medium=1;iac-security-low=1" 27 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # kics-scan disable=b03a748a-542d-44f4-bb86-9199ab4fd2d5,fd54f200-402c-4333-a5a4-36ef6709af2f 2 | # disable kics Healthcheck result 3 | # and "Missing User Instruction" since 2ms container is stopped after scan 4 | 5 | # Builder image 6 | FROM checkmarx/go:1.25.3-r0-b47cbbc1194cd0@sha256:b47cbbc1194cd0d801fe7739fca12091d610117b0d30c32b52fc900217a0821a AS builder 7 | 8 | WORKDIR /app 9 | 10 | #Copy go mod and sum files 11 | COPY go.mod . 12 | COPY go.sum . 13 | 14 | # Get dependencies - will also be cached if we won't change mod/sum 15 | RUN go mod download 16 | 17 | # COPY the source code as the last step 18 | COPY . . 19 | 20 | RUN GOOS=linux GOARCH=amd64 go build -buildvcs=false -ldflags="-s -w" -a -o /app/2ms . 21 | 22 | # Runtime image 23 | FROM checkmarx/git:2.49.0-r2-d7ebbe7c56dc47@sha256:d7ebbe7c56dc478c08aba611c35b30689090d28605d83130ce4d1e15a84f0389 24 | 25 | WORKDIR /app 26 | 27 | RUN chown -R 65532:65532 /app 28 | 29 | USER 65532 30 | 31 | COPY --from=builder /app/2ms /app/2ms 32 | 33 | RUN git config --global --add safe.directory /repo 34 | 35 | ENTRYPOINT [ "/app/2ms" ] -------------------------------------------------------------------------------- /lib/utils/logger.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "io" 5 | "os" 6 | 7 | "github.com/rs/zerolog" 8 | ) 9 | 10 | type SpecificLevelWriter struct { 11 | io.Writer 12 | Levels []zerolog.Level 13 | } 14 | 15 | func (w SpecificLevelWriter) WriteLevel(level zerolog.Level, p []byte) (int, error) { 16 | for _, l := range w.Levels { 17 | if l == level { 18 | return w.Write(p) 19 | } 20 | } 21 | return len(p), nil 22 | } 23 | 24 | func CreateLogger(minimumLevel zerolog.Level) zerolog.Logger { 25 | writer := zerolog.MultiLevelWriter( 26 | SpecificLevelWriter{ 27 | Writer: zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: "15:04:05", NoColor: true}, 28 | Levels: []zerolog.Level{ 29 | zerolog.DebugLevel, zerolog.InfoLevel, zerolog.WarnLevel, 30 | }, 31 | }, 32 | SpecificLevelWriter{ 33 | Writer: zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: "15:04:05", NoColor: true}, 34 | Levels: []zerolog.Level{ 35 | zerolog.ErrorLevel, zerolog.FatalLevel, zerolog.PanicLevel, 36 | }, 37 | }, 38 | ) 39 | return zerolog.New(writer).Level(minimumLevel).With().Timestamp().Logger() 40 | } 41 | -------------------------------------------------------------------------------- /cmd/exit_handler.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/rs/zerolog/log" 7 | ) 8 | 9 | const ( 10 | errorCode = 1 11 | resultsCode = 2 12 | ) 13 | 14 | func isNeedReturnErrorCodeFor(kind ignoreOnExit) bool { 15 | if ignoreOnExitVar == ignoreOnExitNone { 16 | return true 17 | } 18 | 19 | if ignoreOnExitVar == ignoreOnExitAll { 20 | return false 21 | } 22 | 23 | if ignoreOnExitVar != kind { 24 | return true 25 | } 26 | 27 | return false 28 | } 29 | 30 | func exitCodeIfError(err error) int { 31 | if err != nil && isNeedReturnErrorCodeFor("errors") { 32 | log.Error().Err(err).Msg("Failed to run 2ms") 33 | return errorCode 34 | } 35 | 36 | return 0 37 | } 38 | 39 | func exitCodeIfResults(resultsCount int) int { 40 | if resultsCount > 0 && isNeedReturnErrorCodeFor("results") { 41 | return resultsCode 42 | } 43 | 44 | return 0 45 | } 46 | 47 | func Exit(resultsCount int, err error) { 48 | os.Exit(exitCodeIfError(err) + exitCodeIfResults(resultsCount)) 49 | } 50 | 51 | func listenForErrors(errors chan error) { 52 | go func() { 53 | err := <-errors 54 | Exit(0, err) 55 | }() 56 | } 57 | -------------------------------------------------------------------------------- /.github/workflows/validate-readme.yml: -------------------------------------------------------------------------------- 1 | name: Validate README 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - master 7 | merge_group: 8 | 9 | jobs: 10 | validate: 11 | name: README should be updated 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 15 | - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 16 | with: 17 | go-version-file: go.mod 18 | 19 | - name: update README 20 | run: ./.ci/update-readme.sh 21 | - name: validate README wasn't updated 22 | run: | 23 | if ! git diff-index --quiet HEAD; then 24 | # Find the line numbers of the start and end markers 25 | start_line=$(grep -n '<!-- command-line:start -->' README.md | cut -d ":" -f 1) 26 | end_line=$(grep -n '<!-- command-line:end -->' README.md | cut -d ":" -f 1) 27 | 28 | echo "::error file=README.md,title=Outdated README,line=$start_line,endLine=$end_line::README.md is outdated, please run ./.ci/update-readme.sh" 29 | exit 1 30 | fi 31 | -------------------------------------------------------------------------------- /lib/secrets/secret_test.go: -------------------------------------------------------------------------------- 1 | package secrets 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestValidationResultCompareTo(t *testing.T) { 8 | testCases := []struct { 9 | first ValidationResult 10 | second ValidationResult 11 | want compared 12 | message string 13 | }{ 14 | { 15 | first: ValidResult, 16 | second: ValidResult, 17 | want: equal, 18 | message: "Valid should be equal to Valid", 19 | }, 20 | { 21 | first: InvalidResult, 22 | second: ValidResult, 23 | want: second, 24 | message: "Valid should be greater than Invalid", 25 | }, 26 | { 27 | first: ValidResult, 28 | second: UnknownResult, 29 | want: first, 30 | message: "Valid should be greater than Unknown", 31 | }, 32 | { 33 | first: UnknownResult, 34 | second: InvalidResult, 35 | want: second, 36 | message: "Invalid should be greater than Unknown", 37 | }, 38 | } 39 | 40 | for _, tc := range testCases { 41 | t.Run(tc.message, func(t *testing.T) { 42 | got := tc.first.CompareTo(tc.second) 43 | if got != tc.want { 44 | t.Errorf("got %d, want %d", got, tc.want) 45 | } 46 | }, 47 | ) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a bug report to help us improve 4 | title: 'bug(<scope>): <title starting with lowercase letter>' 5 | labels: community, bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | Found a bug? You're welcome to [![GitHub Discussions](https://img.shields.io/badge/chat-discussions-blue.svg?style=flat-square)](https://github.com/Checkmarx/2ms/discussions) 11 | 12 | - **Please make sure to:** 13 | - Describe in details what the problem is 14 | - Attach a log file with relevant data preferably in DEBUG level (`--log-level=DEBUG`) 15 | - Attach the scanned sample files, anonymize the data if the original file cannot be provided 16 | - When attaching files to the issue make sure they are properly formatted 17 | 18 | ### Expected Behavior 19 | 20 | (Which results are expected from 2ms?) 21 | 22 | ### Actual Behavior 23 | 24 | (Formatted logs and samples helps us to better understand the issue) 25 | 26 | ### Steps to Reproduce the Problem 27 | 28 | (Command line arguments and flags used) 29 | 30 | 1. step 1 31 | 2. step 2 32 | 3. step 3 33 | 34 | ### Specifications 35 | 36 | (N/A if not applicable) 37 | 38 | - Version: 39 | - Platform: 40 | - Subsystem: 41 | -------------------------------------------------------------------------------- /.github/workflows/codecov.yaml: -------------------------------------------------------------------------------- 1 | 2 | name: Codecov Scan 3 | 4 | on: 5 | push: 6 | branches: 7 | - main 8 | pull_request: 9 | workflow_dispatch: 10 | 11 | jobs: 12 | run: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - name: Checkout code 17 | uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 18 | 19 | - name: Set up Go 20 | uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 21 | with: 22 | go-version-file: go.mod 23 | env: 24 | GOPROXY: direct 25 | GONOSUMDB: "*" 26 | GOPRIVATE: https://github.com/CheckmarxDev/ 27 | 28 | - name: Install dependencies 29 | run: go install golang.org/x/tools/cmd/cover@latest 30 | 31 | - name: Run tests and generate coverage 32 | run: | 33 | go test ./... -coverpkg=./... -v -coverprofile cover.out 34 | 35 | 36 | - name: Upload coverage to Codecov 37 | uses: codecov/codecov-action@84508663e988701840491b86de86b666e8a86bed # v4.3.0 38 | with: 39 | token: ${{ secrets.CODECOV_TOKEN }} 40 | files: ./cover.out 41 | flags: target=auto 42 | fail_ci_if_error: true 43 | verbose: false 44 | -------------------------------------------------------------------------------- /engine/validation/gitlab.go: -------------------------------------------------------------------------------- 1 | package validation 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | 9 | "github.com/checkmarx/2ms/v4/lib/secrets" 10 | "github.com/rs/zerolog/log" 11 | ) 12 | 13 | type userResponse struct { 14 | WebURL string `json:"web_url"` 15 | } 16 | 17 | func validateGitlab(s *secrets.Secret) (secrets.ValidationResult, string) { 18 | const gitlabURL = "https://gitlab.com/api/v4/user" 19 | 20 | resp, err := sendValidationRequest(gitlabURL, fmt.Sprintf("Bearer %s", s.Value)) 21 | 22 | if err != nil { 23 | log.Warn().Err(err).Msg("Failed to validate secret") 24 | return secrets.UnknownResult, "" 25 | } 26 | defer resp.Body.Close() 27 | 28 | if resp.StatusCode == http.StatusOK { 29 | bodyBytes, err := io.ReadAll(resp.Body) 30 | if err != nil { 31 | log.Warn().Err(err).Msg("Failed to read response body for Gitlab validation") 32 | return secrets.ValidResult, "" 33 | } 34 | 35 | var user userResponse 36 | if err := json.Unmarshal(bodyBytes, &user); err != nil { 37 | log.Warn().Err(err).Msg("Failed to unmarshal response body for Gitlab validation") 38 | return secrets.ValidResult, "" 39 | } 40 | 41 | return secrets.ValidResult, user.WebURL 42 | } 43 | return secrets.InvalidResult, "" 44 | } 45 | -------------------------------------------------------------------------------- /engine/linecontent/linecontent.go: -------------------------------------------------------------------------------- 1 | package linecontent 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | const ( 9 | lineMaxParseSize = 10000 10 | contextLeftSizeLimit = 250 11 | contextRightSizeLimit = 250 12 | ) 13 | 14 | func GetLineContent(line, secret string) (string, error) { 15 | lineSize := len(line) 16 | if lineSize == 0 { 17 | return "", fmt.Errorf("line empty") 18 | } 19 | 20 | if secret == "" { 21 | return "", fmt.Errorf("secret empty") 22 | } 23 | 24 | // Truncate lineContent to max size 25 | if lineSize > lineMaxParseSize { 26 | line = line[:lineMaxParseSize] 27 | lineSize = lineMaxParseSize 28 | } 29 | 30 | // Find the secret's position in the line 31 | secretStartIndex := strings.Index(line, secret) 32 | if secretStartIndex == -1 { 33 | // Secret not found, return truncated content based on context limits 34 | maxSize := contextLeftSizeLimit + contextRightSizeLimit 35 | if lineSize < maxSize { 36 | return line, nil 37 | } 38 | return line[:maxSize], nil 39 | } 40 | 41 | // Calculate bounds for the result 42 | secretEndIndex := secretStartIndex + len(secret) 43 | start := max(secretStartIndex-contextLeftSizeLimit, 0) 44 | end := min(secretEndIndex+contextRightSizeLimit, lineSize) 45 | 46 | return line[start:end], nil 47 | } 48 | -------------------------------------------------------------------------------- /engine/rules/hardcodedPassword.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules" 7 | "github.com/zricethezav/gitleaks/v8/config" 8 | ) 9 | 10 | func HardcodedPassword() *config.Rule { 11 | // This regex is the output regex of 'generic-api-key' rule from gitleaks, with the next changes: 12 | // 1. gitleaks/gitleaks#1267 13 | // 2. gitleaks/gitleaks#1265 14 | // 3. Minimum length of 4 characters (was 10) 15 | regex := regexp.MustCompile( 16 | `(?i)(?:key|api|token|secret|client|passwd|password|auth|access)` + 17 | `(?:[0-9a-z\-_\t .]{0,20})(?:\s|'\s|"|\\){0,3}` + 18 | `(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)` + 19 | `(?:'|\"|\\|\s|=|\x60){0,5}([0-9a-z\-_.=!@#\$%\^\&\*]{4,150})` + 20 | `(?:['\"\\|\s\x60;<]|$)`, 21 | ) 22 | return &config.Rule{ 23 | Description: "Hardcoded password", 24 | RuleID: "hardcoded-password", 25 | Regex: regex, 26 | Keywords: []string{ 27 | "key", 28 | "api", 29 | "token", 30 | "secret", 31 | "client", 32 | "passwd", 33 | "password", 34 | "auth", 35 | "access", 36 | }, 37 | Entropy: 0, 38 | SecretGroup: 1, 39 | Allowlists: []*config.Allowlist{ 40 | { 41 | StopWords: rules.DefaultStopWords, 42 | }, 43 | }, 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /.github/workflows/trivy-vulnerability-scan.yaml: -------------------------------------------------------------------------------- 1 | name: Trivy-scan 2 | on: 3 | push: 4 | workflow_dispatch: 5 | pull_request: 6 | branches: 7 | - master 8 | schedule: 9 | - cron: '5 6 * * *' # Runs every day at 06:05 UTC 10 | 11 | jobs: 12 | trivy-scan: 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout Source 16 | uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 17 | 18 | - name: Build and load (not push) 19 | uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0 20 | with: 21 | load: true 22 | context: . 23 | file: ./Dockerfile 24 | platforms: linux/amd64 25 | push: false 26 | tags: checkmarx/2ms:scanme 27 | 28 | - name: Run Trivy Scan 29 | uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # v0.28.0 30 | with: 31 | image-ref: checkmarx/2ms:scanme 32 | vuln-type: os,library 33 | format: table 34 | ignore-unfixed: true 35 | severity: CRITICAL,HIGH,MEDIUM,LOW,UNKNOWN 36 | trivy-config: trivy.yaml 37 | exit-code: '1' 38 | env: 39 | TRIVY_SKIP_DB_UPDATE: true 40 | TRIVY_SKIP_JAVA_DB_UPDATE: true 41 | -------------------------------------------------------------------------------- /.github/workflows/security.yml: -------------------------------------------------------------------------------- 1 | name: Security Scans 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | merge_group: 11 | schedule: 12 | - cron: "0 0 * * *" 13 | 14 | jobs: 15 | gosec: 16 | runs-on: ubuntu-latest 17 | env: 18 | GO111MODULE: on 19 | steps: 20 | - name: Checkout Source 21 | uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 22 | 23 | - name: Run Gosec Security Scanner 24 | uses: securego/gosec@26e57d6b340778c2983cd61775bc7e8bb41d002a # v2.19.0 25 | with: 26 | args: "-no-fail -fmt sarif -out results.sarif -exclude-dir=.ci -exclude-dir=tests ./..." 27 | 28 | - name: Upload Gosec Results 29 | uses: github/codeql-action/upload-sarif@4355270be187e1b672a7a1c7c7bae5afdc1ab94a #v3.24.10 30 | with: 31 | sarif_file: results.sarif 32 | 33 | - name: Set up Docker Buildx 34 | uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb #v3.3.0 35 | 36 | secret-scanning: 37 | runs-on: ubuntu-latest 38 | steps: 39 | - name: Checkout Source 40 | uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 41 | with: 42 | fetch-depth: 0 43 | 44 | - name: Run 2ms Scan 45 | run: docker run -v $(pwd):/repo checkmarx/2ms:latest git /repo --config /repo/.2ms.yml -------------------------------------------------------------------------------- /pkg/scanner.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | import ( 4 | "github.com/checkmarx/2ms/v4/engine" 5 | "github.com/checkmarx/2ms/v4/internal/resources" 6 | "github.com/checkmarx/2ms/v4/lib/reporting" 7 | "github.com/checkmarx/2ms/v4/plugins" 8 | ) 9 | 10 | type ScanItem struct { 11 | Content *string 12 | // Unique identifier of the item 13 | ID string 14 | // User-friendly description and/or link to the item 15 | Source string 16 | } 17 | 18 | var _ plugins.ISourceItem = (*ScanItem)(nil) 19 | 20 | func (i ScanItem) GetContent() *string { 21 | return i.Content 22 | } 23 | 24 | func (i ScanItem) GetID() string { 25 | return i.ID 26 | } 27 | 28 | func (i ScanItem) GetSource() string { 29 | return i.Source 30 | } 31 | 32 | func (i ScanItem) GetGitInfo() *plugins.GitInfo { 33 | return nil 34 | } 35 | 36 | type Scanner interface { 37 | Reset(scanConfig resources.ScanConfig, opts ...engine.EngineOption) error 38 | Scan(scanItems []ScanItem, scanConfig resources.ScanConfig, opts ...engine.EngineOption) (reporting.IReport, error) 39 | // ScanDynamic performs a scans with custom input of items and optional custom plugin channels. 40 | // 41 | // To provide custom plugin channels, use engine.WithPluginChannels: 42 | // 43 | // pluginChannels := plugins.NewChannels(func(c *plugins.Channels) { 44 | // c.Items = make(chan plugins.ISourceItem, 100) 45 | // }) 46 | // s.ScanDynamic(ScanConfig{}, engine.WithPluginChannels(pluginChannels)) 47 | ScanDynamic(itemsIn <-chan ScanItem, scanConfig resources.ScanConfig, opts ...engine.EngineOption) (reporting.IReport, error) 48 | } 49 | -------------------------------------------------------------------------------- /engine/rules/1password.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/zricethezav/gitleaks/v8/config" 7 | ) 8 | 9 | // OnePasswordSecretKey Reference: 10 | // - https://1passwordstatic.com/files/security/1password-white-paper.pdf 11 | func OnePasswordSecretKey() *config.Rule { 12 | // 1Password secret keys include several hyphens but these are only for readability 13 | // and are stripped during 1Password login. This means that the following are technically 14 | // the same valid key: 15 | // - A3ASWWYB798JRYLJVD423DC286TVMH43EB 16 | // - A-3-A-S-W-W-Y-B-7-9-8-J-R-Y-L-J-V-D-4-2-3-D-C-2-8-6-T-V-M-H-4-3-E-B 17 | // But in practice, when these keys are added to a vault, exported in an emergency kit, or 18 | // copied, they have hyphens that follow one of two patterns I can find: 19 | // - A3-ASWWYB-798JRY-LJVD4-23DC2-86TVM-H43EB (every key I've generated has this pattern) 20 | // - A3-ASWWYB-798JRYLJVD4-23DC2-86TVM-H43EB (the whitepaper includes this example, which could just be a typo) 21 | // To avoid a complicated regex that checks for every possible situation it's probably best 22 | // to scan for the these two patterns. 23 | return &config.Rule{ 24 | Description: "Uncovered a possible 1Password secret key, potentially compromising access to secrets in vaults.", 25 | RuleID: "1password-secret-key", 26 | Regex: regexp.MustCompile(`\bA3-[A-Z0-9]{6}-(?:(?:[A-Z0-9]{11})|(?:[A-Z0-9]{6}-[A-Z0-9]{5}))-[A-Z0-9]{5}-[A-Z0-9]{5}-[A-Z0-9]{5}\b`), //nolint:lll 27 | Entropy: 3.8, 28 | Keywords: []string{"A3-"}, 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /engine/extra/extra.go: -------------------------------------------------------------------------------- 1 | package extra 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "fmt" 7 | "strings" 8 | 9 | "github.com/checkmarx/2ms/v4/lib/secrets" 10 | ) 11 | 12 | type addExtraFunc = func(*secrets.Secret) interface{} 13 | 14 | var ruleIDToFunction = map[string]addExtraFunc{ 15 | "jwt": addExtraJWT, 16 | } 17 | 18 | func AddExtraToSecret(secret *secrets.Secret) { 19 | if addExtra, ok := ruleIDToFunction[secret.RuleID]; ok { 20 | extraData := addExtra(secret) 21 | if extraData != nil && extraData != "" { 22 | UpdateExtraField(secret, "secretDetails", extraData) 23 | } 24 | } 25 | } 26 | 27 | var mtxs = &NamedMutex{} 28 | 29 | func UpdateExtraField(secret *secrets.Secret, extraName string, extraData interface{}) { 30 | mtxs.Lock(secret.ID) 31 | defer mtxs.Unlock(secret.ID) 32 | 33 | if secret.ExtraDetails == nil { 34 | secret.ExtraDetails = make(map[string]interface{}) 35 | } 36 | secret.ExtraDetails[extraName] = extraData 37 | } 38 | 39 | func addExtraJWT(secret *secrets.Secret) interface{} { 40 | tokenString := secret.Value 41 | 42 | parts := strings.Split(tokenString, ".") 43 | if len(parts) != 3 { 44 | return "Invalid JWT token" 45 | } 46 | 47 | payload, err := base64.RawURLEncoding.DecodeString(parts[1]) 48 | if err != nil { 49 | return fmt.Sprintf("Failed to decode JWT payload: %s", err) 50 | } 51 | 52 | var claims map[string]interface{} 53 | err = json.Unmarshal(payload, &claims) 54 | if err != nil { 55 | return fmt.Sprintf("Failed to unmarshal JWT payload: %s", string(payload)) 56 | } 57 | 58 | return claims 59 | } 60 | -------------------------------------------------------------------------------- /.github/workflows/trivy-cache.yaml: -------------------------------------------------------------------------------- 1 | # Note: This workflow only updates the cache. You should create a separate workflow for your actual Trivy scans. 2 | # In your scan workflow, set TRIVY_SKIP_DB_UPDATE=true and TRIVY_SKIP_JAVA_DB_UPDATE=true. 3 | name: Update Trivy Cache 4 | 5 | on: 6 | schedule: 7 | - cron: '0 0 * * *' # Run daily at midnight UTC 8 | workflow_dispatch: # Allow manual triggering 9 | 10 | jobs: 11 | update-trivy-db: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: Setup oras 15 | uses: oras-project/setup-oras@5c0b487ce3fe0ce3ab0d034e63669e426e294e4d #v1.2.2 16 | 17 | - name: Get current date 18 | id: date 19 | run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT 20 | 21 | - name: Download and extract the vulnerability DB 22 | run: | 23 | mkdir -p $GITHUB_WORKSPACE/.cache/trivy/db 24 | oras pull ghcr.io/aquasecurity/trivy-db:2 25 | tar -xzf db.tar.gz -C $GITHUB_WORKSPACE/.cache/trivy/db 26 | rm db.tar.gz 27 | 28 | - name: Download and extract the Java DB 29 | run: | 30 | mkdir -p $GITHUB_WORKSPACE/.cache/trivy/java-db 31 | oras pull ghcr.io/aquasecurity/trivy-java-db:1 32 | tar -xzf javadb.tar.gz -C $GITHUB_WORKSPACE/.cache/trivy/java-db 33 | rm javadb.tar.gz 34 | 35 | - name: Cache DBs 36 | uses: actions/cache/save@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2.0 37 | with: 38 | path: ${{ github.workspace }}/.cache/trivy 39 | key: cache-trivy-${{ steps.date.outputs.date }} 40 | -------------------------------------------------------------------------------- /lib/secrets/secret.go: -------------------------------------------------------------------------------- 1 | package secrets 2 | 3 | type ValidationResult string 4 | 5 | const ( 6 | ValidResult ValidationResult = "Valid" 7 | InvalidResult ValidationResult = "Invalid" 8 | UnknownResult ValidationResult = "Unknown" 9 | ) 10 | 11 | type compared int 12 | 13 | const ( 14 | first compared = -1 15 | second compared = 1 16 | equal compared = 0 17 | ) 18 | 19 | func (v ValidationResult) CompareTo(other ValidationResult) compared { 20 | if v == other { 21 | return equal 22 | } 23 | if v == UnknownResult { 24 | return second 25 | } 26 | if other == UnknownResult { 27 | return first 28 | } 29 | if v == InvalidResult { 30 | return second 31 | } 32 | return first 33 | } 34 | 35 | type Secret struct { 36 | ID string `json:"id"` 37 | Source string `json:"source"` 38 | RuleID string `json:"ruleId"` 39 | StartLine int `json:"startLine"` 40 | EndLine int `json:"endLine"` 41 | LineContent string `json:"lineContent"` 42 | StartColumn int `json:"startColumn"` 43 | EndColumn int `json:"endColumn"` 44 | Value string `json:"value"` 45 | ValidationStatus ValidationResult `json:"validationStatus,omitempty"` 46 | RuleDescription string `json:"ruleDescription,omitempty"` 47 | ExtraDetails map[string]interface{} `json:"extraDetails,omitempty"` 48 | CvssScore float64 `json:"cvssScore,omitempty"` 49 | } 50 | -------------------------------------------------------------------------------- /plugins/plugins.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "runtime" 5 | "sync" 6 | 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | type ISourceItem interface { 11 | GetContent() *string 12 | GetID() string 13 | GetSource() string 14 | GetGitInfo() *GitInfo 15 | } 16 | 17 | type item struct { 18 | Content *string 19 | // Unique identifier of the item 20 | ID string 21 | // User friendly description and/or link to the item 22 | Source string 23 | GitInfo *GitInfo 24 | } 25 | 26 | var _ ISourceItem = (*item)(nil) 27 | 28 | func (i item) GetContent() *string { 29 | return i.Content 30 | } 31 | 32 | func (i item) GetID() string { 33 | return i.ID 34 | } 35 | 36 | func (i item) GetSource() string { 37 | return i.Source 38 | } 39 | 40 | func (i item) GetGitInfo() *GitInfo { 41 | return i.GitInfo 42 | } 43 | 44 | type Plugin struct { 45 | ID string 46 | Limit chan struct{} 47 | } 48 | 49 | type Channels struct { 50 | Items chan ISourceItem 51 | Errors chan error 52 | wg *sync.WaitGroup 53 | } 54 | 55 | type PluginChannels interface { 56 | GetItemsCh() chan ISourceItem 57 | GetErrorsCh() chan error 58 | } 59 | 60 | type Option func(*Channels) 61 | 62 | func NewChannels(opts ...Option) PluginChannels { 63 | channels := &Channels{ 64 | Items: make(chan ISourceItem, runtime.GOMAXPROCS(0)), 65 | Errors: make(chan error, 4), 66 | } 67 | 68 | for _, opt := range opts { 69 | opt(channels) 70 | } 71 | 72 | return channels 73 | } 74 | 75 | func (c *Channels) GetItemsCh() chan ISourceItem { 76 | return c.Items 77 | } 78 | 79 | func (c *Channels) GetErrorsCh() chan error { 80 | return c.Errors 81 | } 82 | 83 | type IPlugin interface { 84 | GetName() string 85 | DefineCommand(items chan ISourceItem, errors chan error) (*cobra.Command, error) 86 | } 87 | -------------------------------------------------------------------------------- /engine/extra/extra_test.go: -------------------------------------------------------------------------------- 1 | package extra 2 | 3 | import ( 4 | "encoding/base64" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/checkmarx/2ms/v4/lib/secrets" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestAddExtraToSecret(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | secretValue string 16 | expectedOutput interface{} 17 | }{ 18 | { 19 | name: "Valid JWT", 20 | secretValue: "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6Im1vY2tOYW1lIn0.dummysignature", 21 | expectedOutput: map[string]interface{}{ 22 | "sub": "1234567890", 23 | "name": "mockName", 24 | }, 25 | }, 26 | { 27 | name: "Invalid JWT format - it should contain exactly three parts separated by '.'", 28 | secretValue: "invalidJWT.token", 29 | expectedOutput: "Invalid JWT token", 30 | }, 31 | { 32 | name: "Base64 decoding failure", 33 | secretValue: "header." + base64.RawURLEncoding.EncodeToString([]byte("invalid_payload")) + ".signature", 34 | expectedOutput: "Failed to unmarshal JWT payload: invalid_payload", 35 | }, 36 | { 37 | name: "Malformed base64", 38 | secretValue: fmt.Sprintf("header.%s.signature", 39 | base64.RawURLEncoding.EncodeToString([]byte("{malformed_json"))), 40 | expectedOutput: "Failed to unmarshal JWT payload: {malformed_json", 41 | }, 42 | } 43 | 44 | for _, tt := range tests { 45 | t.Run(tt.name, func(t *testing.T) { 46 | secret := &secrets.Secret{ 47 | ID: "test-secret", 48 | RuleID: "jwt", 49 | Value: tt.secretValue, 50 | ExtraDetails: make(map[string]interface{}), 51 | } 52 | 53 | AddExtraToSecret(secret) 54 | 55 | assert.Equal(t, tt.expectedOutput, secret.ExtraDetails["secretDetails"]) 56 | }) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /engine/validation/validator.go: -------------------------------------------------------------------------------- 1 | package validation 2 | 3 | import ( 4 | "github.com/checkmarx/2ms/v4/engine/extra" 5 | "github.com/checkmarx/2ms/v4/lib/secrets" 6 | ) 7 | 8 | type validationFunc = func(*secrets.Secret) (secrets.ValidationResult, string) 9 | 10 | var ruleIDToFunction = map[string]validationFunc{ 11 | "github-fine-grained-pat": validateGithub, 12 | "github-pat": validateGithub, 13 | "gitlab-pat": validateGitlab, 14 | "gcp-api-key": validateGCP, 15 | } 16 | 17 | type Validator struct { 18 | pairsCollector *pairsCollector 19 | } 20 | 21 | func NewValidator() *Validator { 22 | return &Validator{pairsCollector: newPairsCollector()} 23 | } 24 | 25 | func (v *Validator) RegisterForValidation(secret *secrets.Secret) { 26 | if validate, ok := ruleIDToFunction[secret.RuleID]; ok { 27 | status, extra := validate(secret) 28 | secret.ValidationStatus = status 29 | addExtraToSecret(secret, extra) 30 | } else if !v.pairsCollector.addIfNeeded(secret) { 31 | secret.ValidationStatus = secrets.UnknownResult 32 | } 33 | } 34 | 35 | func (v *Validator) Validate() { 36 | for generalKey, bySource := range v.pairsCollector.pairs { 37 | for _, byRule := range bySource { 38 | v.pairsCollector.validate(generalKey, byRule) 39 | } 40 | } 41 | } 42 | 43 | func IsCanValidateRule(ruleID string) bool { 44 | if _, ok := ruleIDToFunction[ruleID]; ok { 45 | return true 46 | } 47 | if _, ok := ruleToGeneralKey[ruleID]; ok { 48 | return true 49 | } 50 | 51 | return false 52 | } 53 | 54 | func addExtraToSecret(secret *secrets.Secret, extraData string) { 55 | if extraData == "" { 56 | return 57 | } 58 | 59 | if secret.ExtraDetails == nil { 60 | secret.ExtraDetails = make(map[string]interface{}) 61 | } 62 | 63 | extra.UpdateExtraField(secret, "validationDetails", extraData) 64 | } 65 | -------------------------------------------------------------------------------- /lib/reporting/sarif_test.go: -------------------------------------------------------------------------------- 1 | package reporting 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestCreateMessageText(t *testing.T) { 11 | ruleName := "Test Rule" 12 | messagePrefix := ruleName + " has detected secret for file %s." 13 | 14 | tests := []struct { 15 | Name string 16 | FilePath string 17 | ExpectedMessage string 18 | }{ 19 | { 20 | Name: "Filesystem file name", 21 | FilePath: "folder/filename.txt", 22 | ExpectedMessage: fmt.Sprintf(messagePrefix, "folder/filename.txt"), 23 | }, 24 | { 25 | Name: "Simple git filename", 26 | FilePath: "git show 1a9f3c87b4d029f54e8c72d8b11a78f6a3c29d2e:folder/filename.txt", 27 | ExpectedMessage: fmt.Sprintf(messagePrefix, "folder/filename.txt"), 28 | }, 29 | { 30 | Name: "Broken git file name with no commit hash", 31 | FilePath: "git show folder/filename.txt", 32 | ExpectedMessage: fmt.Sprintf(messagePrefix, "git show folder/filename.txt"), 33 | }, 34 | { 35 | Name: "Git file name with one colon character", 36 | FilePath: "git show d8e914f06d8d4494bd4f9ab2a2c9c88f78ef25ad:folder/filename:secondpart.txt", 37 | ExpectedMessage: fmt.Sprintf(messagePrefix, "folder/filename:secondpart.txt"), 38 | }, 39 | { 40 | Name: "Git file name with multiple colon character", 41 | FilePath: "git show a73b5cf94f0b29e1cc6e71a092f6b8ebc1d0e002:folder:secondfolderpart/filename:secondpart.txt", 42 | ExpectedMessage: fmt.Sprintf(messagePrefix, "folder:secondfolderpart/filename:secondpart.txt"), 43 | }, 44 | } 45 | 46 | for _, tt := range tests { 47 | t.Run(tt.Name, func(t *testing.T) { 48 | message := createMessageText(ruleName, tt.FilePath) 49 | fmt.Printf("%v", message) 50 | assert.Equal(t, tt.ExpectedMessage, message) 51 | }) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /benches/README.md: -------------------------------------------------------------------------------- 1 | # 2MS Benchmarks 2 | 3 | ## Build Tag Setup 4 | 5 | These benchmarks are excluded from regular test runs using the `//go:build bench` build tag. This prevents the heavy benchmark setup (which creates 10,000 test files) from running during normal `go test` executions. 6 | 7 | - **Regular tests**: `go test` (benchmarks won't run) 8 | - **Run benchmarks**: Use the `-tags=bench` flag as shown below 9 | 10 | ## Process Items Benchmark 11 | 12 | This benchmark (`BenchmarkProcessItems`) tests the performance of secret detection processing across different configurations. 13 | 14 | ### What it Tests 15 | 16 | 1. **Worker Pool Scaling** 17 | - Tests different worker pool sizes based on CPU count 18 | - Ranges from half the CPU count up to 32x CPU count 19 | - Example for 8-core machine: tests 4, 8, 16, 32, 64, 128, and 256 workers 20 | 21 | 2. **Input Load Testing** 22 | - Tests various input sizes: 50, 100, 500, 1000, and 10000 items 23 | 24 | 3. **Realistic Content** 25 | - Simulates different file types: 26 | - JavaScript configurations 27 | - Python scripts 28 | - Shell scripts 29 | - YAML configurations 30 | - JSON configurations 31 | - Includes actual secret patterns: 32 | - GitHub Personal Access Tokens 33 | - API keys 34 | - JWTs 35 | - Varies file sizes (1KB, 10KB, 50KB) 36 | - Maintains a 60/40 ratio of files with/without secrets 37 | 38 | ### Running the Benchmark 39 | 40 | ```bash 41 | go test -tags=bench -timeout 0 -bench BenchmarkProcessItems -count 5 -run=^$ 42 | ``` 43 | 44 | #### Command Flags Explained 45 | - `-tags=bench`: Enables compilation of benchmark code (required due to build tag) 46 | - `-timeout 0`: Disables test timeout (needed for long benchmarks) 47 | - `-bench BenchmarkProcessItems`: Runs only this specific benchmark 48 | - `-count 5`: Runs the benchmark 5 times for better statistical significance 49 | - `-run=^$`: Skips regular tests (only runs benchmarks) 50 | -------------------------------------------------------------------------------- /engine/validation/pairs.go: -------------------------------------------------------------------------------- 1 | package validation 2 | 3 | import ( 4 | "github.com/checkmarx/2ms/v4/lib/secrets" 5 | ) 6 | 7 | type pairsByRuleId map[string][]*secrets.Secret 8 | type pairsBySource map[string]pairsByRuleId 9 | type pairsByGeneralKey map[string]pairsBySource 10 | 11 | type pairsCollector struct { 12 | pairs pairsByGeneralKey 13 | } 14 | 15 | func newPairsCollector() *pairsCollector { 16 | return &pairsCollector{pairs: make(pairsByGeneralKey)} 17 | } 18 | 19 | func (p *pairsCollector) addIfNeeded(secret *secrets.Secret) bool { 20 | generalKey, ok := ruleToGeneralKey[secret.RuleID] 21 | if !ok { 22 | return false 23 | } 24 | 25 | if _, ok := p.pairs[generalKey]; !ok { 26 | p.pairs[generalKey] = make(pairsBySource) 27 | } 28 | if _, ok := p.pairs[generalKey][secret.Source]; !ok { 29 | p.pairs[generalKey][secret.Source] = make(pairsByRuleId) 30 | } 31 | if _, ok := p.pairs[generalKey][secret.Source][secret.RuleID]; !ok { 32 | p.pairs[generalKey][secret.Source][secret.RuleID] = make([]*secrets.Secret, 0) 33 | } 34 | 35 | p.pairs[generalKey][secret.Source][secret.RuleID] = append(p.pairs[generalKey][secret.Source][secret.RuleID], secret) 36 | return true 37 | } 38 | 39 | func (p *pairsCollector) validate(generalKey string, rulesById pairsByRuleId) { 40 | generalKeyToValidation[generalKey](rulesById) 41 | } 42 | 43 | type pairsValidationFunc func(pairsByRuleId) 44 | 45 | var generalKeyToValidation = map[string]pairsValidationFunc{ 46 | "alibaba": validateAlibaba, 47 | } 48 | 49 | var generalKeyToRules = map[string][]string{ 50 | "alibaba": {"alibaba-access-key-id", "alibaba-secret-key"}, 51 | } 52 | 53 | func generateRuleToGeneralKey() map[string]string { 54 | ruleToGeneralKey := make(map[string]string) 55 | for key, rules := range generalKeyToRules { 56 | for _, rule := range rules { 57 | ruleToGeneralKey[rule] = key 58 | } 59 | } 60 | return ruleToGeneralKey 61 | } 62 | 63 | var ruleToGeneralKey = generateRuleToGeneralKey() 64 | -------------------------------------------------------------------------------- /lib/reporting/yaml.go: -------------------------------------------------------------------------------- 1 | package reporting 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | "gopkg.in/yaml.v3" 8 | ) 9 | 10 | func writeYaml(report *Report) (string, error) { 11 | estimatedSize := 1024 + len(report.Results)*512 12 | var builder strings.Builder 13 | builder.Grow(estimatedSize) 14 | 15 | fmt.Fprintf(&builder, "totalitemsscanned: %d\n", report.TotalItemsScanned) 16 | fmt.Fprintf(&builder, "totalsecretsfound: %d\n", report.TotalSecretsFound) 17 | if report.TotalSecretsFound == 0 { 18 | fmt.Fprint(&builder, "results: {}\n") 19 | } else { 20 | builder.WriteString("results:\n") 21 | for _, secretsList := range report.Results { 22 | if len(secretsList) > 0 { 23 | fmt.Fprintf(&builder, " %s:\n", secretsList[0].ID) 24 | } 25 | for _, s := range secretsList { 26 | fmt.Fprintf(&builder, " - id: %s\n", s.ID) 27 | fmt.Fprintf(&builder, " source: %s\n", s.Source) 28 | fmt.Fprintf(&builder, " ruleid: %s\n", s.RuleID) 29 | fmt.Fprintf(&builder, " startline: %d\n", s.StartLine) 30 | fmt.Fprintf(&builder, " endline: %d\n", s.EndLine) 31 | fmt.Fprintf(&builder, " linecontent: %q\n", s.LineContent) 32 | fmt.Fprintf(&builder, " startcolumn: %d\n", s.StartColumn) 33 | fmt.Fprintf(&builder, " endcolumn: %d\n", s.EndColumn) 34 | fmt.Fprintf(&builder, " value: %s\n", s.Value) 35 | fmt.Fprintf(&builder, " validationstatus: %q\n", fmt.Sprintf("%v", s.ValidationStatus)) 36 | fmt.Fprintf(&builder, " ruledescription: %s\n", s.RuleDescription) 37 | if len(s.ExtraDetails) > 0 { 38 | builder.WriteString(" extradetails:\n") 39 | marshaled, err := yaml.Marshal(s.ExtraDetails) 40 | if err != nil { 41 | fmt.Fprintf(&builder, " error: %v\n", err) 42 | return "", err 43 | } else { 44 | lines := strings.Split(string(marshaled), "\n") 45 | for _, line := range lines { 46 | if line != "" { 47 | fmt.Fprintf(&builder, " %s\n", line) 48 | } 49 | } 50 | } 51 | } 52 | fmt.Fprintf(&builder, " cvssscore: %.1f\n", s.CvssScore) 53 | } 54 | } 55 | } 56 | 57 | return builder.String(), nil 58 | } 59 | -------------------------------------------------------------------------------- /lib/utils/http.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "context" 5 | "encoding/base64" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | 10 | "github.com/rs/zerolog/log" 11 | ) 12 | 13 | type ICredentials interface { 14 | GetCredentials() (string, string) 15 | } 16 | 17 | func CreateBasicAuthCredentials(credentials ICredentials) string { 18 | username, password := credentials.GetCredentials() 19 | return "Basic " + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", username, password))) 20 | } 21 | 22 | type IAuthorizationHeader interface { 23 | GetAuthorizationHeader() string 24 | } 25 | 26 | type RetrySettings struct { 27 | MaxRetries int 28 | ErrorCodes []int 29 | } 30 | 31 | func HttpRequest(method, url string, authorization IAuthorizationHeader, retry RetrySettings) ([]byte, *http.Response, error) { 32 | request, err := http.NewRequestWithContext(context.Background(), method, url, http.NoBody) 33 | if err != nil { 34 | return nil, nil, fmt.Errorf("unexpected error creating an http request %w", err) 35 | } 36 | 37 | if authorization.GetAuthorizationHeader() != "" { 38 | request.Header.Set("Authorization", authorization.GetAuthorizationHeader()) 39 | } 40 | 41 | // TODO: do not recreate this client for each request 42 | client := &http.Client{} 43 | response, err := client.Do(request) 44 | if err != nil { 45 | return nil, response, fmt.Errorf("unable to send http request %w", err) 46 | } 47 | 48 | if response.StatusCode < 200 || response.StatusCode >= 300 { 49 | if retry.MaxRetries > 0 { 50 | for _, code := range retry.ErrorCodes { 51 | if response.StatusCode == code { 52 | log.Warn().Msgf("retrying http request %v", url) 53 | return HttpRequest( 54 | method, 55 | url, 56 | authorization, 57 | RetrySettings{MaxRetries: retry.MaxRetries - 1, ErrorCodes: retry.ErrorCodes}, 58 | ) 59 | } 60 | } 61 | } 62 | return nil, response, fmt.Errorf("error calling http url \"%v\". status code: %v", url, response) 63 | } 64 | 65 | body, err := io.ReadAll(response.Body) 66 | if err != nil { 67 | return nil, response, fmt.Errorf("unexpected error reading http response body %w", err) 68 | } 69 | 70 | return body, response, nil 71 | } 72 | -------------------------------------------------------------------------------- /tests/testData/expectedReport/multi_line_secret_report.json: -------------------------------------------------------------------------------- 1 | { 2 | "totalItemsScanned": 1, 3 | "totalSecretsFound": 2, 4 | "results": { 5 | "0a444c3960dbca51baeacf6d5193f64b5ddf0d66": [ 6 | { 7 | "id": "0a444c3960dbca51baeacf6d5193f64b5ddf0d66", 8 | "source": "testData/input/multi_line_secret.txt", 9 | "ruleId": "private-key", 10 | "startLine": 3, 11 | "endLine": 4, 12 | "lineContent": " -----BEGIN RSA PRIVATE KEY----- MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu KUpRKfFLfRYC9AIKjbJTWit+Cq\n vjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp79mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs /5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00 -----END RSA PRIVATE KEY-----", 13 | "startColumn": 9, 14 | "endColumn": 376, 15 | "value": "-----BEGIN RSA PRIVATE KEY----- MIIBOgIBAAJBAKj34GkxFhD90vcNLYLInFEX6Ppy1tPf9Cnzj4p4WGeKLs1Pt8Qu KUpRKfFLfRYC9AIKjbJTWit+Cq\r\n vjWYzvQwECAwEAAQJAIJLixBy2qpFoS4DSmoEm o3qGy0t6z09AIJtH+5OeRV1be+N4cDYJKffGzDa88vQENZiRm0GRq6a+HPGQMd2k TQIhAKMSvzIBnni7ot/OSie2TmJLY4SwTQAevXysE2RbFDYdAiEBCUEaRQnMnbp79mxDXDf6AU0cN/RPBjb9qSHDcWZHGzUCIG2Es59z8ugGrDY+pxLQnwfotadxd+Uy v/Ow5T0q5gIJAiEAyS4RaI9YG8EWx/2w0T67ZUVAw8eOMB6BIUg0Xcu+3okCIBOs /5OiPgoTdSy7bcF9IGpSE8ZgGKzgYQVZeN97YE00 -----END RSA PRIVATE KEY-----", 16 | "ruleDescription": "Identified a Private Key, which may compromise cryptographic security and sensitive data encryption.", 17 | "cvssScore": 8.2 18 | } 19 | ], 20 | "bc98bd8fae8e5b167ac3b692a75b1c9a794a0143": [ 21 | { 22 | "id": "bc98bd8fae8e5b167ac3b692a75b1c9a794a0143", 23 | "source": "testData/input/multi_line_secret.txt", 24 | "ruleId": "generic-api-key", 25 | "startLine": 2, 26 | "endLine": 2, 27 | "lineContent": "\t\t`\"client_secret\" : \"6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde\",`", 28 | "startColumn": 5, 29 | "endColumn": 87, 30 | "value": "6da89121079f83b2eb6acccf8219ea982c3d79bccc3e9c6a85856480661f8fde", 31 | "ruleDescription": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", 32 | "cvssScore": 8.2 33 | } 34 | ] 35 | } 36 | } -------------------------------------------------------------------------------- /engine/semaphore/semaphore_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: semaphore.go 3 | // 4 | // Generated by this command: 5 | // 6 | // mockgen -source=semaphore.go -destination=semaphore_mock.go -package=semaphore 7 | // 8 | 9 | // Package semaphore is a generated GoMock package. 10 | package semaphore 11 | 12 | import ( 13 | context "context" 14 | reflect "reflect" 15 | 16 | gomock "go.uber.org/mock/gomock" 17 | ) 18 | 19 | // MockISemaphore is a mock of ISemaphore interface. 20 | type MockISemaphore struct { 21 | ctrl *gomock.Controller 22 | recorder *MockISemaphoreMockRecorder 23 | isgomock struct{} 24 | } 25 | 26 | // MockISemaphoreMockRecorder is the mock recorder for MockISemaphore. 27 | type MockISemaphoreMockRecorder struct { 28 | mock *MockISemaphore 29 | } 30 | 31 | // NewMockISemaphore creates a new mock instance. 32 | func NewMockISemaphore(ctrl *gomock.Controller) *MockISemaphore { 33 | mock := &MockISemaphore{ctrl: ctrl} 34 | mock.recorder = &MockISemaphoreMockRecorder{mock} 35 | return mock 36 | } 37 | 38 | // EXPECT returns an object that allows the caller to indicate expected use. 39 | func (m *MockISemaphore) EXPECT() *MockISemaphoreMockRecorder { 40 | return m.recorder 41 | } 42 | 43 | // AcquireMemoryWeight mocks base method. 44 | func (m *MockISemaphore) AcquireMemoryWeight(ctx context.Context, weight int64) error { 45 | m.ctrl.T.Helper() 46 | ret := m.ctrl.Call(m, "AcquireMemoryWeight", ctx, weight) 47 | ret0, _ := ret[0].(error) 48 | return ret0 49 | } 50 | 51 | // AcquireMemoryWeight indicates an expected call of AcquireMemoryWeight. 52 | func (mr *MockISemaphoreMockRecorder) AcquireMemoryWeight(ctx, weight any) *gomock.Call { 53 | mr.mock.ctrl.T.Helper() 54 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AcquireMemoryWeight", reflect.TypeOf((*MockISemaphore)(nil).AcquireMemoryWeight), ctx, weight) 55 | } 56 | 57 | // ReleaseMemoryWeight mocks base method. 58 | func (m *MockISemaphore) ReleaseMemoryWeight(weight int64) { 59 | m.ctrl.T.Helper() 60 | m.ctrl.Call(m, "ReleaseMemoryWeight", weight) 61 | } 62 | 63 | // ReleaseMemoryWeight indicates an expected call of ReleaseMemoryWeight. 64 | func (mr *MockISemaphoreMockRecorder) ReleaseMemoryWeight(weight any) *gomock.Call { 65 | mr.mock.ctrl.T.Helper() 66 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReleaseMemoryWeight", reflect.TypeOf((*MockISemaphore)(nil).ReleaseMemoryWeight), weight) 67 | } 68 | -------------------------------------------------------------------------------- /engine/validation/gcp.go: -------------------------------------------------------------------------------- 1 | package validation 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "io" 7 | "net/http" 8 | "strings" 9 | 10 | "github.com/checkmarx/2ms/v4/lib/secrets" 11 | "github.com/rs/zerolog/log" 12 | ) 13 | 14 | type errorResponse struct { 15 | Error struct { 16 | Message string `json:"message"` 17 | Details []struct { 18 | Type string `json:"@type"` 19 | Metadata struct { 20 | Consumer string `json:"consumer"` 21 | } `json:"metadata,omitempty"` 22 | } `json:"details"` 23 | } `json:"error"` 24 | } 25 | 26 | func validateGCP(s *secrets.Secret) (secrets.ValidationResult, string) { 27 | testURL := "https://youtube.googleapis.com/youtube/v3/search?part=snippet&key=" + s.Value 28 | 29 | req, err := http.NewRequestWithContext(context.Background(), "GET", testURL, http.NoBody) 30 | if err != nil { 31 | log.Warn().Err(err).Msg("Failed to validate secret") 32 | return secrets.UnknownResult, "" 33 | } 34 | 35 | client := &http.Client{} 36 | resp, err := client.Do(req) 37 | if err != nil { 38 | log.Warn().Err(err).Msg("Failed to validate secret") 39 | return secrets.UnknownResult, "" 40 | } 41 | defer resp.Body.Close() 42 | 43 | result, extra, err := checkGCPErrorResponse(resp) 44 | if err != nil { 45 | log.Warn().Err(err).Msg("Failed to validate secret") 46 | } 47 | return result, extra 48 | } 49 | 50 | func checkGCPErrorResponse(resp *http.Response) (secrets.ValidationResult, string, error) { 51 | if resp.StatusCode == http.StatusOK { 52 | return secrets.ValidResult, "", nil 53 | } 54 | 55 | if resp.StatusCode != http.StatusForbidden { 56 | return secrets.InvalidResult, "", nil 57 | } 58 | 59 | bodyBytes, err := io.ReadAll(resp.Body) 60 | if err != nil { 61 | return secrets.UnknownResult, "", err 62 | } 63 | 64 | // Unmarshal the response body into the ErrorResponse struct 65 | var errorResponse errorResponse 66 | err = json.Unmarshal(bodyBytes, &errorResponse) 67 | if err != nil { 68 | return secrets.UnknownResult, "", err 69 | } 70 | 71 | if strings.Contains(errorResponse.Error.Message, "YouTube Data API v3 has not been used in project") { 72 | extra := "" 73 | for _, detail := range errorResponse.Error.Details { 74 | if detail.Type == "type.googleapis.com/google.rpc.ErrorInfo" { 75 | extra = detail.Metadata.Consumer 76 | } 77 | } 78 | return secrets.ValidResult, extra, nil 79 | } 80 | 81 | return secrets.UnknownResult, "", nil 82 | } 83 | -------------------------------------------------------------------------------- /engine/semaphore/semaphore_test.go: -------------------------------------------------------------------------------- 1 | package semaphore 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestComputeMemoryBudget(t *testing.T) { 12 | mib := 1024 * 1024 // 1MiB 13 | safety := 200 * mib // 200MiB 14 | 15 | type testCase struct { 16 | name string 17 | hostMemory uint64 18 | cgroupLimit uint64 19 | expectedBudget int64 20 | } 21 | testCases := []testCase{ 22 | { 23 | name: "host memory only", 24 | hostMemory: uint64(4 * 1024 * mib), // 4GiB 25 | cgroupLimit: ^uint64(0), 26 | expectedBudget: int64((4*1024*mib - safety) / 2), // 2GiB - 200MiB 27 | }, 28 | { 29 | name: "cgroup tighter than host", 30 | hostMemory: uint64(4 * 1024 * mib), // 4GiB 31 | cgroupLimit: uint64(2 * 1024 * mib), // 2GiB 32 | expectedBudget: int64((2*1024*mib - safety) / 2), // 1GiB - 200MiB 33 | }, 34 | { 35 | name: "floor budget to 256MiB", 36 | hostMemory: uint64(300 * mib), // 300MiB 37 | cgroupLimit: 0, 38 | expectedBudget: int64(256 * mib), // 256MiB 39 | }, 40 | } 41 | 42 | for _, tc := range testCases { 43 | t.Run(tc.name, func(t *testing.T) { 44 | budget := computeMemoryBudget(tc.hostMemory, tc.cgroupLimit) 45 | assert.Equal(t, tc.expectedBudget, budget, "Expected budget does not match actual budget") 46 | }) 47 | } 48 | } 49 | 50 | func TestAcquireReleaseMemoryWeight(t *testing.T) { 51 | weight := int64(1024) // 1KiB 52 | defaultMemoryBudget := int64(1024 * 1024) // 1MiB 53 | type testCase struct { 54 | name string 55 | memoryBudget int64 56 | expectedError error 57 | } 58 | 59 | testCases := []testCase{ 60 | { 61 | name: "successful acquisition and release", 62 | memoryBudget: defaultMemoryBudget, 63 | }, 64 | { 65 | name: "failed acquisition - over budget", 66 | memoryBudget: weight - 1, 67 | expectedError: fmt.Errorf("buffer size %d exceeds memory budget %d", weight, weight-1), 68 | }, 69 | } 70 | for _, tc := range testCases { 71 | t.Run(tc.name, func(t *testing.T) { 72 | sem := NewSemaphoreWithBudget(tc.memoryBudget) 73 | 74 | err := sem.AcquireMemoryWeight(context.Background(), weight) 75 | if err == nil { 76 | sem.ReleaseMemoryWeight(weight) 77 | } else { 78 | assert.Equal(t, tc.expectedError, err) 79 | } 80 | }) 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /.github/workflows/pr-validation.yml: -------------------------------------------------------------------------------- 1 | name: PR Validation 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - master 7 | merge_group: 8 | 9 | jobs: 10 | test: 11 | strategy: 12 | matrix: 13 | os: [ubuntu-latest] 14 | 15 | runs-on: ${{ matrix.os }} 16 | 17 | steps: 18 | - name: Checkout code 19 | uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 20 | with: 21 | fetch-depth: 0 # Required for 2ms to have visibility to all commit history 22 | 23 | - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 24 | with: 25 | go-version-file: go.mod 26 | 27 | - name: go mod tidy 28 | run: | 29 | go mod tidy 30 | git diff --exit-code 31 | 32 | - name: install linter 33 | run: make get-linter 34 | 35 | - name: run linter 36 | run: make lint 37 | 38 | - name: Go Test 39 | run: go test -v ./... 40 | 41 | - name: Run 2ms Scan 42 | run: go run . git . --config .2ms.yml 43 | 44 | build: 45 | runs-on: ubuntu-latest 46 | steps: 47 | - name: Checkout code 48 | uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 49 | 50 | - name: Set up Docker Buildx 51 | uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb #v3.3.0 52 | 53 | - run: make build 54 | - name: docker run 55 | run: | 56 | docker run -v "$(pwd)":/repo -t checkmarx/2ms:latest git /repo --report-path output/results.json --ignore-on-exit results 57 | 58 | kics: 59 | runs-on: ubuntu-latest 60 | steps: 61 | - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 62 | - run: mkdir -p kics-results 63 | 64 | - name: Run KICS scan 65 | uses: checkmarx/kics-github-action@03c9abe351b01c3e4dbe60fa00ff79ee07d73f44 # master 66 | with: 67 | path: . 68 | output_path: kics-results 69 | output_formats: json,sarif 70 | enable_comments: ${{ github.event_name == 'pull_request'}} 71 | fail_on: high,medium 72 | enable_jobs_summary: true 73 | - name: Show KICS results 74 | if: failure() 75 | run: cat kics-results/results.json 76 | # - name: Upload SARIF file 77 | # uses: github/codeql-action/upload-sarif@4355270be187e1b672a7a1c7c7bae5afdc1ab94a #v3.24.10 78 | # with: 79 | # sarif_file: kics-results/results.sarif 80 | -------------------------------------------------------------------------------- /.github/workflows/ci-projects.yaml: -------------------------------------------------------------------------------- 1 | name: CI Projects 2 | on: 3 | pull_request: 4 | types: [closed] 5 | branches: 6 | - master 7 | 8 | env: 9 | ENGINE_VERSION: ${{ vars.ENGINE_VERSION }} 10 | PLATFORM: "LINUX_X64" 11 | ENGINE: "2ms" 12 | 13 | jobs: 14 | build: 15 | if: github.event.pull_request.merged == true 16 | runs-on: ubuntu-latest 17 | 18 | steps: 19 | - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 20 | with: 21 | ref: ${{ github.event.pull_request.merge_commit_sha }} 22 | path: 2ms 23 | 24 | - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 25 | with: 26 | go-version-file: 2ms/go.mod 27 | cache-dependency-path: 2ms/go.sum 28 | cache: true 29 | 30 | - name: Build 2ms Binary 31 | env: 32 | CGO_ENABLED: 0 33 | GOOS: linux 34 | GOARCH: amd64 35 | run: | 36 | cd $GITHUB_WORKSPACE/2ms 37 | go build -buildvcs=false -ldflags "-s -w" -a -o $GITHUB_WORKSPACE/2ms/dist/2ms main.go 38 | chmod +x $GITHUB_WORKSPACE/2ms/dist/2ms 39 | 40 | - name: Create Metadata File 41 | run: | 42 | COMMIT_TIMESTAMP=$(git -C "$GITHUB_WORKSPACE/2ms" log -1 --format=%ct) 43 | METADATA_PATH="$GITHUB_WORKSPACE/pr-metadata.json" 44 | echo '{ 45 | "seq": "'"${COMMIT_TIMESTAMP}"'", 46 | "tag": "'"${{ github.event.number }}"'", 47 | "comment": "'"${{ github.event.pull_request.title }}"'", 48 | "commit": "'"${{ github.sha }}"'", 49 | "owner": "'"${{ github.actor }}"'", 50 | "branch": "'"${{ github.base_ref }}"'", 51 | "engine": "'"${ENGINE}"'", 52 | "platform": "'"${PLATFORM}"'", 53 | "version": "'"${ENGINE_VERSION}"'" 54 | }' > "$METADATA_PATH" 55 | 56 | - name: Zip 2ms Folder 57 | run: | 58 | cd $GITHUB_WORKSPACE 59 | zip -qr 2ms.zip 2ms/ 60 | 61 | - name: Save 2ms 62 | uses: actions/upload-artifact@v4 63 | with: 64 | name: 2ms 65 | path: ${{ github.workspace }}/2ms.zip 66 | retention-days: 1 67 | 68 | - name: Pr parameters 69 | uses: actions/upload-artifact@v4 70 | with: 71 | name: Metadata 72 | path: ${{ github.workspace }}/pr-metadata.json 73 | retention-days: 1 74 | 75 | ci-projects: 76 | needs: build 77 | uses: ./.github/workflows/run-projects.yaml 78 | with: 79 | machines-count: 10 80 | secrets: inherit 81 | -------------------------------------------------------------------------------- /lib/utils/flags.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "strings" 7 | 8 | "github.com/rs/zerolog/log" 9 | "github.com/spf13/cobra" 10 | "github.com/spf13/pflag" 11 | "github.com/spf13/viper" 12 | ) 13 | 14 | func LoadConfig(v *viper.Viper, configFilePath string) error { 15 | if configFilePath == "" { 16 | return nil 17 | } 18 | 19 | configType := strings.TrimPrefix(filepath.Ext(configFilePath), ".") 20 | 21 | v.SetConfigType(configType) 22 | v.SetConfigFile(configFilePath) 23 | return v.ReadInConfig() 24 | } 25 | 26 | // TODO: can be a package 27 | 28 | // BindFlags fill flags values with config file or environment variables data 29 | func BindFlags(cmd *cobra.Command, v *viper.Viper, envPrefix string) error { 30 | commandHierarchy := getCommandHierarchy(cmd) 31 | 32 | bindFlag := func(f *pflag.Flag) { 33 | fullFlagName := fmt.Sprintf("%s%s", commandHierarchy, f.Name) 34 | bindEnvVarIntoViper(v, fullFlagName, envPrefix) 35 | 36 | if f.Changed { 37 | return 38 | } 39 | 40 | if v.IsSet(fullFlagName) { 41 | val := v.Get(fullFlagName) 42 | applyViperFlagToCommand(f, val) 43 | } 44 | } 45 | cmd.PersistentFlags().VisitAll(bindFlag) 46 | cmd.Flags().VisitAll(bindFlag) 47 | 48 | for _, subCmd := range cmd.Commands() { 49 | if err := BindFlags(subCmd, v, envPrefix); err != nil { 50 | return err 51 | } 52 | } 53 | 54 | return nil 55 | } 56 | 57 | func bindEnvVarIntoViper(v *viper.Viper, fullFlagName, envPrefix string) { 58 | envVarSuffix := strings.ToUpper(strings.ReplaceAll(strings.ReplaceAll(fullFlagName, "-", "_"), ".", "_")) 59 | envVarName := fmt.Sprintf("%s_%s", envPrefix, envVarSuffix) 60 | 61 | if err := v.BindEnv(fullFlagName, envVarName, strings.ToLower(envVarName)); err != nil { 62 | log.Err(err).Msg("Failed to bind Viper flags") 63 | } 64 | } 65 | 66 | func applyViperFlagToCommand(flag *pflag.Flag, val interface{}) { 67 | switch t := val.(type) { 68 | case []interface{}: 69 | for _, param := range t { 70 | if err := flag.Value.Set(param.(string)); err != nil { 71 | log.Err(err).Msg("Failed to set Viper flags") 72 | } 73 | } 74 | default: 75 | newVal := fmt.Sprintf("%v", val) 76 | if err := flag.Value.Set(newVal); err != nil { 77 | log.Err(err).Msg("Failed to set Viper flags") 78 | } 79 | } 80 | flag.Changed = true 81 | } 82 | 83 | func getCommandHierarchy(cmd *cobra.Command) string { 84 | names := []string{} 85 | if !cmd.HasParent() { 86 | return "" 87 | } 88 | 89 | for parent := cmd; parent.HasParent() && parent.Name() != ""; parent = parent.Parent() { 90 | names = append([]string{parent.Name()}, names...) 91 | } 92 | 93 | if len(names) == 0 { 94 | return "" 95 | } 96 | 97 | return strings.Join(names, ".") + "." 98 | } 99 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | run: 3 | tests: false 4 | linters: 5 | default: none 6 | enable: 7 | - bodyclose 8 | - dogsled 9 | - dupl 10 | - errcheck 11 | - forbidigo 12 | - funlen 13 | - gochecknoinits 14 | - goconst 15 | - gocritic 16 | - gocyclo 17 | - goprintffuncname 18 | - gosec 19 | - govet 20 | - ineffassign 21 | - lll 22 | - misspell 23 | - nakedret 24 | - noctx 25 | - nolintlint 26 | - rowserrcheck 27 | - staticcheck 28 | - unconvert 29 | - unparam 30 | - unused 31 | - whitespace 32 | settings: 33 | forbidigo: 34 | forbid: 35 | - pattern: ^fmt\.Print$ 36 | msg: "use structured logging instead of fmt.Print* functions" 37 | - pattern: ^log\.Fatal$ 38 | msg: "use proper error handling instead of log.Fatal* functions" 39 | exclude-godoc-examples: false 40 | analyze-types: false 41 | dupl: 42 | threshold: 100 43 | funlen: 44 | lines: 100 45 | statements: 50 46 | goconst: 47 | min-len: 2 48 | min-occurrences: 3 49 | gocritic: 50 | disabled-checks: 51 | - dupImport 52 | - ifElseChain 53 | - octalLiteral 54 | - whyNoLint 55 | - wrapperFunc 56 | - importShadow 57 | - unnamedResult 58 | enabled-tags: 59 | - diagnostic 60 | - experimental 61 | - opinionated 62 | - performance 63 | - style 64 | gocyclo: 65 | min-complexity: 15 66 | govet: 67 | settings: 68 | printf: 69 | funcs: 70 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof 71 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf 72 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf 73 | - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf 74 | lll: 75 | line-length: 140 76 | misspell: 77 | locale: US 78 | nolintlint: 79 | require-explanation: false 80 | require-specific: false 81 | allow-unused: false 82 | exclusions: 83 | generated: lax 84 | presets: 85 | - comments 86 | - common-false-positives 87 | - legacy 88 | - std-error-handling 89 | rules: 90 | - path: _test\.go 91 | linters: [ '*' ] 92 | paths: 93 | - third_party$ 94 | - builtin$ 95 | - examples$ 96 | formatters: 97 | enable: 98 | - gofmt 99 | - goimports 100 | settings: 101 | goimports: 102 | local-prefixes: 103 | - github.com/golangci/golangci-lint 104 | exclusions: 105 | generated: lax 106 | paths: 107 | - third_party$ 108 | - builtin$ 109 | - examples$ 110 | -------------------------------------------------------------------------------- /.github/workflows/cesar.yaml: -------------------------------------------------------------------------------- 1 | name: CESARt 2 | on: 3 | pull_request: 4 | types: [ labeled ] 5 | 6 | env: 7 | ENGINE_VERSION: ${{ vars.ENGINE_VERSION }} 8 | PLATFORM: "LINUX_X64" 9 | ENGINE: "2ms" 10 | REMOVE_HISTORY: "true" 11 | 12 | jobs: 13 | build: 14 | if: (github.event.label.name == 'cesar' && github.event.pull_request.mergeable == true) 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 19 | with: 20 | ref: refs/pull/${{ github.event.pull_request.number }}/merge 21 | path: 2ms 22 | 23 | - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 24 | with: 25 | go-version-file: 2ms/go.mod 26 | cache-dependency-path: 2ms/go.sum 27 | cache: true 28 | 29 | - name: Build 2ms Binary 30 | env: 31 | CGO_ENABLED: 0 32 | GOOS: linux 33 | GOARCH: amd64 34 | run: | 35 | cd $GITHUB_WORKSPACE/2ms 36 | go build -buildvcs=false -ldflags "-s -w" -a -o $GITHUB_WORKSPACE/2ms/dist/2ms main.go 37 | chmod +x $GITHUB_WORKSPACE/2ms/dist/2ms 38 | 39 | - name: Create Metadata File 40 | run: | 41 | COMMIT_TIMESTAMP=$(git -C "$GITHUB_WORKSPACE/2ms" log -1 --format=%ct) 42 | METADATA_PATH="$GITHUB_WORKSPACE/pr-metadata.json" 43 | CURR_TIMESTAMP=$(date +%s) 44 | echo '{ 45 | "seq": "'"${CURR_TIMESTAMP}"'", 46 | "tag": "'"${{ github.event.number }}"'", 47 | "comment": "'"${{ github.event.pull_request.title }}"'", 48 | "commit": "'"${{ github.event.pull_request.head.sha }}"'", 49 | "owner": "'"${{ github.actor }}"'", 50 | "branch": "'"${{ github.head_ref }}"'", 51 | "engine": "'"${ENGINE}"'", 52 | "platform": "'"${PLATFORM}"'", 53 | "version": "'"${ENGINE_VERSION}"'", 54 | "forkSeq": "'"${CURR_TIMESTAMP}"'", 55 | "forkBranch": "'"${{ github.base_ref }}"'", 56 | "removeHistory" : "'"${REMOVE_HISTORY}"'" 57 | }' > "$METADATA_PATH" 58 | 59 | - name: Zip 2ms Folder 60 | run: | 61 | cd $GITHUB_WORKSPACE 62 | zip -qr 2ms.zip 2ms/ 63 | 64 | - name: Save 2ms 65 | uses: actions/upload-artifact@v4 66 | with: 67 | name: 2ms 68 | path: ${{ github.workspace }}/2ms.zip 69 | retention-days: 1 70 | 71 | - name: Pr parameters 72 | uses: actions/upload-artifact@v4 73 | with: 74 | name: Metadata 75 | path: ${{ github.workspace }}/pr-metadata.json 76 | retention-days: 1 77 | 78 | ci-projects: 79 | needs: build 80 | uses: ./.github/workflows/run-projects.yaml 81 | with: 82 | machines-count: 10 83 | secrets: inherit 84 | -------------------------------------------------------------------------------- /engine/config.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/zricethezav/gitleaks/v8/config" 7 | ) 8 | 9 | // Taken from gitleaks config 10 | // https://github.com/gitleaks/gitleaks/blob/6c52f878cc48a513849900a9aa6f9d68e1c2dbdd/config/gitleaks.toml#L15-L26 11 | var baseConfig = config.Config{ 12 | Allowlists: []*config.Allowlist{ 13 | { 14 | Paths: []*regexp.Regexp{ 15 | regexp.MustCompile(`gitleaks\.toml`), 16 | regexp.MustCompile(`(?i)\.(?:bmp|gif|jpe?g|png|svg|tiff?)$`), 17 | regexp.MustCompile(`(?i)\.(?:eot|[ot]tf|woff2?)$`), 18 | regexp.MustCompile(`(?i)\.(?:docx?|xlsx?|pdf|bin|socket|vsidx|v2|suo|wsuo|.dll|pdb|exe|gltf)$`), 19 | regexp.MustCompile(`go\.(?:mod|sum|work(?:\.sum)?)$`), 20 | regexp.MustCompile(`(?:^|/)vendor/modules\.txt$`), 21 | regexp.MustCompile(`(?:^|/)vendor/(?:github\.com|golang\.org/x|google\.golang\.org|gopkg\.in|istio\.io|k8s\.io|sigs\.k8s\.io)(?:/.*)?$`), //nolint:lll 22 | regexp.MustCompile(`(?:^|/)gradlew(?:\.bat)?$`), 23 | regexp.MustCompile(`(?:^|/)gradle\.lockfile$`), 24 | regexp.MustCompile(`(?:^|/)mvnw(?:\.cmd)?$`), 25 | regexp.MustCompile(`(?:^|/)\.mvn/wrapper/MavenWrapperDownloader\.java$`), 26 | regexp.MustCompile(`(?:^|/)node_modules(?:/.*)?$`), 27 | regexp.MustCompile(`(?:^|/)(?:deno\.lock|npm-shrinkwrap\.json|package-lock\.json|pnpm-lock\.yaml|yarn\.lock)$`), 28 | regexp.MustCompile(`(?:^|/)bower_components(?:/.*)?$`), 29 | regexp.MustCompile(`(?:^|/)(?:angular|bootstrap|jquery(?:-?ui)?|plotly|swagger-?ui)[a-zA-Z0-9.-]*(?:\.min)?\.js(?:\.map)?$`), 30 | regexp.MustCompile(`(?:^|/)javascript\.json$`), 31 | regexp.MustCompile(`(?:^|/)(?:Pipfile|poetry)\.lock$`), 32 | regexp.MustCompile(`(?i)(?:^|/)(?:v?env|virtualenv)/lib(?:64)?(?:/.*)?$`), 33 | regexp.MustCompile(`(?i)(?:^|/)(?:lib(?:64)?/python[23](?:\.\d{1,2})+|python/[23](?:\.\d{1,2})+/lib(?:64)?)(?:/.*)?$`), 34 | regexp.MustCompile(`(?i)(?:^|/)[a-z0-9_.]+-[0-9.]+\.dist-info(?:/.+)?$`), 35 | regexp.MustCompile(`(?:^|/)vendor/(?:bundle|ruby)(?:/.*?)?$`), 36 | regexp.MustCompile(`\.gem$`), 37 | regexp.MustCompile(`verification-metadata\.xml`), 38 | regexp.MustCompile(`Database.refactorlog`), 39 | regexp.MustCompile(`(?:^|/)\.git$`), 40 | }, 41 | }, 42 | }, 43 | } 44 | 45 | func deepCopyConfig() *config.Config { 46 | dst := &config.Config{ 47 | Allowlists: make([]*config.Allowlist, len(baseConfig.Allowlists)), 48 | } 49 | 50 | for i, allowlist := range baseConfig.Allowlists { 51 | if allowlist == nil { 52 | dst.Allowlists[i] = nil 53 | continue 54 | } 55 | 56 | dst.Allowlists[i] = &config.Allowlist{ 57 | Paths: make([]*regexp.Regexp, len(allowlist.Paths)), 58 | } 59 | 60 | // Copy regexp pointers - regexp.Regexp is immutable after compilation 61 | // so sharing pointers is safe and efficient 62 | copy(dst.Allowlists[i].Paths, allowlist.Paths) 63 | } 64 | 65 | return dst 66 | } 67 | 68 | func newConfig() *config.Config { 69 | return deepCopyConfig() 70 | } 71 | -------------------------------------------------------------------------------- /pkg/testData/expectedReportWithIgnoredRule.json: -------------------------------------------------------------------------------- 1 | { 2 | "totalItemsScanned" : 3, 3 | "totalSecretsFound" : 3, 4 | "results" : { 5 | "dac14c6111d3a02a23c4fc31ee4759387a7395cd" : [ { 6 | "id" : "dac14c6111d3a02a23c4fc31ee4759387a7395cd", 7 | "source" : "testData/secrets/jwt.txt", 8 | "ruleId" : "jwt", 9 | "startLine" : 0, 10 | "endLine" : 0, 11 | "lineContent" : "TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1 TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 TextExample\r", 12 | "startColumn" : 129, 13 | "endColumn" : 232, 14 | "value" : "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2", 15 | "ruleDescription" : "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", 16 | "extraDetails" : { 17 | "secretDetails" : { 18 | "name" : "mockName2", 19 | "sub" : "mockSub2" 20 | } 21 | }, 22 | "cvssScore" : 8.2 23 | }, { 24 | "id" : "dac14c6111d3a02a23c4fc31ee4759387a7395cd", 25 | "source" : "testData/secrets/jwt.txt", 26 | "ruleId" : "jwt", 27 | "startLine" : 1, 28 | "endLine" : 1, 29 | "lineContent": " Text_Example = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2", 30 | "startColumn" : 63, 31 | "endColumn" : 166, 32 | "value" : "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2", 33 | "ruleDescription" : "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", 34 | "extraDetails" : { 35 | "secretDetails" : { 36 | "name" : "mockName2", 37 | "sub" : "mockSub2" 38 | } 39 | }, 40 | "cvssScore" : 8.2 41 | } ], 42 | "4b0fb9bf4c96bd11404f2a3b187acbb621d8ca0c" : [ { 43 | "id" : "4b0fb9bf4c96bd11404f2a3b187acbb621d8ca0c", 44 | "source" : "testData/secrets/jwt.txt", 45 | "ruleId" : "jwt", 46 | "startLine" : 0, 47 | "endLine" : 0, 48 | "lineContent" : "TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1 TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 TextExample\r", 49 | "startColumn" : 13, 50 | "endColumn" : 116, 51 | "value" : "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1", 52 | "ruleDescription" : "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", 53 | "extraDetails" : { 54 | "secretDetails" : { 55 | "name" : "mockName1", 56 | "sub" : "mockSub1" 57 | } 58 | }, 59 | "cvssScore" : 8.2 60 | } ] 61 | } 62 | } -------------------------------------------------------------------------------- /engine/validation/alibaba.go: -------------------------------------------------------------------------------- 1 | package validation 2 | 3 | import ( 4 | "context" 5 | "crypto/hmac" 6 | "crypto/sha1" //nolint:gosec // SHA1 is required by Alibaba API for HMAC-SHA1 signatures 7 | "encoding/base64" 8 | "fmt" 9 | "net/http" 10 | "net/url" 11 | "strconv" 12 | "strings" 13 | "time" 14 | 15 | "github.com/checkmarx/2ms/v4/lib/secrets" 16 | "github.com/rs/zerolog/log" 17 | ) 18 | 19 | // https://www.alibabacloud.com/help/en/sdk/alibaba-cloud-api-overview 20 | // https://www.alibabacloud.com/help/en/sdk/product-overview/rpc-mechanism#sectiondiv-y9b-x9s-wvp 21 | 22 | func validateAlibaba(secretsPairs pairsByRuleId) { 23 | accessKeys := secretsPairs["alibaba-access-key-id"] 24 | secretKeys := secretsPairs["alibaba-secret-key"] 25 | 26 | for _, accessKey := range accessKeys { 27 | accessKey.ValidationStatus = secrets.UnknownResult 28 | 29 | for _, secretKey := range secretKeys { 30 | status, err := alibabaRequest(accessKey.Value, secretKey.Value) 31 | if err != nil { 32 | log.Warn().Err(err).Str("service", "alibaba").Msg("Failed to validate secret") 33 | } 34 | 35 | secretKey.ValidationStatus = status 36 | if accessKey.ValidationStatus.CompareTo(status) > 0 { 37 | accessKey.ValidationStatus = status 38 | } 39 | } 40 | } 41 | } 42 | 43 | func alibabaRequest(accessKey, secretKey string) (secrets.ValidationResult, error) { 44 | req, err := http.NewRequestWithContext(context.Background(), "GET", "https://ecs.aliyuncs.com/", http.NoBody) 45 | if err != nil { 46 | return secrets.UnknownResult, err 47 | } 48 | 49 | // Workaround for gitleaks returns the key ends with " 50 | // https://github.com/gitleaks/gitleaks/pull/1350 51 | accessKey = strings.TrimSuffix(accessKey, "\"") 52 | secretKey = strings.TrimSuffix(secretKey, "\"") 53 | 54 | params := req.URL.Query() 55 | params.Add("AccessKeyId", accessKey) 56 | params.Add("Action", "DescribeRegions") 57 | params.Add("SignatureMethod", "HMAC-SHA1") 58 | params.Add("SignatureNonce", strconv.FormatInt(time.Now().UnixNano(), 10)) 59 | params.Add("SignatureVersion", "1.0") 60 | params.Add("Timestamp", time.Now().UTC().Format(time.RFC3339)) 61 | params.Add("Version", "2014-05-26") 62 | 63 | stringToSign := "GET&%2F&" + url.QueryEscape(params.Encode()) 64 | hmac := hmac.New(sha1.New, []byte(secretKey+"&")) 65 | hmac.Write([]byte(stringToSign)) 66 | signature := base64.StdEncoding.EncodeToString(hmac.Sum(nil)) 67 | 68 | params.Add("Signature", signature) 69 | req.URL.RawQuery = params.Encode() 70 | 71 | client := &http.Client{} 72 | resp, err := client.Do(req) 73 | if err != nil { 74 | return secrets.UnknownResult, err 75 | } 76 | defer resp.Body.Close() 77 | log.Debug().Str("service", "alibaba").Int("status_code", resp.StatusCode) 78 | 79 | // If the access key is invalid, the response will be 404 80 | // If the secret key is invalid, the response will be 400 along with other signautre Errors 81 | if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusBadRequest { 82 | return secrets.InvalidResult, nil 83 | } 84 | 85 | if resp.StatusCode == http.StatusOK { 86 | return secrets.ValidResult, nil 87 | } 88 | 89 | err = fmt.Errorf("unexpected status code: %d", resp.StatusCode) 90 | return secrets.UnknownResult, err 91 | } 92 | -------------------------------------------------------------------------------- /engine/chunk/chunk_mock.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: chunk.go 3 | // 4 | // Generated by this command: 5 | // 6 | // mockgen -source=chunk.go -destination=chunk_mock.go -package=chunk 7 | // 8 | 9 | // Package chunk is a generated GoMock package. 10 | package chunk 11 | 12 | import ( 13 | bufio "bufio" 14 | reflect "reflect" 15 | 16 | gomock "go.uber.org/mock/gomock" 17 | ) 18 | 19 | // MockIChunk is a mock of IChunk interface. 20 | type MockIChunk struct { 21 | ctrl *gomock.Controller 22 | recorder *MockIChunkMockRecorder 23 | isgomock struct{} 24 | } 25 | 26 | // MockIChunkMockRecorder is the mock recorder for MockIChunk. 27 | type MockIChunkMockRecorder struct { 28 | mock *MockIChunk 29 | } 30 | 31 | // NewMockIChunk creates a new mock instance. 32 | func NewMockIChunk(ctrl *gomock.Controller) *MockIChunk { 33 | mock := &MockIChunk{ctrl: ctrl} 34 | mock.recorder = &MockIChunkMockRecorder{mock} 35 | return mock 36 | } 37 | 38 | // EXPECT returns an object that allows the caller to indicate expected use. 39 | func (m *MockIChunk) EXPECT() *MockIChunkMockRecorder { 40 | return m.recorder 41 | } 42 | 43 | // GetFileThreshold mocks base method. 44 | func (m *MockIChunk) GetFileThreshold() int64 { 45 | m.ctrl.T.Helper() 46 | ret := m.ctrl.Call(m, "GetFileThreshold") 47 | ret0, _ := ret[0].(int64) 48 | return ret0 49 | } 50 | 51 | // GetFileThreshold indicates an expected call of GetFileThreshold. 52 | func (mr *MockIChunkMockRecorder) GetFileThreshold() *gomock.Call { 53 | mr.mock.ctrl.T.Helper() 54 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFileThreshold", reflect.TypeOf((*MockIChunk)(nil).GetFileThreshold)) 55 | } 56 | 57 | // GetMaxPeekSize mocks base method. 58 | func (m *MockIChunk) GetMaxPeekSize() int { 59 | m.ctrl.T.Helper() 60 | ret := m.ctrl.Call(m, "GetMaxPeekSize") 61 | ret0, _ := ret[0].(int) 62 | return ret0 63 | } 64 | 65 | // GetMaxPeekSize indicates an expected call of GetMaxPeekSize. 66 | func (mr *MockIChunkMockRecorder) GetMaxPeekSize() *gomock.Call { 67 | mr.mock.ctrl.T.Helper() 68 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMaxPeekSize", reflect.TypeOf((*MockIChunk)(nil).GetMaxPeekSize)) 69 | } 70 | 71 | // GetSize mocks base method. 72 | func (m *MockIChunk) GetSize() int { 73 | m.ctrl.T.Helper() 74 | ret := m.ctrl.Call(m, "GetSize") 75 | ret0, _ := ret[0].(int) 76 | return ret0 77 | } 78 | 79 | // GetSize indicates an expected call of GetSize. 80 | func (mr *MockIChunkMockRecorder) GetSize() *gomock.Call { 81 | mr.mock.ctrl.T.Helper() 82 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSize", reflect.TypeOf((*MockIChunk)(nil).GetSize)) 83 | } 84 | 85 | // ReadChunk mocks base method. 86 | func (m *MockIChunk) ReadChunk(reader *bufio.Reader, totalLines int) (string, error) { 87 | m.ctrl.T.Helper() 88 | ret := m.ctrl.Call(m, "ReadChunk", reader, totalLines) 89 | ret0, _ := ret[0].(string) 90 | ret1, _ := ret[1].(error) 91 | return ret0, ret1 92 | } 93 | 94 | // ReadChunk indicates an expected call of ReadChunk. 95 | func (mr *MockIChunkMockRecorder) ReadChunk(reader, totalLines any) *gomock.Call { 96 | mr.mock.ctrl.T.Helper() 97 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadChunk", reflect.TypeOf((*MockIChunk)(nil).ReadChunk), reader, totalLines) 98 | } 99 | -------------------------------------------------------------------------------- /cmd/main_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | "time" 7 | 8 | "github.com/checkmarx/2ms/v4/engine" 9 | "github.com/checkmarx/2ms/v4/internal/resources" 10 | "github.com/spf13/cobra" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestPreRun(t *testing.T) { 15 | tests := []struct { 16 | name string 17 | stdoutFormatVar string 18 | reportPath []string 19 | engineConfigVar engine.EngineConfig 20 | expectedInitErr error 21 | expectedPreRunErr error 22 | }{ 23 | { 24 | name: "error in validateFormat", 25 | stdoutFormatVar: "invalid", 26 | reportPath: []string{"report.json"}, 27 | engineConfigVar: engine.EngineConfig{}, 28 | expectedPreRunErr: errInvalidOutputFormat, 29 | }, 30 | { 31 | name: "error in engine.Init", 32 | stdoutFormatVar: "json", 33 | reportPath: []string{"mock.json"}, 34 | engineConfigVar: engine.EngineConfig{ 35 | SelectedList: []string{"mockInvalid"}, 36 | }, 37 | expectedInitErr: engine.ErrNoRulesSelected, 38 | }, 39 | { 40 | name: "successfully started go routines with validateVar enabled", 41 | stdoutFormatVar: "json", 42 | reportPath: []string{"mock.json"}, 43 | engineConfigVar: engine.EngineConfig{ 44 | ScanConfig: resources.ScanConfig{ 45 | WithValidation: true, 46 | }, 47 | }, 48 | expectedPreRunErr: nil, 49 | }, 50 | { 51 | name: "successfully started go routines with validateVar disabled", 52 | stdoutFormatVar: "json", 53 | reportPath: []string{"mock.json"}, 54 | engineConfigVar: engine.EngineConfig{}, 55 | expectedPreRunErr: nil, 56 | }, 57 | } 58 | 59 | for _, tt := range tests { 60 | t.Run(tt.name, func(t *testing.T) { 61 | stdoutFormatVar = tt.stdoutFormatVar 62 | reportPathVar = tt.reportPath 63 | engineConfigVar = tt.engineConfigVar 64 | 65 | engineInstance, err := engine.Init(&engineConfigVar) 66 | if tt.expectedInitErr != nil { 67 | assert.ErrorIs(t, err, tt.expectedInitErr) 68 | return 69 | } 70 | defer engineInstance.Shutdown() 71 | rootCmd := &cobra.Command{ 72 | Use: "2ms", 73 | Short: "2ms Secrets Detection", 74 | Long: "2ms Secrets Detection: A tool to detect secrets in public websites and communication services.", 75 | Version: Version, 76 | } 77 | 78 | time.AfterFunc(50*time.Millisecond, func() { 79 | close(engineInstance.GetPluginChannels().GetItemsCh()) 80 | }) 81 | err = preRun("mock", engineInstance, rootCmd, nil) 82 | assert.ErrorIs(t, err, tt.expectedPreRunErr) 83 | }) 84 | } 85 | } 86 | 87 | // TODO temporary, move it to organized integrations tests later 88 | func TestVersionFlagExitZero(t *testing.T) { 89 | // Preserve and restore process globals altered in the test. 90 | oldArgs := os.Args 91 | t.Cleanup(func() { 92 | os.Args = oldArgs 93 | }) 94 | 95 | // Simulate: 2ms --version 96 | os.Args = []string{"2ms", "--version"} 97 | 98 | code, err := Execute() 99 | if err != nil { 100 | t.Fatalf("expected no error, got: %v", err) 101 | } 102 | if code != 0 { 103 | t.Fatalf("expected exit code 0, got: %d", code) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /cmd/plugins_mock_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/checkmarx/2ms/v4/plugins (interfaces: ISourceItem) 3 | // 4 | // Generated by this command: 5 | // 6 | // mockgen -destination=plugins_mock_test.go -package=cmd github.com/checkmarx/2ms/v4/plugins ISourceItem 7 | // 8 | 9 | // Package cmd is a generated GoMock package. 10 | package cmd 11 | 12 | import ( 13 | reflect "reflect" 14 | 15 | plugins "github.com/checkmarx/2ms/v4/plugins" 16 | gomock "go.uber.org/mock/gomock" 17 | ) 18 | 19 | // MockISourceItem is a mock of ISourceItem interface. 20 | type MockISourceItem struct { 21 | ctrl *gomock.Controller 22 | recorder *MockISourceItemMockRecorder 23 | isgomock struct{} 24 | } 25 | 26 | // MockISourceItemMockRecorder is the mock recorder for MockISourceItem. 27 | type MockISourceItemMockRecorder struct { 28 | mock *MockISourceItem 29 | } 30 | 31 | // NewMockISourceItem creates a new mock instance. 32 | func NewMockISourceItem(ctrl *gomock.Controller) *MockISourceItem { 33 | mock := &MockISourceItem{ctrl: ctrl} 34 | mock.recorder = &MockISourceItemMockRecorder{mock} 35 | return mock 36 | } 37 | 38 | // EXPECT returns an object that allows the caller to indicate expected use. 39 | func (m *MockISourceItem) EXPECT() *MockISourceItemMockRecorder { 40 | return m.recorder 41 | } 42 | 43 | // GetContent mocks base method. 44 | func (m *MockISourceItem) GetContent() *string { 45 | m.ctrl.T.Helper() 46 | ret := m.ctrl.Call(m, "GetContent") 47 | ret0, _ := ret[0].(*string) 48 | return ret0 49 | } 50 | 51 | // GetContent indicates an expected call of GetContent. 52 | func (mr *MockISourceItemMockRecorder) GetContent() *gomock.Call { 53 | mr.mock.ctrl.T.Helper() 54 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetContent", reflect.TypeOf((*MockISourceItem)(nil).GetContent)) 55 | } 56 | 57 | // GetGitInfo mocks base method. 58 | func (m *MockISourceItem) GetGitInfo() *plugins.GitInfo { 59 | m.ctrl.T.Helper() 60 | ret := m.ctrl.Call(m, "GetGitInfo") 61 | ret0, _ := ret[0].(*plugins.GitInfo) 62 | return ret0 63 | } 64 | 65 | // GetGitInfo indicates an expected call of GetGitInfo. 66 | func (mr *MockISourceItemMockRecorder) GetGitInfo() *gomock.Call { 67 | mr.mock.ctrl.T.Helper() 68 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetGitInfo", reflect.TypeOf((*MockISourceItem)(nil).GetGitInfo)) 69 | } 70 | 71 | // GetID mocks base method. 72 | func (m *MockISourceItem) GetID() string { 73 | m.ctrl.T.Helper() 74 | ret := m.ctrl.Call(m, "GetID") 75 | ret0, _ := ret[0].(string) 76 | return ret0 77 | } 78 | 79 | // GetID indicates an expected call of GetID. 80 | func (mr *MockISourceItemMockRecorder) GetID() *gomock.Call { 81 | mr.mock.ctrl.T.Helper() 82 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetID", reflect.TypeOf((*MockISourceItem)(nil).GetID)) 83 | } 84 | 85 | // GetSource mocks base method. 86 | func (m *MockISourceItem) GetSource() string { 87 | m.ctrl.T.Helper() 88 | ret := m.ctrl.Call(m, "GetSource") 89 | ret0, _ := ret[0].(string) 90 | return ret0 91 | } 92 | 93 | // GetSource indicates an expected call of GetSource. 94 | func (mr *MockISourceItemMockRecorder) GetSource() *gomock.Call { 95 | mr.mock.ctrl.T.Helper() 96 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSource", reflect.TypeOf((*MockISourceItem)(nil).GetSource)) 97 | } 98 | -------------------------------------------------------------------------------- /engine/plugins_mock_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: github.com/checkmarx/2ms/v4/plugins (interfaces: ISourceItem) 3 | // 4 | // Generated by this command: 5 | // 6 | // mockgen -destination=plugins_mock_test.go -package=engine github.com/checkmarx/2ms/v4/plugins ISourceItem 7 | // 8 | 9 | // Package engine is a generated GoMock package. 10 | package engine 11 | 12 | import ( 13 | reflect "reflect" 14 | 15 | plugins "github.com/checkmarx/2ms/v4/plugins" 16 | gomock "go.uber.org/mock/gomock" 17 | ) 18 | 19 | // MockISourceItem is a mock of ISourceItem interface. 20 | type MockISourceItem struct { 21 | ctrl *gomock.Controller 22 | recorder *MockISourceItemMockRecorder 23 | isgomock struct{} 24 | } 25 | 26 | // MockISourceItemMockRecorder is the mock recorder for MockISourceItem. 27 | type MockISourceItemMockRecorder struct { 28 | mock *MockISourceItem 29 | } 30 | 31 | // NewMockISourceItem creates a new mock instance. 32 | func NewMockISourceItem(ctrl *gomock.Controller) *MockISourceItem { 33 | mock := &MockISourceItem{ctrl: ctrl} 34 | mock.recorder = &MockISourceItemMockRecorder{mock} 35 | return mock 36 | } 37 | 38 | // EXPECT returns an object that allows the caller to indicate expected use. 39 | func (m *MockISourceItem) EXPECT() *MockISourceItemMockRecorder { 40 | return m.recorder 41 | } 42 | 43 | // GetContent mocks base method. 44 | func (m *MockISourceItem) GetContent() *string { 45 | m.ctrl.T.Helper() 46 | ret := m.ctrl.Call(m, "GetContent") 47 | ret0, _ := ret[0].(*string) 48 | return ret0 49 | } 50 | 51 | // GetContent indicates an expected call of GetContent. 52 | func (mr *MockISourceItemMockRecorder) GetContent() *gomock.Call { 53 | mr.mock.ctrl.T.Helper() 54 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetContent", reflect.TypeOf((*MockISourceItem)(nil).GetContent)) 55 | } 56 | 57 | // GetGitInfo mocks base method. 58 | func (m *MockISourceItem) GetGitInfo() *plugins.GitInfo { 59 | m.ctrl.T.Helper() 60 | ret := m.ctrl.Call(m, "GetGitInfo") 61 | ret0, _ := ret[0].(*plugins.GitInfo) 62 | return ret0 63 | } 64 | 65 | // GetGitInfo indicates an expected call of GetGitInfo. 66 | func (mr *MockISourceItemMockRecorder) GetGitInfo() *gomock.Call { 67 | mr.mock.ctrl.T.Helper() 68 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetGitInfo", reflect.TypeOf((*MockISourceItem)(nil).GetGitInfo)) 69 | } 70 | 71 | // GetID mocks base method. 72 | func (m *MockISourceItem) GetID() string { 73 | m.ctrl.T.Helper() 74 | ret := m.ctrl.Call(m, "GetID") 75 | ret0, _ := ret[0].(string) 76 | return ret0 77 | } 78 | 79 | // GetID indicates an expected call of GetID. 80 | func (mr *MockISourceItemMockRecorder) GetID() *gomock.Call { 81 | mr.mock.ctrl.T.Helper() 82 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetID", reflect.TypeOf((*MockISourceItem)(nil).GetID)) 83 | } 84 | 85 | // GetSource mocks base method. 86 | func (m *MockISourceItem) GetSource() string { 87 | m.ctrl.T.Helper() 88 | ret := m.ctrl.Call(m, "GetSource") 89 | ret0, _ := ret[0].(string) 90 | return ret0 91 | } 92 | 93 | // GetSource indicates an expected call of GetSource. 94 | func (mr *MockISourceItemMockRecorder) GetSource() *gomock.Call { 95 | mr.mock.ctrl.T.Helper() 96 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSource", reflect.TypeOf((*MockISourceItem)(nil).GetSource)) 97 | } 98 | -------------------------------------------------------------------------------- /lib/reporting/report.go: -------------------------------------------------------------------------------- 1 | package reporting 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "strings" 7 | "sync" 8 | 9 | "github.com/checkmarx/2ms/v4/lib/config" 10 | "github.com/checkmarx/2ms/v4/lib/secrets" 11 | "github.com/rs/zerolog/log" 12 | ) 13 | 14 | const ( 15 | jsonFormat = "json" 16 | longYamlFormat = "yaml" 17 | shortYamlFormat = "yml" 18 | sarifFormat = "sarif" 19 | ) 20 | 21 | type IReport interface { 22 | ShowReport(format string, cfg *config.Config) error 23 | WriteFile(reportPath []string, cfg *config.Config) error 24 | GetOutput(format string, cfg *config.Config) (string, error) 25 | GetResults() map[string][]*secrets.Secret 26 | SetResults(results map[string][]*secrets.Secret) 27 | GetTotalItemsScanned() int 28 | GetTotalSecretsFound() int 29 | IncTotalItemsScanned(n int) 30 | IncTotalSecretsFound(n int) 31 | } 32 | 33 | type Report struct { 34 | TotalItemsScanned int `json:"totalItemsScanned"` 35 | TotalSecretsFound int `json:"totalSecretsFound"` 36 | Results map[string][]*secrets.Secret `json:"results"` 37 | 38 | mu sync.RWMutex 39 | } 40 | 41 | func New() IReport { 42 | return &Report{ 43 | Results: make(map[string][]*secrets.Secret), 44 | } 45 | } 46 | 47 | func (r *Report) ShowReport(format string, cfg *config.Config) error { 48 | output, err := r.GetOutput(format, cfg) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | log.Info().Msg("\n" + output) 54 | return nil 55 | } 56 | 57 | func (r *Report) WriteFile(reportPath []string, cfg *config.Config) error { 58 | for _, path := range reportPath { 59 | err := os.MkdirAll(filepath.Dir(path), 0750) 60 | if err != nil { 61 | return err 62 | } 63 | 64 | file, err := os.Create(path) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | fileExtension := filepath.Ext(path) 70 | format := strings.TrimPrefix(fileExtension, ".") 71 | output, err := r.GetOutput(format, cfg) 72 | if err != nil { 73 | return err 74 | } 75 | 76 | _, err = file.WriteString(output) 77 | if err != nil { 78 | return err 79 | } 80 | } 81 | return nil 82 | } 83 | 84 | func (r *Report) GetOutput(format string, cfg *config.Config) (string, error) { 85 | var output string 86 | var err error 87 | switch format { 88 | case jsonFormat: 89 | output, err = writeJson(r) 90 | case longYamlFormat, shortYamlFormat: 91 | output, err = writeYaml(r) 92 | case sarifFormat: 93 | output, err = writeSarif(r, cfg) 94 | } 95 | return output, err 96 | } 97 | 98 | func (r *Report) GetTotalItemsScanned() int { 99 | r.mu.RLock() 100 | defer r.mu.RUnlock() 101 | return r.TotalItemsScanned 102 | } 103 | 104 | func (r *Report) GetTotalSecretsFound() int { 105 | r.mu.RLock() 106 | defer r.mu.RUnlock() 107 | return r.TotalSecretsFound 108 | } 109 | 110 | func (r *Report) IncTotalItemsScanned(n int) { 111 | r.mu.Lock() 112 | defer r.mu.Unlock() 113 | r.TotalItemsScanned += n 114 | } 115 | 116 | func (r *Report) IncTotalSecretsFound(n int) { 117 | r.mu.Lock() 118 | defer r.mu.Unlock() 119 | r.TotalSecretsFound += n 120 | } 121 | 122 | func (r *Report) GetResults() map[string][]*secrets.Secret { 123 | r.mu.RLock() 124 | defer r.mu.RUnlock() 125 | return r.Results 126 | } 127 | 128 | func (r *Report) SetResults(results map[string][]*secrets.Secret) { 129 | r.mu.Lock() 130 | defer r.mu.Unlock() 131 | r.Results = results 132 | } 133 | -------------------------------------------------------------------------------- /plugins/filesystem.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/rs/zerolog/log" 9 | "github.com/spf13/cobra" 10 | ) 11 | 12 | const ( 13 | flagFolder = "path" 14 | flagProjectName = "project-name" 15 | flagIgnored = "ignore-pattern" 16 | ) 17 | 18 | var ignoredFolders = []string{".git"} 19 | 20 | type FileSystemPlugin struct { 21 | Plugin 22 | Path string 23 | ProjectName string 24 | Ignored []string 25 | } 26 | 27 | func (p *FileSystemPlugin) GetName() string { 28 | return "filesystem" 29 | } 30 | 31 | func (p *FileSystemPlugin) DefineCommand(items chan ISourceItem, errors chan error) (*cobra.Command, error) { 32 | var cmd = &cobra.Command{ 33 | Use: fmt.Sprintf("%s --%s PATH", p.GetName(), flagFolder), 34 | Short: "Scan local folder", 35 | Long: "Scan local folder for sensitive information", 36 | Run: func(cmd *cobra.Command, args []string) { 37 | log.Info().Msg("Folder plugin started") 38 | fileList, err := p.getFiles() 39 | if err != nil { 40 | errors <- err 41 | return 42 | } 43 | p.sendItems(items, fileList) 44 | }, 45 | } 46 | 47 | flags := cmd.Flags() 48 | flags.StringVar(&p.Path, flagFolder, "", "Local folder path [required]") 49 | if err := cmd.MarkFlagDirname(flagFolder); err != nil { 50 | return nil, fmt.Errorf("error while marking '%s' flag as directory: %w", flagFolder, err) 51 | } 52 | if err := cmd.MarkFlagRequired(flagFolder); err != nil { 53 | return nil, fmt.Errorf("error while marking '%s' flag as required: %w", flagFolder, err) 54 | } 55 | 56 | flags.StringSliceVar(&p.Ignored, flagIgnored, []string{}, "Pattern of a folder/file name to ignore") 57 | flags.StringVar(&p.ProjectName, flagProjectName, "", "Project name to differentiate between filesystem scans") 58 | 59 | return cmd, nil 60 | } 61 | 62 | func (p *FileSystemPlugin) getFiles() ([]string, error) { 63 | fileList := make([]string, 0) 64 | // TODO: use concurrency for directory walk 65 | err := filepath.Walk(p.Path, func(path string, fInfo os.FileInfo, err error) error { 66 | if err != nil { 67 | return err 68 | } 69 | for _, ignoredFolder := range ignoredFolders { 70 | if fInfo.Name() == ignoredFolder && fInfo.IsDir() { 71 | return filepath.SkipDir 72 | } 73 | } 74 | for _, ignoredPattern := range p.Ignored { 75 | matched, err := filepath.Match(ignoredPattern, filepath.Base(path)) 76 | if err != nil { 77 | return err 78 | } 79 | if matched && fInfo.IsDir() { 80 | return filepath.SkipDir 81 | } 82 | if matched { 83 | return nil 84 | } 85 | } 86 | if fInfo.Size() == 0 { 87 | return nil 88 | } 89 | if !fInfo.IsDir() { 90 | fileList = append(fileList, path) 91 | } 92 | return err 93 | }) 94 | 95 | if err != nil { 96 | return fileList, fmt.Errorf("error while walking through the directory: %w", err) 97 | } 98 | 99 | return fileList, nil 100 | } 101 | 102 | func (p *FileSystemPlugin) sendItems(items chan ISourceItem, fileList []string) { 103 | defer close(items) 104 | for _, filePath := range fileList { 105 | actualFile := p.getItem(filePath) 106 | items <- *actualFile 107 | } 108 | } 109 | 110 | func (p *FileSystemPlugin) getItem(filePath string) *item { 111 | log.Debug().Str("file", filePath).Msg("sending file item") 112 | 113 | item := &item{ 114 | ID: fmt.Sprintf("%s-%s-%s", p.GetName(), p.ProjectName, filePath), 115 | Source: filePath, 116 | } 117 | return item 118 | } 119 | -------------------------------------------------------------------------------- /plugins/paligo_test.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "golang.org/x/time/rate" 7 | "net/http" 8 | "testing" 9 | "time" 10 | ) 11 | 12 | func TestReserveRateLimit(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | response *http.Response 16 | limiter *rate.Limiter 17 | inputErr error 18 | expectedErrSub string 19 | expectedBurst int 20 | minSleep int64 21 | maxSleep int64 22 | }{ 23 | { 24 | name: "Non-429 status returns input error", 25 | response: &http.Response{ 26 | StatusCode: 200, 27 | }, 28 | limiter: rate.NewLimiter(1, 10), 29 | inputErr: fmt.Errorf("non rate limit error"), 30 | expectedErrSub: "non rate limit error", 31 | expectedBurst: 10, 32 | minSleep: 0, 33 | maxSleep: 0, 34 | }, 35 | { 36 | name: "429 status missing Retry-After header returns error", 37 | response: &http.Response{ 38 | StatusCode: 429, 39 | Header: http.Header{}, 40 | }, 41 | limiter: rate.NewLimiter(1, 10), 42 | inputErr: nil, 43 | expectedErrSub: "Retry-After header not found", 44 | expectedBurst: 10, 45 | minSleep: 0, 46 | maxSleep: 0, 47 | }, 48 | { 49 | name: "429 status with invalid Retry-After header returns error", 50 | response: &http.Response{ 51 | StatusCode: 429, 52 | Header: http.Header{ 53 | "Retry-After": []string{"abc"}, 54 | }, 55 | }, 56 | limiter: rate.NewLimiter(1, 10), 57 | inputErr: nil, 58 | expectedErrSub: "error parsing Retry-After header", 59 | expectedBurst: 10, 60 | minSleep: 0, 61 | maxSleep: 0, 62 | }, 63 | { 64 | name: "429 status with valid Retry-After header (0) returns nil and sets burst to 1 with minimal sleep", 65 | response: &http.Response{ 66 | StatusCode: 429, 67 | Header: http.Header{ 68 | "Retry-After": []string{"0"}, 69 | }, 70 | }, 71 | limiter: rate.NewLimiter(1, 10), 72 | inputErr: nil, 73 | expectedErrSub: "", 74 | expectedBurst: 1, 75 | minSleep: 0, 76 | maxSleep: 50, 77 | }, 78 | { 79 | name: "429 status with valid Retry-After header (1) returns nil and sets burst to 1 with ~1 sec sleep", 80 | response: &http.Response{ 81 | StatusCode: 429, 82 | Header: http.Header{ 83 | "Retry-After": []string{"1"}, 84 | }, 85 | }, 86 | limiter: rate.NewLimiter(1, 10), 87 | inputErr: nil, 88 | expectedErrSub: "", 89 | expectedBurst: 1, 90 | minSleep: 1000, 91 | maxSleep: 1050, 92 | }, 93 | } 94 | 95 | for _, tc := range tests { 96 | t.Run(tc.name, func(t *testing.T) { 97 | start := time.Now() 98 | err := reserveRateLimit(tc.response, tc.limiter, tc.inputErr) 99 | duration := time.Since(start).Milliseconds() 100 | 101 | if tc.expectedErrSub != "" { 102 | assert.Error(t, err, "expected an error") 103 | assert.Contains(t, err.Error(), tc.expectedErrSub, "error message mismatch") 104 | } else { 105 | assert.NoError(t, err, "expected no error") 106 | if tc.maxSleep > 0 { 107 | assert.GreaterOrEqual(t, duration, tc.minSleep, "expected sleep of at least %d ms", tc.minSleep) 108 | assert.Less(t, duration, tc.maxSleep, "expected sleep of less than %d ms", tc.maxSleep) 109 | } 110 | } 111 | 112 | assert.Equal(t, tc.expectedBurst, tc.limiter.Burst(), "limiter burst mismatch") 113 | }) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /engine/semaphore/semaphore.go: -------------------------------------------------------------------------------- 1 | package semaphore 2 | 3 | //go:generate mockgen -source=$GOFILE -destination=${GOPACKAGE}_mock.go -package=${GOPACKAGE} 4 | 5 | import ( 6 | "context" 7 | "fmt" 8 | "os" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/shirou/gopsutil/mem" 13 | "golang.org/x/sync/semaphore" 14 | ) 15 | 16 | type Semaphore struct { 17 | memoryBudget int64 18 | sem *semaphore.Weighted 19 | } 20 | 21 | type ISemaphore interface { 22 | AcquireMemoryWeight(ctx context.Context, weight int64) error 23 | ReleaseMemoryWeight(weight int64) 24 | } 25 | 26 | func NewSemaphore() *Semaphore { 27 | b := chooseMemoryBudget() 28 | return NewSemaphoreWithBudget(b) 29 | } 30 | 31 | func NewSemaphoreWithBudget(b int64) *Semaphore { 32 | return &Semaphore{ 33 | memoryBudget: b, 34 | sem: semaphore.NewWeighted(b), 35 | } 36 | } 37 | 38 | // AcquireMemoryWeight acquires semaphore with a specified weight 39 | func (s *Semaphore) AcquireMemoryWeight(ctx context.Context, weight int64) error { 40 | if weight > s.memoryBudget { 41 | return fmt.Errorf("buffer size %d exceeds memory budget %d", weight, s.memoryBudget) 42 | } 43 | if err := s.sem.Acquire(ctx, weight); err != nil { 44 | return fmt.Errorf("failed to acquire semaphore: %w", err) 45 | } 46 | return nil 47 | } 48 | 49 | // ReleaseMemoryWeight releases semaphore with a specified weight 50 | func (s *Semaphore) ReleaseMemoryWeight(weight int64) { 51 | s.sem.Release(weight) 52 | } 53 | 54 | // getCgroupMemoryLimit returns the memory cap imposed by cgroups in bytes 55 | func getCgroupMemoryLimit() uint64 { 56 | // Try cgroup v2: unified hierarchy 57 | if data, err := os.ReadFile("/sys/fs/cgroup/memory.max"); err == nil { 58 | s := strings.TrimSpace(string(data)) 59 | if s != "max" { 60 | if v, err := strconv.ParseUint(s, 10, 64); err == nil { 61 | return v 62 | } 63 | } 64 | } 65 | // Fallback cgroup v1 66 | if data, err := os.ReadFile("/sys/fs/cgroup/memory/memory.limit_in_bytes"); err == nil { 67 | if v, err := strconv.ParseUint(strings.TrimSpace(string(data)), 10, 64); err == nil { 68 | return v 69 | } 70 | } 71 | // No limit detected 72 | return ^uint64(0) // max uint64 73 | } 74 | 75 | // getTotalMemory returns the total physical RAM in bytes 76 | func getTotalMemory() uint64 { 77 | if vm, err := mem.VirtualMemory(); err == nil { 78 | return vm.Total 79 | } 80 | return ^uint64(0) // max uint64 81 | } 82 | 83 | // computeMemoryBudget computes the memory budget based on the host memory and cgroup limits 84 | func computeMemoryBudget(totalHost, cgroupLimit uint64) int64 { 85 | // Effective total = min(host, cgroup) 86 | var effectiveTotal uint64 87 | if totalHost < cgroupLimit { 88 | effectiveTotal = totalHost 89 | } else { 90 | effectiveTotal = cgroupLimit 91 | } 92 | 93 | // use 50% but cap to [256 MiB -> total − safety margin] 94 | safetyMargin := uint64(200 * 1024 * 1024) // reserve 200 MiB for OS/other processes 95 | avail := effectiveTotal 96 | if effectiveTotal > safetyMargin { 97 | avail = effectiveTotal - safetyMargin 98 | } 99 | budget := int64(avail / 2) //nolint:gosec // avail is guaranteed to be within safe range by design 100 | if budget < 256*1024*1024 { 101 | budget = 256 * 1024 * 1024 102 | } 103 | return budget 104 | } 105 | 106 | // chooseMemoryBudget picks 50% of total RAM (but at least 256 MiB) 107 | func chooseMemoryBudget() int64 { 108 | // Physical RAM 109 | totalHost := getTotalMemory() 110 | // Cgroup limit 111 | cgroupLimit := getCgroupMemoryLimit() 112 | 113 | return computeMemoryBudget(totalHost, cgroupLimit) 114 | } 115 | -------------------------------------------------------------------------------- /lib/utils/http_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "errors" 5 | "github.com/stretchr/testify/assert" 6 | "net/http" 7 | "net/http/httptest" 8 | "testing" 9 | ) 10 | 11 | type MockAuthorization struct { 12 | header string 13 | } 14 | 15 | func (m *MockAuthorization) GetAuthorizationHeader() string { 16 | return m.header 17 | } 18 | 19 | func TestHttpRequest(t *testing.T) { 20 | tests := []struct { 21 | name string 22 | method string 23 | url string 24 | statusCode int 25 | authorization string 26 | retry RetrySettings 27 | responseBody string 28 | bodyError bool 29 | expectedError error 30 | }{ 31 | { 32 | name: "Successful request", 33 | method: "GET", 34 | statusCode: http.StatusOK, 35 | responseBody: "Success", 36 | }, 37 | { 38 | name: "Request with authorization", 39 | method: "GET", 40 | statusCode: http.StatusOK, 41 | authorization: "Bearer token123", 42 | responseBody: "Authorized", 43 | }, 44 | { 45 | name: "Retry on failure", 46 | method: "GET", 47 | statusCode: http.StatusInternalServerError, 48 | retry: RetrySettings{MaxRetries: 1, ErrorCodes: []int{http.StatusInternalServerError}}, 49 | expectedError: errors.New("error calling http url"), 50 | }, 51 | { 52 | name: "Client error (no retry)", 53 | method: "GET", 54 | statusCode: http.StatusBadRequest, 55 | expectedError: errors.New("error calling http url"), 56 | }, 57 | { 58 | name: "Error creating request", 59 | method: "GET", 60 | url: "::://invalid-url", 61 | expectedError: errors.New("unexpected error creating an http request"), 62 | }, 63 | { 64 | name: "Error sending request", 65 | method: "GET", 66 | url: "http://localhost:9999", 67 | expectedError: errors.New("unable to send http request"), 68 | }, 69 | { 70 | name: "Error reading response body", 71 | method: "GET", 72 | statusCode: http.StatusOK, 73 | bodyError: true, 74 | expectedError: errors.New("unexpected error reading http response body"), 75 | }, 76 | } 77 | 78 | for _, test := range tests { 79 | t.Run(test.name, func(t *testing.T) { 80 | var server *httptest.Server 81 | if test.url == "" { 82 | server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 83 | if test.authorization != "" { 84 | assert.Equal(t, test.authorization, r.Header.Get("Authorization"), "Authorization header mismatch") 85 | } 86 | w.WriteHeader(test.statusCode) 87 | if test.bodyError { 88 | _, err := w.Write([]byte("corrupt data")) 89 | assert.NoError(t, err) 90 | w.(http.Flusher).Flush() 91 | conn, _, _ := w.(http.Hijacker).Hijack() 92 | err = conn.Close() 93 | assert.NoError(t, err) 94 | } else { 95 | _, _ = w.Write([]byte(test.responseBody)) 96 | } 97 | })) 98 | test.url = server.URL 99 | defer server.Close() 100 | } 101 | 102 | mockAuth := &MockAuthorization{header: test.authorization} 103 | body, response, err := HttpRequest(test.method, test.url, mockAuth, test.retry) 104 | 105 | if test.expectedError != nil { 106 | assert.Error(t, err, "Expected an error but got none") 107 | assert.Contains(t, err.Error(), test.expectedError.Error(), "Unexpected error message") 108 | } else { 109 | assert.NoError(t, err, "Unexpected error occurred") 110 | assert.Equal(t, test.statusCode, response.StatusCode, "Unexpected status code") 111 | assert.Equal(t, test.responseBody, string(body), "Unexpected response body") 112 | } 113 | }) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /engine/rules/utils.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | ) 8 | 9 | const ( 10 | // case insensitive prefix 11 | caseInsensitive = `(?i)` 12 | 13 | // identifier prefix (just an ignore group) 14 | identifierCaseInsensitivePrefix = `[\w.-]{0,50}?(?i:` 15 | identifierCaseInsensitiveSuffix = `)` 16 | identifierPrefix = `[\w.-]{0,50}?(?:` 17 | identifierSuffix = `)(?:[0-9a-z\-_\t .]{0,20})(?:[\s|']|[\s|"]){0,3}` 18 | identifierSuffixIncludingXml = `)(?:[0-9a-z\-_\t .]{0,20})(?:<\/key>\s{0,10}<string)?(?:[\s|']|[\s|"]){0,3}` 19 | 20 | // commonly used assignment operators or function call 21 | // operator = `(?:=|>|:{1,3}=|\|\|:|<=|=>|:|\?=)` 22 | operator = `(?:=|>|:{1,3}=|\|\||:|=>|\?=|,)` 23 | 24 | // boundaries for the secret 25 | // \x60 = ` 26 | secretPrefixUnique = `\b(` 27 | secretPrefix = `[\x60'"\s=]{0,20}(` //nolint:gosec // This is a regex pattern 28 | secretSuffix = `)(?:[\x60'"\s;]|\\[nr]|$)` //nolint:gosec // This is a regex pattern 29 | secretSuffixIncludingXml = `)(?:['|\"|\n|\r|\s|\x60|;]|\\n|\\r|$|\s{0,10}<\/string>)` //nolint:gosec // This is a regex pattern 30 | ) 31 | 32 | func generateSemiGenericRegex(identifiers []string, secretRegex string, isCaseInsensitive bool) *regexp.Regexp { 33 | var sb strings.Builder 34 | // The identifiers should always be case-insensitive. 35 | // This is inelegant but prevents an extraneous `(?i:)` from being added to the pattern; it could be removed. 36 | if isCaseInsensitive { 37 | sb.WriteString(caseInsensitive) 38 | writeIdentifiers(&sb, identifiers) 39 | } else { 40 | sb.WriteString(identifierCaseInsensitivePrefix) 41 | writeIdentifiers(&sb, identifiers) 42 | sb.WriteString(identifierCaseInsensitiveSuffix) 43 | } 44 | sb.WriteString(operator) 45 | sb.WriteString(secretPrefix) 46 | sb.WriteString(secretRegex) 47 | sb.WriteString(secretSuffix) 48 | return regexp.MustCompile(sb.String()) 49 | } 50 | 51 | func writeIdentifiers(sb *strings.Builder, identifiers []string) { 52 | sb.WriteString(identifierPrefix) 53 | sb.WriteString(strings.Join(identifiers, "|")) 54 | sb.WriteString(identifierSuffix) 55 | } 56 | 57 | func generateUniqueTokenRegex(secretRegex string, isCaseInsensitive bool) *regexp.Regexp { 58 | var sb strings.Builder 59 | if isCaseInsensitive { 60 | sb.WriteString(caseInsensitive) 61 | } 62 | sb.WriteString(secretPrefixUnique) 63 | sb.WriteString(secretRegex) 64 | sb.WriteString(secretSuffix) 65 | return regexp.MustCompile(sb.String()) 66 | } 67 | 68 | func alphaNumeric(size string) string { 69 | return fmt.Sprintf(`[a-z0-9]{%s}`, size) 70 | } 71 | 72 | // generateSemiGenericRegexIncludingXml generates a regex that includes XML detection patterns 73 | func generateSemiGenericRegexIncludingXml(identifiers []string, secretRegex string, isCaseInsensitive bool) *regexp.Regexp { 74 | var sb strings.Builder 75 | // The identifiers should always be case-insensitive. 76 | // This is inelegant but prevents an extraneous `(?i:)` from being added to the pattern; it could be removed. 77 | if isCaseInsensitive { 78 | sb.WriteString(caseInsensitive) 79 | writeIdentifiersIncludingXml(&sb, identifiers) 80 | } else { 81 | sb.WriteString(identifierCaseInsensitivePrefix) 82 | writeIdentifiersIncludingXml(&sb, identifiers) 83 | sb.WriteString(identifierCaseInsensitiveSuffix) 84 | } 85 | sb.WriteString(operator) 86 | sb.WriteString(secretPrefix) 87 | sb.WriteString(secretRegex) 88 | sb.WriteString(secretSuffixIncludingXml) 89 | return regexp.MustCompile(sb.String()) 90 | } 91 | 92 | func writeIdentifiersIncludingXml(sb *strings.Builder, identifiers []string) { 93 | sb.WriteString(identifierPrefix) 94 | sb.WriteString(strings.Join(identifiers, "|")) 95 | sb.WriteString(identifierSuffixIncludingXml) 96 | } 97 | -------------------------------------------------------------------------------- /.github/workflows/run-projects.yaml: -------------------------------------------------------------------------------- 1 | name: Run CI Projects 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | machines-count: 7 | description: 'Total number of machines' 8 | required: true 9 | type: number 10 | 11 | env: 12 | ENGINE: "2ms" 13 | CES_ENVIRONMENT: "prod" 14 | 15 | jobs: 16 | setup: 17 | runs-on: ubuntu-latest 18 | outputs: 19 | machines: ${{ steps.set-machines.outputs.machines }} 20 | steps: 21 | - name: Generate Machine Matrix 22 | id: set-machines 23 | run: | 24 | machines=$(seq -s, 0 $((${{ inputs.machines-count }} - 1))) 25 | echo "machines=[$machines]" >> "$GITHUB_OUTPUT" 26 | 27 | run-projects: 28 | needs: setup 29 | runs-on: ubuntu-latest 30 | env: 31 | AWS_ACCESS_KEY_ID: ${{ secrets.CES_BUCKET_AWS_ACCESS_KEY }} 32 | AWS_SECRET_ACCESS_KEY: ${{ secrets.CES_BUCKET_AWS_SECRET_ACCESS_KEY }} 33 | AWS_REGION: ${{ secrets.CES_BUCKET_AWS_REGION }} 34 | 35 | strategy: 36 | fail-fast: false 37 | max-parallel: 10 38 | matrix: 39 | machine: ${{ fromJSON(needs.setup.outputs.machines) }} 40 | 41 | steps: 42 | - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 43 | with: 44 | repository: ${{ secrets.CES_EXECUTOR_REPO }} 45 | token: ${{ secrets.CX_CEBOT_GITHUB_TOKEN_CHECKMARX }} 46 | path: cli 47 | ref: master 48 | 49 | - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 50 | with: 51 | go-version-file: cli/go.mod 52 | cache: false 53 | 54 | - name: Download 2ms 55 | uses: actions/download-artifact@v4 56 | with: 57 | name: 2ms 58 | path: . 59 | 60 | - name: Unzip 2ms 61 | run: | 62 | unzip -q 2ms.zip 63 | 64 | - name: Download Json 65 | uses: actions/download-artifact@v4 66 | with: 67 | name: Metadata 68 | path: . 69 | 70 | - name: Build Engines Executor 71 | run: | 72 | cd cli 73 | go build -o executor 74 | 75 | - name: Set log file 76 | run: | 77 | LOG_FILE="$GITHUB_WORKSPACE/log_${{ matrix.machine }}.log" 78 | echo "LOG_FILE=$LOG_FILE" >> $GITHUB_ENV 79 | 80 | - name: Select Projects 81 | run: | 82 | mkdir -p "$GITHUB_WORKSPACE/zips/" 83 | cd cli 84 | ./executor sources \ 85 | -s $GITHUB_WORKSPACE/zips/ \ 86 | -e $ENGINE \ 87 | --chunk ${{ matrix.machine }} \ 88 | --machines ${{ inputs.machines-count }} \ 89 | >> $LOG_FILE 2>&1 90 | 91 | - name: Prepare Projects 92 | run: | 93 | cd "$GITHUB_WORKSPACE/zips/" 94 | for zip in *.zip; do 95 | [ -e "$zip" ] || continue 96 | zip_name=$(basename "$zip" .zip) 97 | echo "::add-mask::$zip_name" 98 | unzip -qqo "$zip" -d "./$zip_name" >> $LOG_FILE 2>&1 99 | done 100 | 101 | - name: Run Engines Executor 102 | run: | 103 | mkdir -p $GITHUB_WORKSPACE/results 104 | ./cli/executor run 2ms \ 105 | -b $GITHUB_WORKSPACE/2ms/dist/2ms \ 106 | -s $GITHUB_WORKSPACE/zips/ \ 107 | -r $GITHUB_WORKSPACE/results \ 108 | -j $GITHUB_WORKSPACE/pr-metadata.json \ 109 | --env $CES_ENVIRONMENT \ 110 | >> $LOG_FILE 2>&1 111 | 112 | - name: Upload log 113 | if: failure() 114 | run: | 115 | ./cli/executor save-log \ 116 | -e $ENGINE \ 117 | -j $GITHUB_WORKSPACE/pr-metadata.json \ 118 | -l $LOG_FILE \ 119 | --env $CES_ENVIRONMENT \ 120 | > /dev/null 2>&1 121 | -------------------------------------------------------------------------------- /cmd/exit_handler_test.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "testing" 7 | ) 8 | 9 | func TestExitHandler_IsNeedReturnErrorCode(t *testing.T) { 10 | 11 | var onErrorsTests = []struct { 12 | userInput ignoreOnExit 13 | expectedResult bool 14 | }{ 15 | { 16 | userInput: ignoreOnExitNone, 17 | expectedResult: true, 18 | }, 19 | { 20 | userInput: ignoreOnExitAll, 21 | expectedResult: false, 22 | }, 23 | { 24 | userInput: ignoreOnExitResults, 25 | expectedResult: true, 26 | }, 27 | { 28 | userInput: ignoreOnExitErrors, 29 | expectedResult: false, 30 | }, 31 | } 32 | 33 | for idx, testCase := range onErrorsTests { 34 | t.Run(fmt.Sprintf("Print test case %d", idx), func(t *testing.T) { 35 | ignoreOnExitVar = testCase.userInput 36 | result := isNeedReturnErrorCodeFor("errors") 37 | if result != testCase.expectedResult { 38 | t.Errorf("Expected %v, got %v", testCase.expectedResult, result) 39 | } 40 | }) 41 | } 42 | 43 | var onResultsTests = []struct { 44 | userInput ignoreOnExit 45 | expectedResult bool 46 | }{ 47 | { 48 | userInput: ignoreOnExitNone, 49 | expectedResult: true, 50 | }, 51 | { 52 | userInput: ignoreOnExitAll, 53 | expectedResult: false, 54 | }, 55 | { 56 | userInput: ignoreOnExitResults, 57 | expectedResult: false, 58 | }, 59 | { 60 | userInput: ignoreOnExitErrors, 61 | expectedResult: true, 62 | }, 63 | } 64 | 65 | for idx, testCase := range onResultsTests { 66 | t.Run(fmt.Sprintf("Print test case %d", idx), func(t *testing.T) { 67 | ignoreOnExitVar = testCase.userInput 68 | result := isNeedReturnErrorCodeFor("results") 69 | if result != testCase.expectedResult { 70 | t.Errorf("Expected %v, got %v", testCase.expectedResult, result) 71 | } 72 | }) 73 | } 74 | } 75 | 76 | func TestExitCodeIfError(t *testing.T) { 77 | testCases := []struct { 78 | name string 79 | err error 80 | ignoreOnExit ignoreOnExit 81 | expectedCode int 82 | }{ 83 | { 84 | name: "No error, ignoreOnExitNone", 85 | err: nil, 86 | ignoreOnExit: ignoreOnExitNone, 87 | expectedCode: 0, 88 | }, 89 | { 90 | name: "Error present, ignoreOnExitNone", 91 | err: fmt.Errorf("sample error"), 92 | ignoreOnExit: ignoreOnExitNone, 93 | expectedCode: errorCode, 94 | }, 95 | { 96 | name: "Error present, ignoreOnExitAll", 97 | err: fmt.Errorf("sample error"), 98 | ignoreOnExit: ignoreOnExitAll, 99 | expectedCode: 0, 100 | }, 101 | } 102 | 103 | for _, tc := range testCases { 104 | t.Run(tc.name, func(t *testing.T) { 105 | ignoreOnExitVar = tc.ignoreOnExit 106 | code := exitCodeIfError(tc.err) 107 | assert.Equal(t, tc.expectedCode, code) 108 | }) 109 | } 110 | } 111 | 112 | func TestExitCodeIfResults(t *testing.T) { 113 | testCases := []struct { 114 | name string 115 | resultsCount int 116 | ignoreOnExit ignoreOnExit 117 | expectedCode int 118 | }{ 119 | { 120 | name: "No results, ignoreOnExitNone", 121 | resultsCount: 0, 122 | ignoreOnExit: ignoreOnExitNone, 123 | expectedCode: 0, 124 | }, 125 | { 126 | name: "Results present, ignoreOnExitNone", 127 | resultsCount: 5, 128 | ignoreOnExit: ignoreOnExitNone, 129 | expectedCode: resultsCode, 130 | }, 131 | { 132 | name: "Results present, ignoreOnExitAll", 133 | resultsCount: 5, 134 | ignoreOnExit: ignoreOnExitAll, 135 | expectedCode: 0, 136 | }, 137 | } 138 | 139 | for _, tc := range testCases { 140 | t.Run(tc.name, func(t *testing.T) { 141 | ignoreOnExitVar = tc.ignoreOnExit 142 | code := exitCodeIfResults(tc.resultsCount) 143 | assert.Equal(t, tc.expectedCode, code) 144 | }) 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL=/bin/bash 2 | 3 | image_label ?= latest 4 | image_name ?= checkmarx/2ms:$(image_label) 5 | image_file_name ?= checkmarx-2ms-$(image_label).tar 6 | 7 | GREEN := $(shell printf "\033[32m") 8 | RED := $(shell printf "\033[31m") 9 | RESET := $(shell printf "\033[0m") 10 | 11 | COVERAGE_REQUIRED := 55 12 | MOCKGEN_VERSION := 0.5.2 13 | LINTER_VERSION ?= latest 14 | 15 | .PHONY: lint 16 | lint: check-linter-version 17 | go fmt ./... 18 | golangci-lint run -c ./.golangci.yml 19 | 20 | get-linter: 21 | command -v golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ 22 | | sh -s -- -b $(shell go env GOPATH)/bin $(LINTER_VERSION) 23 | 24 | modtidy: 25 | go mod tidy 26 | go mod vendor 27 | 28 | .PHONY: test 29 | test: 30 | go test -race -vet all -coverprofile=cover.out.tmp ./... 31 | grep -v -e "_mock\.go:" -e "/mocks/" -e "/docs/" cover.out.tmp > cover.out 32 | go tool cover -func=cover.out 33 | rm cover.out.tmp 34 | 35 | save: build 36 | docker save $(image_name) > $(image_file_name) 37 | 38 | build: 39 | docker build -t $(image_name) . 40 | 41 | build-local: 42 | GOOS=linux GOARCH=amd64 go build -buildvcs=false -ldflags="-s -w" -a -o ./2ms . 43 | 44 | generate: check-mockgen-version 45 | go generate ./... 46 | 47 | check: lint test coverage-check 48 | 49 | .PHONY: coverage-check 50 | coverage-check: test 51 | @coverage=$$(go tool cover -func=cover.out | grep '^total:' | awk '{print $$3}' | sed 's/%//g'); \ 52 | if awk "BEGIN {exit !($$coverage < $(COVERAGE_REQUIRED))}"; then \ 53 | echo "error: coverage ($$coverage%) must be at least $(COVERAGE_REQUIRED)%"; \ 54 | exit 1; \ 55 | else \ 56 | echo "test coverage: $$coverage% (threshold: $(COVERAGE_REQUIRED)%)"; \ 57 | fi 58 | 59 | .PHONY: test-coverage 60 | test-coverage: test coverage-check 61 | 62 | ## cover-report: show html report 63 | ## If you don't have the cover.out file yet, just run the tests with make test 64 | cover-report: 65 | go tool cover -html=cover.out 66 | .PHONY: coverage-check 67 | 68 | check-mockgen-version: 69 | @echo "Checking mockgen version..." 70 | @if command -v mockgen >/dev/null 2>&1; then \ 71 | INSTALLED_VERSION=$$(mockgen -version | grep -oE '[0-9]+\.[0-9]+\.[0-9]+'); \ 72 | if [ "$$INSTALLED_VERSION" = "$(MOCKGEN_VERSION)" ]; then \ 73 | echo "$(GREEN)[OK]$(RESET) mockgen version $(MOCKGEN_VERSION) is installed"; \ 74 | else \ 75 | echo "$(RED)[ERROR]$(RESET) Wrong mockgen version: $$INSTALLED_VERSION (required: $(MOCKGEN_VERSION))"; \ 76 | echo "Please install the correct version using:"; \ 77 | echo " go install go.uber.org/mock/mockgen@v$(MOCKGEN_VERSION)"; \ 78 | exit 1; \ 79 | fi; \ 80 | else \ 81 | echo "$(RED)[ERROR]$(RESET) mockgen is not installed"; \ 82 | echo "Please install it using:"; \ 83 | echo " go install go.uber.org/mock/mockgen@v$(MOCKGEN_VERSION)"; \ 84 | exit 1; \ 85 | fi 86 | 87 | check-linter-version: 88 | @echo "Checking golangci-lint version..." 89 | @if ! command -v golangci-lint >/dev/null 2>&1; then \ 90 | echo "$(RED)[ERROR]$(RESET) golangci-lint is not installed"; \ 91 | echo "Please install it using:"; \ 92 | echo " make get-linter"; \ 93 | exit 1; \ 94 | fi; \ 95 | INSTALLED_VERSION=$$(golangci-lint --version | grep -oE 'version [0-9]+\.[0-9]+\.[0-9]+' | cut -d' ' -f2); \ 96 | if [ "$(LINTER_VERSION)" = "latest" ]; then \ 97 | echo "$(GREEN)[OK]$(RESET) golangci-lint $$INSTALLED_VERSION is installed (latest accepted)"; \ 98 | else \ 99 | if [ "$$INSTALLED_VERSION" = "$(LINTER_VERSION)" ]; then \ 100 | echo "$(GREEN)[OK]$(RESET) golangci-lint version $(LINTER_VERSION) is installed"; \ 101 | else \ 102 | echo "$(RED)[ERROR]$(RESET) Wrong golangci-lint version: $$INSTALLED_VERSION (required: $(LINTER_VERSION))"; \ 103 | echo "Please install the correct version using:"; \ 104 | echo " make get-linter"; \ 105 | exit 1; \ 106 | fi; \ 107 | fi -------------------------------------------------------------------------------- /benches/test_data.go: -------------------------------------------------------------------------------- 1 | package benches 2 | 3 | // SecretPatterns contains realistic secret patterns that will trigger detection 4 | var SecretPatterns = []string{ 5 | "github_pat_11ABCDEFG1234567890abcdefghijklmnopqrstuvwxyz123456", 6 | "sk-1234567890abcdefghijklmnopqrstuvwxyz", 7 | "ghp_abcdefghijklmnopqrstuvwxyz1234567890", 8 | "AIzaSyC1234567890abcdefghijklmnopqrstuv", 9 | "xoxb-123456789012-1234567890123-abcdefghijklmnopqrstuvwx", 10 | "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c", //nolint:lll 11 | } 12 | 13 | // ContentTemplates contains realistic file content templates simulating different file types 14 | // Templates 0-4 contain secrets, template 5 is clean code 15 | var ContentTemplates = []string{ 16 | // JavaScript config file 17 | `const config = { 18 | apiKey: '%s', 19 | endpoint: 'https://api.example.com', 20 | timeout: 5000, 21 | retries: 3, 22 | debug: process.env.NODE_ENV === 'development' 23 | }; 24 | 25 | module.exports = config;`, 26 | 27 | // Python script 28 | `import requests 29 | import os 30 | 31 | API_KEY = '%s' 32 | BASE_URL = 'https://api.service.com/v1' 33 | 34 | def make_request(endpoint): 35 | headers = { 36 | 'Authorization': f'Bearer {API_KEY}', 37 | 'Content-Type': 'application/json' 38 | } 39 | return requests.get(f'{BASE_URL}/{endpoint}', headers=headers) 40 | 41 | if __name__ == '__main__': 42 | response = make_request('users') 43 | print(response.json())`, 44 | 45 | // Shell script 46 | `#!/bin/bash 47 | 48 | # Configuration 49 | export API_TOKEN='%s' 50 | export SERVICE_URL="https://service.example.com" 51 | export ENVIRONMENT="production" 52 | 53 | # Function to call API 54 | call_api() { 55 | curl -H "Authorization: Bearer $API_TOKEN" \ 56 | -H "Content-Type: application/json" \ 57 | "$SERVICE_URL/api/$1" 58 | } 59 | 60 | # Main execution 61 | call_api "status"`, 62 | 63 | // YAML config 64 | `apiVersion: v1 65 | kind: ConfigMap 66 | metadata: 67 | name: app-config 68 | data: 69 | database_url: postgresql://user:pass@localhost/db 70 | api_key: %s 71 | redis_url: redis://localhost:6379 72 | log_level: info`, 73 | 74 | // JSON config 75 | `{ 76 | "name": "production-app", 77 | "version": "1.0.0", 78 | "config": { 79 | "api": { 80 | "key": "%s", 81 | "endpoint": "https://api.production.com", 82 | "timeout": 30000 83 | }, 84 | "database": { 85 | "host": "db.production.com", 86 | "port": 5432 87 | } 88 | } 89 | }`, 90 | 91 | // No secret - regular Go code 92 | `package utils 93 | 94 | import ( 95 | "fmt" 96 | "strings" 97 | "time" 98 | ) 99 | 100 | func ProcessData(input string) (string, error) { 101 | if input == "" { 102 | return "", fmt.Errorf("input cannot be empty") 103 | } 104 | 105 | processed := strings.ToUpper(input) 106 | timestamp := time.Now().Format(time.RFC3339) 107 | 108 | return fmt.Sprintf("%s - %s", processed, timestamp), nil 109 | } 110 | 111 | func ValidateInput(data []byte) bool { 112 | return len(data) > 0 && len(data) < 1048576 113 | }`, 114 | } 115 | 116 | // FileExtensions maps content template indices to appropriate file extensions 117 | var FileExtensions = []string{ 118 | ".js", // JavaScript config 119 | ".py", // Python script 120 | ".sh", // Shell script 121 | ".yml", // YAML config 122 | ".json", // JSON config 123 | ".go", // Go code (no secret) 124 | } 125 | 126 | // PaddingPatterns contains common code patterns used for generating realistic file padding 127 | var PaddingPatterns = []string{ 128 | "\n\n// Helper functions\n", 129 | "function helper() { return true; }\n", 130 | "const data = { id: 1, name: 'test' };\n", 131 | "if (condition) { console.log('debug'); }\n", 132 | "// TODO: refactor this later\n", 133 | "/* eslint-disable no-unused-vars */\n", 134 | "import { util } from './utils';\n", 135 | "export default class Component {}\n", 136 | } 137 | -------------------------------------------------------------------------------- /.ci/check_new_rules.go: -------------------------------------------------------------------------------- 1 | // Scripts to check if all the rules that exist in the latest version of "gitleaks" are included in our list of rules (in secret.go file) 2 | package main 3 | 4 | import ( 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "os" 10 | "regexp" 11 | ) 12 | 13 | var ( 14 | regexGitleaksRules = regexp.MustCompile(`(?m)^[^/\n\r]\s*rules\.([a-zA-Z0-9_]+)\(`) 15 | regex2msRules = regexp.MustCompile(`(?m)^[^/\n\r]\s*(?:// )?{Rule:\s*\*(?:rules\.)?([a-zA-Z0-9_]+)\(\),`) 16 | ) 17 | 18 | func main() { 19 | 20 | latestGitleaksRelease, err := fetchGitleaksLatestRelease() 21 | if err != nil { 22 | fmt.Printf("%s\n", err) 23 | os.Exit(1) 24 | } 25 | fmt.Printf("Latest Gitleaks release: %s\n", latestGitleaksRelease) 26 | 27 | gitleaksRules, err := fetchGitleaksRules(latestGitleaksRelease) 28 | if err != nil { 29 | fmt.Printf("%s\n", err) 30 | os.Exit(1) 31 | } 32 | 33 | matchesGitleaksRules := regexGitleaksRules.FindAllStringSubmatch(string(gitleaksRules), -1) 34 | if len(matchesGitleaksRules) == 0 { 35 | fmt.Println("No rules found in the latest version of Gitleaks.") 36 | os.Exit(1) 37 | } 38 | fmt.Printf("Total rules in the latest version of Gitleaks: %d\n", len(matchesGitleaksRules)) 39 | 40 | ourRules, err := fetchOurRules() 41 | if err != nil { 42 | fmt.Printf("%s\n", err) 43 | os.Exit(1) 44 | } 45 | match2msRules := regex2msRules.FindAllStringSubmatch(string(ourRules), -1) 46 | if len(match2msRules) == 0 { 47 | fmt.Println("No rules found in 2ms.") 48 | os.Exit(1) 49 | } 50 | fmt.Printf("Total rules in 2ms: %d\n", len(match2msRules)) 51 | 52 | map2msRules := make(map[string]bool) 53 | for _, match := range match2msRules { 54 | map2msRules[match[1]] = true 55 | } 56 | 57 | missingRulesIn2ms := []string{} 58 | for _, rule := range matchesGitleaksRules { 59 | if _, found := map2msRules[rule[1]]; !found { 60 | missingRulesIn2ms = append(missingRulesIn2ms, rule[1]) 61 | } 62 | } 63 | 64 | if len(missingRulesIn2ms) > 0 { 65 | fmt.Printf("%d rules exist in the latest version of Gitleaks but missing on 2ms: \n\n", len(missingRulesIn2ms)) 66 | for _, rule := range missingRulesIn2ms { 67 | fmt.Printf("%s \n", rule) 68 | } 69 | 70 | fmt.Printf("\nLink to Gitleaks main.go file of version: %s:\n", latestGitleaksRelease) 71 | fmt.Println(getGitleaksRulesRawURL(latestGitleaksRelease)) 72 | 73 | os.Exit(1) 74 | } else { 75 | fmt.Println("No differences found.") 76 | os.Exit(0) 77 | } 78 | } 79 | 80 | type Release struct { 81 | TagName string `json:"tag_name"` 82 | } 83 | 84 | func fetchGitleaksLatestRelease() (string, error) { 85 | var release Release 86 | 87 | response, err := http.Get("https://api.github.com/repos/zricethezav/gitleaks/releases/latest") 88 | if err != nil { 89 | return "", fmt.Errorf("failed to get latest release: %w", err) 90 | } 91 | defer response.Body.Close() 92 | 93 | decoder := json.NewDecoder(response.Body) 94 | if err := decoder.Decode(&release); err != nil { 95 | return "", fmt.Errorf("failed to decode latest release JSON: %w", err) 96 | } 97 | 98 | return release.TagName, nil 99 | } 100 | 101 | func fetchGitleaksRules(version string) ([]byte, error) { 102 | rawURLGitleaksRules := getGitleaksRulesRawURL(version) 103 | response, err := http.Get(rawURLGitleaksRules) 104 | if err != nil { 105 | return nil, fmt.Errorf("failed to fetch remote file: %w", err) 106 | } 107 | defer response.Body.Close() 108 | 109 | content, err := io.ReadAll(response.Body) 110 | if err != nil { 111 | return nil, fmt.Errorf("failed to read remote file content: %w", err) 112 | } 113 | 114 | return content, nil 115 | } 116 | 117 | func getGitleaksRulesRawURL(version string) string { 118 | return fmt.Sprintf("https://raw.githubusercontent.com/zricethezav/gitleaks/%s/cmd/generate/config/main.go", version) 119 | } 120 | 121 | func fetchOurRules() ([]byte, error) { 122 | content, err := os.ReadFile("engine/rules/rules.go") 123 | if err != nil { 124 | return nil, fmt.Errorf("failed to read our file content: %w", err) 125 | } 126 | return content, nil 127 | } 128 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/checkmarx/2ms/v4 2 | 3 | go 1.25.5 4 | 5 | replace ( 6 | golang.org/x/oauth2 => golang.org/x/oauth2 v0.30.0 7 | google.golang.org/grpc => google.golang.org/grpc v1.73.0 8 | ) 9 | 10 | require ( 11 | github.com/alitto/pond/v2 v2.5.0 12 | github.com/bwmarrin/discordgo v0.27.1 13 | github.com/gitleaks/go-gitdiff v0.9.1 14 | github.com/h2non/filetype v1.1.3 15 | github.com/rs/zerolog v1.33.0 16 | github.com/shirou/gopsutil v3.21.11+incompatible 17 | github.com/slack-go/slack v0.12.2 18 | github.com/sourcegraph/conc v0.3.0 19 | github.com/spf13/cobra v1.9.1 20 | github.com/spf13/pflag v1.0.6 21 | github.com/spf13/viper v1.20.1 22 | github.com/stretchr/testify v1.10.0 23 | github.com/zricethezav/gitleaks/v8 v8.28.0 24 | go.uber.org/mock v0.5.2 25 | golang.org/x/net v0.47.0 26 | golang.org/x/sync v0.18.0 27 | golang.org/x/time v0.5.0 28 | gopkg.in/yaml.v3 v3.0.1 29 | ) 30 | 31 | require ( 32 | dario.cat/mergo v1.0.1 // indirect 33 | github.com/BobuSumisu/aho-corasick v1.0.3 // indirect 34 | github.com/Masterminds/goutils v1.1.1 // indirect 35 | github.com/Masterminds/semver/v3 v3.3.0 // indirect 36 | github.com/Masterminds/sprig/v3 v3.3.0 // indirect 37 | github.com/STARRY-S/zip v0.2.3 // indirect 38 | github.com/andybalholm/brotli v1.2.0 // indirect 39 | github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect 40 | github.com/bodgit/plumbing v1.3.0 // indirect 41 | github.com/bodgit/sevenzip v1.6.1 // indirect 42 | github.com/bodgit/windows v1.0.1 // indirect 43 | github.com/charmbracelet/lipgloss v0.7.1 // indirect 44 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 45 | github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect 46 | github.com/fatih/semgroup v1.2.0 // indirect 47 | github.com/fsnotify/fsnotify v1.8.0 // indirect 48 | github.com/go-ole/go-ole v1.2.6 // indirect 49 | github.com/go-viper/mapstructure/v2 v2.4.0 // indirect 50 | github.com/google/uuid v1.6.0 // indirect 51 | github.com/gorilla/websocket v1.5.0 // indirect 52 | github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect 53 | github.com/huandu/xstrings v1.5.0 // indirect 54 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 55 | github.com/klauspost/compress v1.18.0 // indirect 56 | github.com/klauspost/pgzip v1.2.6 // indirect 57 | github.com/lucasb-eyer/go-colorful v1.2.0 // indirect 58 | github.com/lucasjones/reggen v0.0.0-20200904144131-37ba4fa293bb // indirect 59 | github.com/mattn/go-colorable v0.1.14 // indirect 60 | github.com/mattn/go-isatty v0.0.20 // indirect 61 | github.com/mattn/go-runewidth v0.0.14 // indirect 62 | github.com/mholt/archives v0.1.5 // indirect 63 | github.com/mikelolasagasti/xz v1.0.1 // indirect 64 | github.com/minio/minlz v1.0.1 // indirect 65 | github.com/mitchellh/copystructure v1.2.0 // indirect 66 | github.com/mitchellh/reflectwalk v1.0.2 // indirect 67 | github.com/muesli/reflow v0.3.0 // indirect 68 | github.com/muesli/termenv v0.15.1 // indirect 69 | github.com/nwaples/rardecode/v2 v2.2.0 // indirect 70 | github.com/pelletier/go-toml/v2 v2.2.3 // indirect 71 | github.com/pierrec/lz4/v4 v4.1.22 // indirect 72 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 73 | github.com/rivo/uniseg v0.4.4 // indirect 74 | github.com/sagikazarmark/locafero v0.9.0 // indirect 75 | github.com/shopspring/decimal v1.4.0 // indirect 76 | github.com/sorairolake/lzip-go v0.3.8 // indirect 77 | github.com/spf13/afero v1.15.0 // indirect 78 | github.com/spf13/cast v1.7.1 // indirect 79 | github.com/subosito/gotenv v1.6.0 // indirect 80 | github.com/tetratelabs/wazero v1.9.0 // indirect 81 | github.com/ulikunitz/xz v0.5.15 // indirect 82 | github.com/wasilibs/go-re2 v1.9.0 // indirect 83 | github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect 84 | github.com/yusufpapurcu/wmi v1.2.4 // indirect 85 | go.uber.org/multierr v1.11.0 // indirect 86 | go4.org v0.0.0-20230225012048-214862532bf5 // indirect 87 | golang.org/x/crypto v0.45.0 // indirect 88 | golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa // indirect 89 | golang.org/x/sys v0.38.0 // indirect 90 | golang.org/x/text v0.31.0 // indirect 91 | ) 92 | -------------------------------------------------------------------------------- /pkg/scan.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/checkmarx/2ms/v4/internal/resources" 9 | "github.com/checkmarx/2ms/v4/plugins" 10 | "github.com/rs/zerolog/log" 11 | "github.com/sourcegraph/conc" 12 | 13 | "github.com/checkmarx/2ms/v4/lib/reporting" 14 | 15 | "github.com/checkmarx/2ms/v4/engine" 16 | ) 17 | 18 | type scanner struct { 19 | engineInstance engine.IEngine 20 | scanConfig resources.ScanConfig 21 | mu sync.RWMutex 22 | } 23 | 24 | type scannerOption func(*scanner) 25 | 26 | func WithPluginChannels(pluginChannels plugins.PluginChannels) scannerOption { 27 | return func(s *scanner) { 28 | s.engineInstance.SetPluginChannels(pluginChannels) 29 | } 30 | } 31 | 32 | func NewScanner() Scanner { 33 | return &scanner{} 34 | } 35 | 36 | func (s *scanner) Reset(scanConfig resources.ScanConfig, opts ...engine.EngineOption) error { 37 | s.mu.Lock() 38 | defer s.mu.Unlock() 39 | 40 | engineInstance, err := engine.Init(&engine.EngineConfig{ 41 | IgnoredIds: scanConfig.IgnoreResultIds, 42 | IgnoreList: scanConfig.IgnoreRules, 43 | ScanConfig: scanConfig, 44 | }, opts...) 45 | if err != nil { 46 | return fmt.Errorf("error initializing engine: %w", err) 47 | } 48 | 49 | s.engineInstance = engineInstance 50 | s.scanConfig = scanConfig 51 | 52 | return nil 53 | } 54 | 55 | func (s *scanner) Scan(scanItems []ScanItem, scanConfig resources.ScanConfig, opts ...engine.EngineOption) (reporting.IReport, error) { 56 | var wg conc.WaitGroup 57 | 58 | err := s.Reset(scanConfig, opts...) 59 | if err != nil { 60 | return nil, fmt.Errorf("error resetting engine: %w", err) 61 | } 62 | 63 | if len(scanItems) == 0 { 64 | return s.engineInstance.GetReport(), nil 65 | } 66 | 67 | bufferedErrors := make(chan error, len(scanItems)+1) 68 | 69 | go func() { 70 | defer close(bufferedErrors) 71 | 72 | for err := range s.engineInstance.GetErrorsCh() { 73 | bufferedErrors <- err 74 | } 75 | }() 76 | 77 | s.engineInstance.Scan(s.scanConfig.PluginName) 78 | 79 | wg.Go(func() { 80 | defer close(s.engineInstance.GetPluginChannels().GetItemsCh()) 81 | 82 | for _, item := range scanItems { 83 | s.engineInstance.GetPluginChannels().GetItemsCh() <- item 84 | } 85 | }) 86 | 87 | wg.Go(func() { 88 | s.engineInstance.Wait() 89 | }) 90 | 91 | wg.Wait() 92 | 93 | close(s.engineInstance.GetErrorsCh()) 94 | 95 | var errs []error 96 | for err = range bufferedErrors { 97 | errs = append(errs, err) 98 | } 99 | if len(errs) > 0 { 100 | return reporting.New().(*reporting.Report), fmt.Errorf("error(s) processing scan items:\n%w", errors.Join(errs...)) 101 | } 102 | 103 | return s.engineInstance.GetReport(), nil 104 | } 105 | 106 | func (s *scanner) ScanDynamic( 107 | itemsIn <-chan ScanItem, 108 | scanConfig resources.ScanConfig, 109 | opts ...engine.EngineOption, 110 | ) (reporting.IReport, error) { 111 | var wg conc.WaitGroup 112 | 113 | err := s.Reset(scanConfig, opts...) 114 | if err != nil { 115 | return reporting.New().(*reporting.Report), fmt.Errorf("error resetting engine: %w", err) 116 | } 117 | 118 | s.engineInstance.Scan(s.scanConfig.PluginName) 119 | 120 | channels := s.engineInstance.GetPluginChannels() 121 | wg.Go(func() { 122 | defer close(channels.GetItemsCh()) 123 | 124 | for item := range itemsIn { 125 | channels.GetItemsCh() <- item 126 | } 127 | 128 | log.Info().Msg("scan dynamic finished sending items to engine") 129 | }) 130 | 131 | bufferedErrors := make(chan error, 2) 132 | 133 | go func() { 134 | defer close(bufferedErrors) 135 | 136 | for err := range s.engineInstance.GetErrorsCh() { 137 | bufferedErrors <- err 138 | } 139 | }() 140 | 141 | wg.Go(func() { 142 | s.engineInstance.Wait() 143 | }) 144 | 145 | wg.Wait() 146 | 147 | close(s.engineInstance.GetErrorsCh()) 148 | 149 | var errs []error 150 | for err = range bufferedErrors { 151 | errs = append(errs, err) 152 | } 153 | if len(errs) > 0 { 154 | return reporting.New().(*reporting.Report), fmt.Errorf("error(s) processing scan items:\n%w", errors.Join(errs...)) 155 | } 156 | 157 | return s.engineInstance.GetReport(), nil 158 | } 159 | -------------------------------------------------------------------------------- /engine/linecontent/linecontent_test.go: -------------------------------------------------------------------------------- 1 | package linecontent 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | ) 7 | 8 | const ( 9 | dummySecret = "DummySecret" 10 | ) 11 | 12 | func TestGetLineContent(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | line string 16 | secret string 17 | expected string 18 | error bool 19 | errorMessage string 20 | }{ 21 | { 22 | name: "Empty line", 23 | line: "", 24 | secret: dummySecret, 25 | expected: "", 26 | error: true, 27 | errorMessage: "line empty", 28 | }, 29 | { 30 | name: "Empty secret", 31 | line: "line", 32 | secret: "", 33 | expected: "", 34 | error: true, 35 | errorMessage: "secret empty", 36 | }, 37 | { 38 | name: "Secret not found with line size smaller than the parse limit", 39 | line: "Dummy content line", 40 | secret: dummySecret, 41 | expected: "Dummy content line", 42 | error: false, 43 | }, 44 | { 45 | name: "Secret not found with secret present and line size larger than the parse limit", 46 | line: "This is the start of a big line content" + strings.Repeat("A", lineMaxParseSize) + dummySecret, 47 | secret: dummySecret, 48 | expected: "This is the start of a big line content" + strings.Repeat( 49 | "A", 50 | contextLeftSizeLimit+contextRightSizeLimit-len("This is the start of a big line content"), 51 | ), 52 | error: false, 53 | }, 54 | { 55 | name: "Secret larger than the line", 56 | line: strings.Repeat("B", contextLeftSizeLimit) + strings.Repeat("A", contextRightSizeLimit), 57 | secret: "large secret" + strings.Repeat("B", contextRightSizeLimit+contextLeftSizeLimit+100), 58 | expected: strings.Repeat("B", contextLeftSizeLimit) + strings.Repeat("A", contextRightSizeLimit), 59 | error: false, 60 | }, 61 | { 62 | name: "Secret at the beginning with line size smaller than the parse limit", 63 | line: "start:" + dummySecret + strings.Repeat("A", lineMaxParseSize/2), 64 | secret: dummySecret, 65 | expected: "start:" + dummySecret + strings.Repeat("A", contextRightSizeLimit), 66 | error: false, 67 | }, 68 | { 69 | name: "Secret found in middle with line size smaller than the parse limit", 70 | line: "start" + strings.Repeat( 71 | "A", 72 | contextLeftSizeLimit, 73 | ) + dummySecret + strings.Repeat( 74 | "A", 75 | contextRightSizeLimit, 76 | ) + "end", 77 | secret: dummySecret, 78 | expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", contextRightSizeLimit), 79 | error: false, 80 | }, 81 | { 82 | name: "Secret at the end with line size smaller than the parse limit", 83 | line: strings.Repeat("A", lineMaxParseSize/2) + dummySecret + ":end", 84 | secret: dummySecret, 85 | expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + ":end", 86 | error: false, 87 | }, 88 | { 89 | name: "Secret at the beginning with line size larger than the parse limit", 90 | line: "start:" + dummySecret + strings.Repeat("A", lineMaxParseSize), 91 | secret: dummySecret, 92 | expected: "start:" + dummySecret + strings.Repeat("A", contextRightSizeLimit), 93 | error: false, 94 | }, 95 | { 96 | name: "Secret found in middle with line size larger than the parse limit", 97 | line: "start" + strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", lineMaxParseSize) + "end", 98 | secret: dummySecret, 99 | expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", contextRightSizeLimit), 100 | error: false, 101 | }, 102 | { 103 | name: "Secret at the end with line size larger than the parse limit", 104 | line: strings.Repeat("A", lineMaxParseSize-100) + dummySecret + strings.Repeat("A", lineMaxParseSize), 105 | secret: dummySecret, 106 | expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", 100-len(dummySecret)), 107 | error: false, 108 | }, 109 | } 110 | 111 | for _, tt := range tests { 112 | t.Run(tt.name, func(t *testing.T) { 113 | got, err := GetLineContent(tt.line, tt.secret) 114 | if (err != nil) != tt.error { 115 | t.Fatalf("GetLineContent() error = %v, wantErr %v", err, tt.error) 116 | } 117 | if err != nil && err.Error() != tt.errorMessage { 118 | t.Errorf("GetLineContent() error message = %v, want %v", err.Error(), tt.errorMessage) 119 | } 120 | if got != tt.expected { 121 | t.Errorf("GetLineContent() = %v, want %v", got, tt.expected) 122 | } 123 | }) 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /engine/chunk/chunk_test.go: -------------------------------------------------------------------------------- 1 | package chunk 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "github.com/stretchr/testify/require" 7 | "io" 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | const ( 13 | chunkSize = 10 14 | maxPeekSize = 5 15 | smallFileThreshold = int64(20) 16 | ) 17 | 18 | func TestGetAndPutBuf(t *testing.T) { 19 | c := New() 20 | data := []byte("test") 21 | buf, ok := c.GetBuf(data) 22 | defer c.PutBuf(buf) 23 | 24 | require.True(t, ok) 25 | require.Equal(t, defaultSize+defaultMaxPeekSize, buf.Cap()) 26 | require.Equal(t, string(data), buf.String()) 27 | } 28 | 29 | func TestGetAndPutPeekedBuf(t *testing.T) { 30 | c := New() 31 | window, ok := c.GetPeekedBuf() 32 | defer c.PutPeekedBuf(window) 33 | 34 | require.True(t, ok) 35 | require.Equal(t, defaultSize+defaultMaxPeekSize, len(*window)) 36 | } 37 | 38 | func TestGetSize(t *testing.T) { 39 | c := New() 40 | require.Equal(t, defaultSize, c.GetSize()) 41 | } 42 | 43 | func TestGetMaxPeekSize(t *testing.T) { 44 | c := New() 45 | require.Equal(t, defaultMaxPeekSize, c.GetMaxPeekSize()) 46 | } 47 | 48 | func TestReadChunk(t *testing.T) { 49 | // Arrange 50 | type testCase struct { 51 | name string 52 | reader io.Reader 53 | expected string 54 | expectedError error 55 | } 56 | testCases := []testCase{ 57 | { 58 | name: "empty", 59 | reader: strings.NewReader(""), 60 | expectedError: io.EOF, 61 | }, 62 | { 63 | name: "unsupported file type", 64 | reader: bytes.NewReader([]byte{'P', 'K', 0x03, 0x04}), 65 | expectedError: ErrUnsupportedFileType, 66 | }, 67 | { 68 | name: "successful read", 69 | reader: strings.NewReader("abc\n"), 70 | expected: "abc\n", 71 | }, 72 | { 73 | name: "successful read - peek size exceeded", 74 | reader: strings.NewReader("abc\ndef\nghi\njkl\nmno\npqr\nstu\nvwx\nyz"), 75 | expected: "abc\ndef\nghi\njkl", 76 | }, 77 | { 78 | name: "successful read - multiple lines with consecutives new lines", 79 | reader: strings.NewReader("abc\ndef\n\n\n\n\nghi\njkl"), 80 | expected: "abc\ndef\n\n\n", 81 | }, 82 | { 83 | name: "multiple lines without consecutives new lines", 84 | reader: strings.NewReader("abc\ndef\nghi\n"), 85 | expected: "abc\ndef\nghi\n", 86 | }, 87 | } 88 | 89 | for _, tc := range testCases { 90 | t.Run(tc.name, func(t *testing.T) { 91 | c := New(WithSize(chunkSize), WithMaxPeekSize(maxPeekSize), WithSmallFileThreshold(smallFileThreshold)) 92 | reader := bufio.NewReaderSize(tc.reader, chunkSize+maxPeekSize) 93 | 94 | // Act 95 | result, err := c.ReadChunk(reader, 0) 96 | require.ErrorIs(t, err, tc.expectedError) 97 | 98 | // Assert 99 | require.Equal(t, tc.expected, result) 100 | }) 101 | } 102 | } 103 | 104 | func TestGenerateChunk(t *testing.T) { 105 | // Arrange 106 | testCases := []struct { 107 | name string 108 | rawData []byte 109 | expected string 110 | }{ 111 | // Current split is fine, exit early. 112 | { 113 | name: "safe original split - LF", 114 | rawData: []byte("abc\ndef\n\n\nghijklmnop\n\nqrstuvwxyz"), 115 | expected: "abc\ndef\n\n\n", 116 | }, 117 | { 118 | name: "safe original split - CRLF", 119 | rawData: []byte("abcdef\r\n\r\nghijklmnop\n"), 120 | expected: "abcdef\r\n\r\n", 121 | }, 122 | // Current split is bad, look for a better one 123 | { 124 | name: "safe split - LF", 125 | rawData: []byte("abcdef\nghi\n\njklmnop\n\nqrstuvwxyz"), 126 | expected: "abcdef\nghi\n\n", 127 | }, 128 | { 129 | name: "safe split - CRLF", 130 | rawData: []byte("abcdef\r\nghi\r\n\r\njklmnopqrstuvwxyz"), 131 | expected: "abcdef\r\nghi\r\n\r\n", 132 | }, 133 | { 134 | name: "safe split - blank line", 135 | rawData: []byte("abcdefghi\n\t \t\njklmnopqrstuvwxyz"), 136 | expected: "abcdefghi\n\t \t\n", 137 | }, 138 | // Current split is bad, exhaust options 139 | { 140 | name: "no safe split", 141 | rawData: []byte("abcdefg\nhijklmnopqrstuvwxyz"), 142 | expected: "abcdefg\nhijklmn", 143 | }, 144 | } 145 | 146 | for _, tc := range testCases { 147 | t.Run(tc.name, func(t *testing.T) { 148 | c := New(WithSize(chunkSize), WithMaxPeekSize(maxPeekSize), WithSmallFileThreshold(smallFileThreshold)) 149 | reader := bufio.NewReaderSize(bytes.NewReader(tc.rawData), c.size+c.maxPeekSize) 150 | peekedBuf := make([]byte, c.size+c.maxPeekSize) 151 | _, err := reader.Read(peekedBuf) 152 | require.NoError(t, err) 153 | 154 | // Act 155 | chunkStr, err := c.generateChunk(peekedBuf) 156 | require.NoError(t, err) 157 | 158 | // Assert 159 | require.Equal(t, tc.expected, chunkStr) 160 | }) 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /engine/score/score.go: -------------------------------------------------------------------------------- 1 | package score 2 | 3 | import ( 4 | "math" 5 | "strings" 6 | 7 | "github.com/checkmarx/2ms/v4/engine/rules" 8 | "github.com/checkmarx/2ms/v4/lib/secrets" 9 | "github.com/zricethezav/gitleaks/v8/config" 10 | ) 11 | 12 | type scorer struct { 13 | rulesBaseRiskScore map[string]float64 14 | withValidation bool 15 | keywords map[string]struct{} 16 | rulesToBeApplied map[string]config.Rule 17 | } 18 | 19 | func NewScorer(selectedRules []*rules.Rule, withValidation bool) *scorer { 20 | rulesToBeApplied := make(map[string]config.Rule) 21 | rulesBaseRiskScore := make(map[string]float64) 22 | keywords := make(map[string]struct{}) 23 | for _, rule := range selectedRules { 24 | rulesToBeApplied[rule.Rule.RuleID] = rule.Rule 25 | rulesBaseRiskScore[rule.Rule.RuleID] = GetBaseRiskScore(rule.ScoreParameters.Category, rule.ScoreParameters.RuleType) 26 | for _, keyword := range rule.Rule.Keywords { 27 | keywords[strings.ToLower(keyword)] = struct{}{} 28 | } 29 | } 30 | return &scorer{ 31 | rulesBaseRiskScore: rulesBaseRiskScore, 32 | withValidation: withValidation, 33 | keywords: keywords, 34 | rulesToBeApplied: rulesToBeApplied, 35 | } 36 | } 37 | 38 | func (s *scorer) Score(secret *secrets.Secret) { 39 | validationStatus := secrets.UnknownResult // default validity 40 | if s.withValidation { 41 | validationStatus = secret.ValidationStatus 42 | } 43 | secret.CvssScore = getCvssScore(s.rulesBaseRiskScore[secret.RuleID], validationStatus) 44 | } 45 | 46 | func getCategoryScore(category rules.RuleCategory) uint8 { 47 | CategoryScore := map[rules.RuleCategory]uint8{ 48 | rules.CategoryAuthenticationAndAuthorization: 4, 49 | rules.CategoryCryptocurrencyExchange: 4, 50 | rules.CategoryFinancialServices: 4, 51 | rules.CategoryPaymentProcessing: 4, 52 | rules.CategorySecurity: 4, 53 | rules.CategoryAPIAccess: 3, 54 | rules.CategoryCICD: 3, 55 | rules.CategoryCloudPlatform: 3, 56 | rules.CategoryDatabaseAsAService: 3, 57 | rules.CategoryDevelopmentPlatform: 3, 58 | rules.CategoryEmailDeliveryService: 3, 59 | rules.CategoryGeneralOrUnknown: 3, 60 | rules.CategoryInfrastructureAsCode: 3, 61 | rules.CategoryPackageManagement: 3, 62 | rules.CategorySourceCodeManagement: 3, 63 | rules.CategoryWebHostingAndDeployment: 3, 64 | rules.CategoryBackgroundProcessingService: 2, 65 | rules.CategoryCDN: 2, 66 | rules.CategoryContentManagementSystem: 2, 67 | rules.CategoryCustomerSupport: 2, 68 | rules.CategoryDataAnalytics: 2, 69 | rules.CategoryFileStorageAndSharing: 2, 70 | rules.CategoryIoTPlatform: 2, 71 | rules.CategoryMappingAndLocationServices: 2, 72 | rules.CategoryNetworking: 2, 73 | rules.CategoryPhotoSharing: 2, 74 | rules.CategorySaaS: 2, 75 | rules.CategoryShipping: 2, 76 | rules.CategorySoftwareDevelopment: 2, 77 | rules.CategoryAIAndMachineLearning: 1, 78 | rules.CategoryApplicationMonitoring: 1, 79 | rules.CategoryECommercePlatform: 1, 80 | rules.CategoryMarketingAutomation: 1, 81 | rules.CategoryNewsAndMedia: 1, 82 | rules.CategoryOnlineSurveyPlatform: 1, 83 | rules.CategoryProjectManagement: 1, 84 | rules.CategorySearchService: 1, 85 | rules.CategorySocialMedia: 1, 86 | } 87 | return CategoryScore[category] 88 | } 89 | 90 | func getValidityScore(baseRiskScore float64, validationStatus secrets.ValidationResult) float64 { 91 | switch validationStatus { 92 | case secrets.ValidResult: 93 | return math.Min(1, 4-baseRiskScore) 94 | case secrets.InvalidResult: 95 | return math.Max(-1, 1-baseRiskScore) 96 | } 97 | return 0.0 98 | } 99 | 100 | func GetBaseRiskScore(category rules.RuleCategory, ruleType uint8) float64 { 101 | categoryScore := getCategoryScore(category) 102 | return float64(categoryScore)*0.6 + float64(ruleType)*0.4 103 | } 104 | 105 | func getCvssScore(baseRiskScore float64, validationStatus secrets.ValidationResult) float64 { 106 | validityScore := getValidityScore(baseRiskScore, validationStatus) 107 | cvssScore := (baseRiskScore+validityScore-1)*3 + 1 108 | return math.Round(cvssScore*10) / 10 109 | } 110 | 111 | func (s *scorer) GetKeywords() map[string]struct{} { 112 | return s.keywords 113 | } 114 | 115 | func (s *scorer) GetRulesToBeApplied() map[string]config.Rule { 116 | return s.rulesToBeApplied 117 | } 118 | 119 | func (s *scorer) GetRulesBaseRiskScore(ruleId string) float64 { 120 | return s.rulesBaseRiskScore[ruleId] 121 | } 122 | -------------------------------------------------------------------------------- /cmd/config.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "regexp" 7 | "strings" 8 | 9 | "github.com/checkmarx/2ms/v4/lib/utils" 10 | "github.com/rs/zerolog" 11 | "github.com/rs/zerolog/log" 12 | "github.com/spf13/cobra" 13 | ) 14 | 15 | var ( 16 | errInvalidOutputFormat = fmt.Errorf("invalid output format") 17 | errInvalidReportExtension = fmt.Errorf("invalid report extension") 18 | ) 19 | 20 | func processFlags(rootCmd *cobra.Command) error { 21 | configFilePath, err := rootCmd.PersistentFlags().GetString(configFileFlag) 22 | if err != nil { 23 | return fmt.Errorf("failed to get config file path: %w", err) 24 | } 25 | 26 | if err := utils.LoadConfig(vConfig, configFilePath); err != nil { 27 | return fmt.Errorf("failed to load config: %w", err) 28 | } 29 | 30 | if err := utils.BindFlags(rootCmd, vConfig, envPrefix); err != nil { 31 | return fmt.Errorf("failed to bind flags: %w", err) 32 | } 33 | 34 | // Apply all flag mappings immediately 35 | engineConfigVar.ScanConfig.WithValidation = validateVar 36 | if len(customRegexRuleVar) > 0 { 37 | engineConfigVar.CustomRegexPatterns = customRegexRuleVar 38 | } 39 | 40 | setupLogging() 41 | 42 | return nil 43 | } 44 | 45 | func setupLogging() { 46 | logLevel := zerolog.InfoLevel 47 | switch strings.ToLower(logLevelVar) { 48 | case "none": 49 | logLevel = zerolog.Disabled 50 | case "trace": 51 | logLevel = zerolog.TraceLevel 52 | case "debug": 53 | logLevel = zerolog.DebugLevel 54 | case "info": 55 | logLevel = zerolog.InfoLevel 56 | case "warn": 57 | logLevel = zerolog.WarnLevel 58 | case "err", "error": 59 | logLevel = zerolog.ErrorLevel 60 | case "fatal": 61 | logLevel = zerolog.FatalLevel 62 | } 63 | zerolog.SetGlobalLevel(logLevel) 64 | log.Logger = log.Logger.Level(logLevel) 65 | } 66 | 67 | func validateFormat(stdout string, reportPath []string) error { 68 | r := regexp.MustCompile(outputFormatRegexpPattern) 69 | if !(r.MatchString(stdout)) { 70 | return fmt.Errorf(`%w: %s, available formats are: json, yaml and sarif`, errInvalidOutputFormat, stdout) 71 | } 72 | 73 | for _, path := range reportPath { 74 | fileExtension := filepath.Ext(path) 75 | format := strings.TrimPrefix(fileExtension, ".") 76 | if !(r.MatchString(format)) { 77 | return fmt.Errorf(`%w: %s, available extensions are: json, yaml and sarif`, errInvalidReportExtension, format) 78 | } 79 | } 80 | 81 | return nil 82 | } 83 | 84 | func setupFlags(rootCmd *cobra.Command) { 85 | rootCmd.PersistentFlags().StringVar(&configFilePath, configFileFlag, "", "config file path") 86 | cobra.CheckErr(rootCmd.MarkPersistentFlagFilename(configFileFlag, "yaml", "yml", "json")) 87 | 88 | rootCmd.PersistentFlags().StringVar(&logLevelVar, logLevelFlagName, "info", "log level (trace, debug, info, warn, error, fatal, none)") 89 | 90 | rootCmd.PersistentFlags(). 91 | StringSliceVar(&reportPathVar, reportPathFlagName, []string{}, 92 | "path to generate report files. The output format will be determined by the file extension (.json, .yaml, .sarif)") 93 | 94 | rootCmd.PersistentFlags(). 95 | StringVar(&stdoutFormatVar, stdoutFormatFlagName, "yaml", "stdout output format, available formats are: json, yaml, sarif") 96 | 97 | rootCmd.PersistentFlags(). 98 | StringArrayVar(&customRegexRuleVar, customRegexRuleFlagName, []string{}, "custom regexes to apply to the scan, must be valid Go regex") 99 | 100 | rootCmd.PersistentFlags(). 101 | StringSliceVar(&engineConfigVar.SelectedList, ruleFlagName, []string{}, "select rules by name or tag to apply to this scan") 102 | 103 | rootCmd.PersistentFlags().StringSliceVar(&engineConfigVar.IgnoreList, ignoreRuleFlagName, []string{}, "ignore rules by name or tag") 104 | rootCmd.PersistentFlags().StringSliceVar(&engineConfigVar.IgnoredIds, ignoreFlagName, []string{}, "ignore specific result by id") 105 | 106 | rootCmd.PersistentFlags(). 107 | StringSliceVar(&engineConfigVar.AllowedValues, allowedValuesFlagName, []string{}, "allowed secrets values to ignore") 108 | 109 | rootCmd.PersistentFlags(). 110 | StringSliceVar(&engineConfigVar.SpecialList, specialRulesFlagName, []string{}, 111 | "special (non-default) rules to apply.\nThis list is not affected by the --rule and --ignore-rule flags.") 112 | 113 | rootCmd.PersistentFlags(). 114 | Var(&ignoreOnExitVar, ignoreOnExitFlagName, 115 | "defines which kind of non-zero exits code should be ignored\naccepts: all, results, errors, none\n"+ 116 | "example: if 'results' is set, only engine errors will make 2ms exit code different from 0") 117 | 118 | rootCmd.PersistentFlags(). 119 | IntVar(&engineConfigVar.MaxTargetMegabytes, maxTargetMegabytesFlagName, 0, 120 | "files larger than this will be skipped.\nOmit or set to 0 to disable this check.") 121 | 122 | rootCmd.PersistentFlags(). 123 | BoolVar(&validateVar, validate, false, "trigger additional validation to check if discovered secrets are valid or invalid") 124 | } 125 | -------------------------------------------------------------------------------- /engine/rules/generic-key.go: -------------------------------------------------------------------------------- 1 | package rules 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/zricethezav/gitleaks/v8/cmd/generate/config/rules" 7 | "github.com/zricethezav/gitleaks/v8/config" 8 | ) 9 | 10 | const GenericApiKeyID = "generic-api-key" 11 | 12 | func GenericCredential() *config.Rule { 13 | regex := generateSemiGenericRegexIncludingXml([]string{ 14 | "access", 15 | "auth", 16 | `(?-i:[Aa]pi|API)`, 17 | "credential", 18 | "creds", 19 | "key", 20 | "passw(?:or)?d", 21 | "secret", 22 | "token", 23 | }, `[\w.=-]{10,150}|[a-z0-9][a-z0-9+/]{11,}={0,3}`, true) 24 | 25 | return &config.Rule{ 26 | RuleID: GenericApiKeyID, 27 | Description: "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.", 28 | Regex: regex, 29 | Keywords: []string{ 30 | "access", 31 | "api", 32 | "auth", 33 | "key", 34 | "credential", 35 | "creds", 36 | "passwd", 37 | "password", 38 | "secret", 39 | "token", 40 | }, 41 | Entropy: 3.5, 42 | Allowlists: []*config.Allowlist{ 43 | { 44 | // NOTE: this is a goofy hack to get around the fact there golang's regex engine does not support positive lookaheads. 45 | // Ideally we would want to ensure the secret contains both numbers and alphabetical characters, not just alphabetical characters. 46 | Regexes: []*regexp.Regexp{ 47 | regexp.MustCompile(`^[a-zA-Z_.-]+$`), 48 | }, 49 | }, 50 | { 51 | Description: "Allowlist for Generic API Keys", 52 | MatchCondition: config.AllowlistMatchOr, 53 | RegexTarget: "match", 54 | Regexes: []*regexp.Regexp{ 55 | regexp.MustCompile(`(?i)(?:` + 56 | // Access 57 | `access(?:ibility|or)` + 58 | `|access[_.-]?id` + 59 | `|random[_.-]?access` + 60 | // API 61 | `|api[_.-]?(?:id|name|version)` + // id/name/version -> not a secret 62 | `|rapid|capital` + // common words containing "api" 63 | `|[a-z0-9-]*?api[a-z0-9-]*?:jar:` + // Maven META-INF dependencies that contain "api" in the name. 64 | // Auth 65 | `|author` + 66 | `|X-MS-Exchange-Organization-Auth` + // email header 67 | `|Authentication-Results` + // email header 68 | // Credentials 69 | `|(?:credentials?[_.-]?id|withCredentials)` + // Jenkins plugins 70 | // IPv4 71 | `|(?:25[0-5]|2[0-4]\d|1?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|1?\d?\d)){3}` + 72 | // Key 73 | `|(?:bucket|foreign|hot|idx|natural|primary|pub(?:lic)?|schema|sequence)[_.-]?key` + 74 | `|(?:turkey)` + 75 | `|key[_.-]?(?:alias|board|code|frame|id|length|mesh|name|pair|press(?:ed)?|ring|selector|signature|size|stone|storetype|word|up|down|left|right)` + //nolint:lll 76 | // Azure KeyVault 77 | `|KeyVault(?:[A-Za-z]*?(?:Administrator|Reader|Contributor|Owner|Operator|User|Officer))\s*[:=]\s*['"]?[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}['"]?` + //nolint:lll 78 | `|key[_.-]?vault[_.-]?(?:id|name)|keyVaultToStoreSecrets` + 79 | `|key(?:store|tab)[_.-]?(?:file|path)` + 80 | `|issuerkeyhash` + // part of ssl cert 81 | `|(?-i:[DdMm]onkey|[DM]ONKEY)|keying` + // common words containing "key" 82 | // Secret 83 | `|(?:secret)[_.-]?(?:length|name|size)` + // name of e.g. env variable 84 | `|UserSecretsId` + // https://learn.microsoft.com/en-us/aspnet/core/security/app-secrets?view=aspnetcore-8.0&tabs=linux 85 | // Token 86 | `|(?:csrf)[_.-]?token` + 87 | // Maven library coordinates. (e.g., https://mvnrepository.com/artifact/io.jsonwebtoken/jjwt) 88 | `|(?:io\.jsonwebtoken[ \t]?:[ \t]?[\w-]+)` + 89 | // General 90 | `|(?:api|credentials|token)[_.-]?(?:endpoint|ur[il])` + 91 | `|public[_.-]?token` + 92 | `|(?:key|token)[_.-]?file` + 93 | // Empty variables capturing the next line (e.g., .env files) 94 | `|(?-i:(?:[A-Z_]+=\n[A-Z_]+=|[a-z_]+=\n[a-z_]+=)(?:\n|\z))` + 95 | `|(?-i:(?:[A-Z.]+=\n[A-Z.]+=|[a-z.]+=\n[a-z.]+=)(?:\n|\z))` + 96 | `)`), 97 | }, 98 | StopWords: append(rules.DefaultStopWords, 99 | "6fe4476ee5a1832882e326b506d14126", // https://github.com/yarnpkg/berry/issues/6201 100 | ), 101 | }, 102 | { 103 | RegexTarget: "line", 104 | Regexes: []*regexp.Regexp{ 105 | // Docker build secrets (https://docs.docker.com/build/building/secrets/#using-build-secrets). 106 | regexp.MustCompile(`--mount=type=secret,`), 107 | // https://github.com/gitleaks/gitleaks/issues/1800 108 | regexp.MustCompile(`import[ \t]+{[ \t\w,]+}[ \t]+from[ \t]+['"][^'"]+['"]`), 109 | }, 110 | }, 111 | { 112 | MatchCondition: config.AllowlistMatchAnd, 113 | RegexTarget: "line", 114 | Regexes: []*regexp.Regexp{ 115 | regexp.MustCompile(`LICENSE[^=]*=\s*"[^"]+`), 116 | regexp.MustCompile(`LIC_FILES_CHKSUM[^=]*=\s*"[^"]+`), 117 | regexp.MustCompile(`SRC[^=]*=\s*"[a-zA-Z0-9]+`), 118 | }, 119 | Paths: []*regexp.Regexp{ 120 | regexp.MustCompile(`\.bb$`), 121 | regexp.MustCompile(`\.bbappend$`), 122 | regexp.MustCompile(`\.bbclass$`), 123 | regexp.MustCompile(`\.inc$`), 124 | }, 125 | }, 126 | }, 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /pkg/testData/expectedReport.json: -------------------------------------------------------------------------------- 1 | { 2 | "totalItemsScanned": 3, 3 | "totalSecretsFound": 6, 4 | "results": { 5 | "4b0fb9bf4c96bd11404f2a3b187acbb621d8ca0c": [ 6 | { 7 | "id": "4b0fb9bf4c96bd11404f2a3b187acbb621d8ca0c", 8 | "source": "testData/secrets/jwt.txt", 9 | "ruleId": "jwt", 10 | "startLine": 0, 11 | "endLine": 0, 12 | "lineContent": "TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1 TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 TextExample", 13 | "startColumn": 13, 14 | "endColumn": 116, 15 | "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1", 16 | "ruleDescription": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", 17 | "extraDetails": { 18 | "secretDetails": { 19 | "name": "mockName1", 20 | "sub": "mockSub1" 21 | } 22 | }, 23 | "cvssScore": 8.2 24 | } 25 | ], 26 | "6d1959886d3a01308b495ccc9d6a30444016aaad": [ 27 | { 28 | "id": "6d1959886d3a01308b495ccc9d6a30444016aaad", 29 | "source": "testData/secrets/github-pat.txt", 30 | "ruleId": "github-pat", 31 | "startLine": 0, 32 | "endLine": 0, 33 | "lineContent": "TextExampleghp_1234567890abcdefghijklmnopqrstuvwxyzTextExampleghp_abcdefghijklmnopqrstuvwxyz1234567890TextExample", 34 | "startColumn": 12, 35 | "endColumn": 51, 36 | "value": "ghp_1234567890abcdefghijklmnopqrstuvwxyz", 37 | "ruleDescription": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", 38 | "cvssScore": 8.2 39 | } 40 | ], 41 | "7eb9787a837e56bf0ee0be36497fdb671b2b9041": [ 42 | { 43 | "id": "7eb9787a837e56bf0ee0be36497fdb671b2b9041", 44 | "source": "testData/secrets/github-pat.txt", 45 | "ruleId": "github-pat", 46 | "startLine": 1, 47 | "endLine": 1, 48 | "lineContent": " Text_Example = ghp_9876543210zyxwvutsrqponmlkjihgfedcba", 49 | "startColumn": 63, 50 | "endColumn": 102, 51 | "value": "ghp_9876543210zyxwvutsrqponmlkjihgfedcba", 52 | "ruleDescription": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", 53 | "cvssScore": 8.2 54 | } 55 | ], 56 | "dac14c6111d3a02a23c4fc31ee4759387a7395cd": [ 57 | { 58 | "id": "dac14c6111d3a02a23c4fc31ee4759387a7395cd", 59 | "source": "testData/secrets/jwt.txt", 60 | "ruleId": "jwt", 61 | "startLine": 0, 62 | "endLine": 0, 63 | "lineContent": "TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1 TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 TextExample", 64 | "startColumn": 129, 65 | "endColumn": 232, 66 | "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2", 67 | "ruleDescription": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", 68 | "extraDetails": { 69 | "secretDetails": { 70 | "name": "mockName2", 71 | "sub": "mockSub2" 72 | } 73 | }, 74 | "cvssScore": 8.2 75 | }, 76 | { 77 | "id": "dac14c6111d3a02a23c4fc31ee4759387a7395cd", 78 | "source": "testData/secrets/jwt.txt", 79 | "ruleId": "jwt", 80 | "startLine": 1, 81 | "endLine": 1, 82 | "lineContent": " Text_Example = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2", 83 | "startColumn": 63, 84 | "endColumn": 166, 85 | "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2", 86 | "ruleDescription": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", 87 | "extraDetails": { 88 | "secretDetails": { 89 | "name": "mockName2", 90 | "sub": "mockSub2" 91 | } 92 | }, 93 | "cvssScore": 8.2 94 | } 95 | ], 96 | "dfe5ea720d7c38631079841b754a34df29c25d93": [ 97 | { 98 | "id": "dfe5ea720d7c38631079841b754a34df29c25d93", 99 | "source": "testData/secrets/github-pat.txt", 100 | "ruleId": "github-pat", 101 | "startLine": 0, 102 | "endLine": 0, 103 | "lineContent": "TextExampleghp_1234567890abcdefghijklmnopqrstuvwxyzTextExampleghp_abcdefghijklmnopqrstuvwxyz1234567890TextExample", 104 | "startColumn": 63, 105 | "endColumn": 102, 106 | "value": "ghp_abcdefghijklmnopqrstuvwxyz1234567890", 107 | "ruleDescription": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", 108 | "cvssScore": 8.2 109 | } 110 | ] 111 | } 112 | } -------------------------------------------------------------------------------- /pkg/testData/expectedReportWithIgnoredResults.json: -------------------------------------------------------------------------------- 1 | { 2 | "totalItemsScanned": 3, 3 | "totalSecretsFound": 6, 4 | "results": { 5 | "4b0fb9bf4c96bd11404f2a3b187acbb621d8ca0c": [ 6 | { 7 | "id": "4b0fb9bf4c96bd11404f2a3b187acbb621d8ca0c", 8 | "source": "testData/secrets/jwt.txt", 9 | "ruleId": "jwt", 10 | "startLine": 0, 11 | "endLine": 0, 12 | "lineContent": "TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1 TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 TextExample", 13 | "startColumn": 13, 14 | "endColumn": 116, 15 | "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1", 16 | "ruleDescription": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", 17 | "extraDetails": { 18 | "secretDetails": { 19 | "name": "mockName1", 20 | "sub": "mockSub1" 21 | } 22 | }, 23 | "cvssScore": 8.2 24 | } 25 | ], 26 | "6d1959886d3a01308b495ccc9d6a30444016aaad": [ 27 | { 28 | "id": "6d1959886d3a01308b495ccc9d6a30444016aaad", 29 | "source": "testData/secrets/github-pat.txt", 30 | "ruleId": "github-pat", 31 | "startLine": 0, 32 | "endLine": 0, 33 | "lineContent": "TextExampleghp_1234567890abcdefghijklmnopqrstuvwxyzTextExampleghp_abcdefghijklmnopqrstuvwxyz1234567890TextExample", 34 | "startColumn": 12, 35 | "endColumn": 51, 36 | "value": "ghp_1234567890abcdefghijklmnopqrstuvwxyz", 37 | "ruleDescription": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", 38 | "cvssScore": 8.2 39 | } 40 | ], 41 | "7eb9787a837e56bf0ee0be36497fdb671b2b9041": [ 42 | { 43 | "id": "7eb9787a837e56bf0ee0be36497fdb671b2b9041", 44 | "source": "testData/secrets/github-pat.txt", 45 | "ruleId": "github-pat", 46 | "startLine": 1, 47 | "endLine": 1, 48 | "lineContent": " Text_Example = ghp_9876543210zyxwvutsrqponmlkjihgfedcba", 49 | "startColumn": 63, 50 | "endColumn": 102, 51 | "value": "ghp_9876543210zyxwvutsrqponmlkjihgfedcba", 52 | "ruleDescription": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", 53 | "cvssScore": 8.2 54 | } 55 | ], 56 | "dac14c6111d3a02a23c4fc31ee4759387a7395cd": [ 57 | { 58 | "id": "dac14c6111d3a02a23c4fc31ee4759387a7395cd", 59 | "source": "testData/secrets/jwt.txt", 60 | "ruleId": "jwt", 61 | "startLine": 0, 62 | "endLine": 0, 63 | "lineContent": "TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1 TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 TextExample", 64 | "startColumn": 129, 65 | "endColumn": 232, 66 | "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2", 67 | "ruleDescription": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", 68 | "extraDetails": { 69 | "secretDetails": { 70 | "name": "mockName2", 71 | "sub": "mockSub2" 72 | } 73 | }, 74 | "cvssScore": 8.2 75 | }, 76 | { 77 | "id": "dac14c6111d3a02a23c4fc31ee4759387a7395cd", 78 | "source": "testData/secrets/jwt.txt", 79 | "ruleId": "jwt", 80 | "startLine": 1, 81 | "endLine": 1, 82 | "lineContent": " Text_Example = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2", 83 | "startColumn": 63, 84 | "endColumn": 166, 85 | "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2", 86 | "ruleDescription": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", 87 | "extraDetails": { 88 | "secretDetails": { 89 | "name": "mockName2", 90 | "sub": "mockSub2" 91 | } 92 | }, 93 | "cvssScore": 8.2 94 | } 95 | ], 96 | "dfe5ea720d7c38631079841b754a34df29c25d93": [ 97 | { 98 | "id": "dfe5ea720d7c38631079841b754a34df29c25d93", 99 | "source": "testData/secrets/github-pat.txt", 100 | "ruleId": "github-pat", 101 | "startLine": 0, 102 | "endLine": 0, 103 | "lineContent": "TextExampleghp_1234567890abcdefghijklmnopqrstuvwxyzTextExampleghp_abcdefghijklmnopqrstuvwxyz1234567890TextExample", 104 | "startColumn": 63, 105 | "endColumn": 102, 106 | "value": "ghp_abcdefghijklmnopqrstuvwxyz1234567890", 107 | "ruleDescription": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", 108 | "cvssScore": 8.2 109 | } 110 | ] 111 | } 112 | } -------------------------------------------------------------------------------- /pkg/testData/expectedReportWithValidation.json: -------------------------------------------------------------------------------- 1 | { 2 | "results": { 3 | "7eb9787a837e56bf0ee0be36497fdb671b2b9041": [ 4 | { 5 | "cvssScore": 5.2, 6 | "endColumn": 102, 7 | "endLine": 1, 8 | "id": "7eb9787a837e56bf0ee0be36497fdb671b2b9041", 9 | "lineContent": " Text_Example = ghp_9876543210zyxwvutsrqponmlkjihgfedcba", 10 | "ruleDescription": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", 11 | "ruleId": "github-pat", 12 | "source": "testData/secrets/github-pat.txt", 13 | "startColumn": 63, 14 | "startLine": 1, 15 | "validationStatus": "Invalid", 16 | "value": "ghp_9876543210zyxwvutsrqponmlkjihgfedcba" 17 | } 18 | ], 19 | "6d1959886d3a01308b495ccc9d6a30444016aaad": [ 20 | { 21 | "cvssScore": 5.2, 22 | "endColumn": 51, 23 | "endLine": 0, 24 | "id": "6d1959886d3a01308b495ccc9d6a30444016aaad", 25 | "lineContent": "TextExampleghp_1234567890abcdefghijklmnopqrstuvwxyzTextExampleghp_abcdefghijklmnopqrstuvwxyz1234567890TextExample", 26 | "ruleDescription": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", 27 | "ruleId": "github-pat", 28 | "source": "testData/secrets/github-pat.txt", 29 | "startColumn": 12, 30 | "startLine": 0, 31 | "validationStatus": "Invalid", 32 | "value": "ghp_1234567890abcdefghijklmnopqrstuvwxyz" 33 | } 34 | ], 35 | "dac14c6111d3a02a23c4fc31ee4759387a7395cd": [ 36 | { 37 | "cvssScore": 8.2, 38 | "endColumn": 232, 39 | "endLine": 0, 40 | "extraDetails": { 41 | "secretDetails": { 42 | "name": "mockName2", 43 | "sub": "mockSub2" 44 | } 45 | }, 46 | "id": "dac14c6111d3a02a23c4fc31ee4759387a7395cd", 47 | "lineContent": "TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1 TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 TextExample", 48 | "ruleDescription": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", 49 | "ruleId": "jwt", 50 | "source": "testData/secrets/jwt.txt", 51 | "startColumn": 129, 52 | "startLine": 0, 53 | "validationStatus": "Unknown", 54 | "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2" 55 | }, 56 | { 57 | "cvssScore": 8.2, 58 | "endColumn": 166, 59 | "endLine": 1, 60 | "extraDetails": { 61 | "secretDetails": { 62 | "name": "mockName2", 63 | "sub": "mockSub2" 64 | } 65 | }, 66 | "id": "dac14c6111d3a02a23c4fc31ee4759387a7395cd", 67 | "lineContent": " Text_Example = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2", 68 | "ruleDescription": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", 69 | "ruleId": "jwt", 70 | "source": "testData/secrets/jwt.txt", 71 | "startColumn": 63, 72 | "startLine": 1, 73 | "validationStatus": "Unknown", 74 | "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2" 75 | } 76 | ], 77 | "4b0fb9bf4c96bd11404f2a3b187acbb621d8ca0c": [ 78 | { 79 | "cvssScore": 8.2, 80 | "endColumn": 116, 81 | "endLine": 0, 82 | "extraDetails": { 83 | "secretDetails": { 84 | "name": "mockName1", 85 | "sub": "mockSub1" 86 | } 87 | }, 88 | "id": "4b0fb9bf4c96bd11404f2a3b187acbb621d8ca0c", 89 | "lineContent": "TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1 TextExample eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMiIsIm5hbWUiOiJtb2NrTmFtZTIifQ.dummysignature2 TextExample", 90 | "ruleDescription": "Uncovered a JSON Web Token, which may lead to unauthorized access to web applications and sensitive user data.", 91 | "ruleId": "jwt", 92 | "source": "testData/secrets/jwt.txt", 93 | "startColumn": 13, 94 | "startLine": 0, 95 | "validationStatus": "Unknown", 96 | "value": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJtb2NrU3ViMSIsIm5hbWUiOiJtb2NrTmFtZTEifQ.dummysignature1" 97 | } 98 | ], 99 | "dfe5ea720d7c38631079841b754a34df29c25d93": [ 100 | { 101 | "cvssScore": 5.2, 102 | "endColumn": 102, 103 | "endLine": 0, 104 | "id": "dfe5ea720d7c38631079841b754a34df29c25d93", 105 | "lineContent": "TextExampleghp_1234567890abcdefghijklmnopqrstuvwxyzTextExampleghp_abcdefghijklmnopqrstuvwxyz1234567890TextExample", 106 | "ruleDescription": "Uncovered a GitHub Personal Access Token, potentially leading to unauthorized repository access and sensitive content exposure.", 107 | "ruleId": "github-pat", 108 | "source": "testData/secrets/github-pat.txt", 109 | "startColumn": 63, 110 | "startLine": 0, 111 | "validationStatus": "Invalid", 112 | "value": "ghp_abcdefghijklmnopqrstuvwxyz1234567890" 113 | } 114 | ] 115 | }, 116 | "totalItemsScanned": 3, 117 | "totalSecretsFound": 6 118 | } 119 | -------------------------------------------------------------------------------- /lib/reporting/sarif.go: -------------------------------------------------------------------------------- 1 | package reporting 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/checkmarx/2ms/v4/lib/config" 9 | "github.com/checkmarx/2ms/v4/lib/secrets" 10 | ) 11 | 12 | func writeSarif(report *Report, cfg *config.Config) (string, error) { 13 | sarif := Sarif{ 14 | Schema: "https://schemastore.azurewebsites.net/schemas/json/sarif-2.1.0-rtm.5.json", 15 | Version: "2.1.0", 16 | Runs: getRuns(report, cfg), 17 | } 18 | 19 | sarifReport, err := json.MarshalIndent(sarif, "", " ") 20 | if err != nil { 21 | return "", fmt.Errorf("failed to create Sarif report with error: %v", err) 22 | } 23 | 24 | return string(sarifReport), nil 25 | } 26 | 27 | func getRuns(report *Report, cfg *config.Config) []Runs { 28 | return []Runs{ 29 | { 30 | Tool: getTool(report, cfg), 31 | Results: getResults(report), 32 | }, 33 | } 34 | } 35 | 36 | func getTool(report *Report, cfg *config.Config) Tool { 37 | tool := Tool{ 38 | Driver: Driver{ 39 | Name: cfg.Name, 40 | SemanticVersion: cfg.Version, 41 | Rules: getRules(report), 42 | }, 43 | } 44 | 45 | return tool 46 | } 47 | 48 | func getRules(report *Report) []*SarifRule { 49 | uniqueRulesMap := make(map[string]*SarifRule) 50 | var reportRules []*SarifRule 51 | for _, reportSecrets := range report.Results { 52 | for _, secret := range reportSecrets { 53 | if _, exists := uniqueRulesMap[secret.RuleID]; !exists { 54 | uniqueRulesMap[secret.RuleID] = &SarifRule{ 55 | ID: secret.RuleID, 56 | FullDescription: &Message{ 57 | Text: secret.RuleDescription, 58 | }, 59 | } 60 | reportRules = append(reportRules, uniqueRulesMap[secret.RuleID]) 61 | } 62 | } 63 | } 64 | return reportRules 65 | } 66 | 67 | func hasNoResults(report *Report) bool { 68 | return len(report.Results) == 0 69 | } 70 | 71 | func createMessageText(ruleName, filePath string) string { 72 | // maintain only the filename if the scan target is git 73 | if strings.HasPrefix(filePath, "git show ") { 74 | filePathParts := strings.SplitN(filePath, ":", 2) 75 | if len(filePathParts) == 2 { 76 | filePath = filePathParts[1] 77 | } 78 | } 79 | 80 | return fmt.Sprintf("%s has detected secret for file %s.", ruleName, filePath) 81 | } 82 | 83 | func getResults(report *Report) []Results { 84 | var results []Results 85 | 86 | // if this report has no results, ensure that it is represented as [] instead of null/nil 87 | if hasNoResults(report) { 88 | results = make([]Results, 0) 89 | return results 90 | } 91 | 92 | for _, secretsSlice := range report.Results { 93 | for _, secret := range secretsSlice { 94 | props := Properties{ 95 | "validationStatus": secret.ValidationStatus, 96 | "cvssScore": secret.CvssScore, 97 | "resultId": secret.ID, 98 | } 99 | 100 | if secret.ExtraDetails != nil { 101 | if pageID, ok := secret.ExtraDetails["confluence.pageId"]; ok { 102 | props["confluence.pageId"] = pageID 103 | } 104 | } 105 | 106 | r := Results{ 107 | Message: Message{ 108 | Text: createMessageText(secret.RuleID, secret.Source), 109 | }, 110 | RuleId: secret.RuleID, 111 | Locations: getLocation(secret), 112 | Properties: props, 113 | } 114 | results = append(results, r) 115 | } 116 | } 117 | return results 118 | } 119 | 120 | func getLocation(secret *secrets.Secret) []Locations { 121 | return []Locations{ 122 | { 123 | PhysicalLocation: PhysicalLocation{ 124 | ArtifactLocation: ArtifactLocation{ 125 | URI: secret.Source, 126 | }, 127 | Region: Region{ 128 | StartLine: secret.StartLine, 129 | EndLine: secret.EndLine, 130 | StartColumn: secret.StartColumn, 131 | EndColumn: secret.EndColumn, 132 | Snippet: Snippet{ 133 | Text: secret.Value, 134 | Properties: Properties{ 135 | "lineContent": strings.TrimSpace(secret.LineContent), 136 | }, 137 | }, 138 | }, 139 | }, 140 | }, 141 | } 142 | } 143 | 144 | type Sarif struct { 145 | Schema string `json:"$schema"` 146 | Version string `json:"version"` 147 | Runs []Runs `json:"runs"` 148 | } 149 | type ShortDescription struct { 150 | Text string `json:"text"` 151 | } 152 | 153 | type Driver struct { 154 | Name string `json:"name"` 155 | SemanticVersion string `json:"semanticVersion"` 156 | Rules []*SarifRule `json:"rules,omitempty"` 157 | } 158 | 159 | type Tool struct { 160 | Driver Driver `json:"driver"` 161 | } 162 | 163 | type SarifRule struct { 164 | ID string `json:"id"` 165 | FullDescription *Message `json:"fullDescription,omitempty"` 166 | } 167 | 168 | type Message struct { 169 | Text string `json:"text"` 170 | } 171 | 172 | type ArtifactLocation struct { 173 | URI string `json:"uri"` 174 | } 175 | 176 | type Region struct { 177 | StartLine int `json:"startLine"` 178 | StartColumn int `json:"startColumn"` 179 | EndLine int `json:"endLine"` 180 | EndColumn int `json:"endColumn"` 181 | Snippet Snippet `json:"snippet"` 182 | } 183 | 184 | type Snippet struct { 185 | Text string `json:"text"` 186 | Properties Properties `json:"properties,omitempty"` 187 | } 188 | 189 | type PhysicalLocation struct { 190 | ArtifactLocation ArtifactLocation `json:"artifactLocation"` 191 | Region Region `json:"region"` 192 | } 193 | 194 | type Locations struct { 195 | PhysicalLocation PhysicalLocation `json:"physicalLocation"` 196 | } 197 | 198 | type Results struct { 199 | Message Message `json:"message"` 200 | RuleId string `json:"ruleId"` 201 | Locations []Locations `json:"locations"` 202 | Properties Properties `json:"properties,omitempty"` 203 | } 204 | 205 | type Runs struct { 206 | Tool Tool `json:"tool"` 207 | Results []Results `json:"results"` 208 | } 209 | 210 | type Properties map[string]interface{} 211 | -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/checkmarx/2ms/v4/engine" 8 | "github.com/checkmarx/2ms/v4/lib/config" 9 | "github.com/checkmarx/2ms/v4/plugins" 10 | "github.com/rs/zerolog" 11 | "github.com/rs/zerolog/log" 12 | "github.com/spf13/cobra" 13 | "github.com/spf13/viper" 14 | ) 15 | 16 | var Version = "0.0.0" 17 | 18 | const ( 19 | outputFormatRegexpPattern = `^(ya?ml|json|sarif)$` 20 | configFileFlag = "config" 21 | 22 | logLevelFlagName = "log-level" 23 | reportPathFlagName = "report-path" 24 | stdoutFormatFlagName = "stdout-format" 25 | customRegexRuleFlagName = "regex" 26 | ruleFlagName = "rule" 27 | ignoreRuleFlagName = "ignore-rule" 28 | ignoreFlagName = "ignore-result" 29 | allowedValuesFlagName = "allowed-values" 30 | specialRulesFlagName = "add-special-rule" 31 | ignoreOnExitFlagName = "ignore-on-exit" 32 | maxTargetMegabytesFlagName = "max-target-megabytes" 33 | validate = "validate" 34 | ) 35 | 36 | var ( 37 | logLevelVar string 38 | reportPathVar []string 39 | stdoutFormatVar string 40 | customRegexRuleVar []string 41 | ignoreOnExitVar = ignoreOnExitNone 42 | engineConfigVar engine.EngineConfig 43 | validateVar bool 44 | ) 45 | 46 | const envPrefix = "2MS" 47 | 48 | var configFilePath string 49 | var vConfig = viper.New() 50 | 51 | var allPlugins = []plugins.IPlugin{ 52 | plugins.NewConfluencePlugin(), 53 | &plugins.DiscordPlugin{}, 54 | &plugins.FileSystemPlugin{}, 55 | &plugins.SlackPlugin{}, 56 | &plugins.PaligoPlugin{}, 57 | plugins.NewGitPlugin(), 58 | } 59 | 60 | func Execute() (int, error) { 61 | vConfig.SetEnvPrefix(envPrefix) 62 | vConfig.AutomaticEnv() 63 | 64 | rootCmd := &cobra.Command{ 65 | Use: "2ms", 66 | Short: "2ms Secrets Detection", 67 | Long: "2ms Secrets Detection: A tool to detect secrets in public websites and communication services.", 68 | Version: Version, 69 | } 70 | 71 | setupFlags(rootCmd) 72 | 73 | rootCmd.AddCommand(engine.GetRulesCommand(&engineConfigVar)) 74 | 75 | // Override detector worker pool size from environment if set 76 | if detectorWorkerPoolSize := vConfig.GetInt("TWOMS_DETECTOR_WORKERPOOL_SIZE"); detectorWorkerPoolSize != 0 { 77 | engineConfigVar.DetectorWorkerPoolSize = detectorWorkerPoolSize 78 | log.Info().Msgf("TWOMS_DETECTOR_WORKERPOOL_SIZE is set to %d", detectorWorkerPoolSize) 79 | } 80 | 81 | group := "Scan Commands" 82 | rootCmd.AddGroup(&cobra.Group{Title: group, ID: group}) 83 | 84 | channels := plugins.NewChannels() 85 | var engineInstance engine.IEngine 86 | 87 | // Process flags and initialize engine with complete configuration 88 | cobra.OnInitialize(func() { 89 | if err := processFlags(rootCmd); err != nil { 90 | cobra.CheckErr(err) 91 | } 92 | 93 | if len(engineConfigVar.CustomRegexPatterns) > 0 { 94 | log.Info().Msgf("Custom regex patterns configured: %v", engineConfigVar.CustomRegexPatterns) 95 | } 96 | if len(engineConfigVar.IgnoreList) > 0 { 97 | log.Info().Msgf("Ignore rules configured: %v", engineConfigVar.IgnoreList) 98 | } 99 | 100 | var err error 101 | engineInstance, err = engine.Init(&engineConfigVar, engine.WithPluginChannels(channels)) 102 | if err != nil { 103 | cobra.CheckErr(fmt.Errorf("failed to initialize engine: %w", err)) 104 | } 105 | }) 106 | 107 | // Set up plugins 108 | for _, plugin := range allPlugins { 109 | subCommand, err := plugin.DefineCommand(channels.GetItemsCh(), channels.GetErrorsCh()) 110 | if err != nil { 111 | return 0, fmt.Errorf("error while defining command for plugin %s: %s", plugin.GetName(), err.Error()) 112 | } 113 | subCommand.GroupID = group 114 | 115 | pluginPreRun := subCommand.PreRunE 116 | // Capture plugin name for closure 117 | pluginName := plugin.GetName() 118 | subCommand.PreRunE = func(cmd *cobra.Command, args []string) error { 119 | // run plugin's own PreRunE (if any) 120 | if pluginPreRun != nil { 121 | if err := pluginPreRun(cmd, args); err != nil { 122 | return err 123 | } 124 | } 125 | // run engine-level PreRunE 126 | return preRun(pluginName, engineInstance, cmd, args) 127 | } 128 | subCommand.PostRunE = func(cmd *cobra.Command, args []string) error { 129 | return postRun(engineInstance) 130 | } 131 | rootCmd.AddCommand(subCommand) 132 | } 133 | 134 | listenForErrors(channels.GetErrorsCh()) 135 | 136 | if err := rootCmd.ExecuteContext(context.Background()); err != nil { 137 | return 0, err 138 | } 139 | 140 | if engineInstance != nil { 141 | return engineInstance.GetReport().GetTotalSecretsFound(), nil 142 | } 143 | 144 | return 0, nil 145 | } 146 | 147 | func preRun(pluginName string, engineInstance engine.IEngine, _ *cobra.Command, _ []string) error { 148 | if engineInstance == nil { 149 | return fmt.Errorf("engine instance not initialized") 150 | } 151 | 152 | if err := validateFormat(stdoutFormatVar, reportPathVar); err != nil { 153 | return err 154 | } 155 | 156 | engineInstance.Scan(pluginName) 157 | 158 | return nil 159 | } 160 | 161 | func postRun(engineInstance engine.IEngine) error { 162 | if engineInstance == nil { 163 | return fmt.Errorf("engine instance not initialized") 164 | } 165 | 166 | engineInstance.Wait() 167 | 168 | cfg := config.LoadConfig("2ms", Version) 169 | report := engineInstance.GetReport() 170 | 171 | if report.GetTotalItemsScanned() > 0 { 172 | if zerolog.GlobalLevel() != zerolog.Disabled { 173 | if err := report.ShowReport(stdoutFormatVar, cfg); err != nil { 174 | return err 175 | } 176 | } 177 | 178 | if len(reportPathVar) > 0 { 179 | err := report.WriteFile(reportPathVar, cfg) 180 | if err != nil { 181 | return fmt.Errorf("failed to create report file with error: %s", err) 182 | } 183 | } 184 | } else { 185 | log.Info().Msg("Scan completed with empty content") 186 | } 187 | 188 | if err := engineInstance.Shutdown(); err != nil { 189 | return err 190 | } 191 | 192 | return nil 193 | } 194 | -------------------------------------------------------------------------------- /plugins/filesystem_test.go: -------------------------------------------------------------------------------- 1 | package plugins 2 | 3 | import ( 4 | "fmt" 5 | "io/fs" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestGetItem(t *testing.T) { 14 | tmpFile, err := os.CreateTemp("", "TestGetItem") 15 | assert.NoError(t, err, "failed to create temp file") 16 | defer func(name string) { 17 | err := os.Remove(name) 18 | assert.NoError(t, err, "failed to remove temp file") 19 | }(tmpFile.Name()) 20 | 21 | err = tmpFile.Close() 22 | assert.NoError(t, err, "failed to close temp file") 23 | 24 | plugin := &FileSystemPlugin{ 25 | ProjectName: "TestProject", 26 | } 27 | 28 | it := plugin.getItem(tmpFile.Name()) 29 | 30 | expectedID := fmt.Sprintf("%s-%s-%s", plugin.GetName(), plugin.ProjectName, tmpFile.Name()) 31 | assert.Equal(t, expectedID, it.ID, "ID should match the expected format") 32 | } 33 | 34 | func TestGetItems(t *testing.T) { 35 | tmpFile, err := os.CreateTemp("", "TestGetItems") 36 | assert.NoError(t, err, "failed to create temporary file") 37 | defer func(name string) { 38 | err := os.Remove(name) 39 | assert.NoError(t, err, "failed to remove temp file") 40 | }(tmpFile.Name()) 41 | 42 | validContent := "valid mock content" 43 | _, err = tmpFile.WriteString(validContent) 44 | assert.NoError(t, err, "failed to write to temporary file") 45 | 46 | err = tmpFile.Close() 47 | assert.NoError(t, err, "failed to close temporary file") 48 | 49 | validFile := tmpFile.Name() 50 | fileList := []string{validFile} 51 | 52 | itemsChan := make(chan ISourceItem, len(fileList)) 53 | errsChan := make(chan error, len(fileList)) 54 | 55 | plugin := &FileSystemPlugin{ 56 | ProjectName: "TestProject", 57 | } 58 | 59 | plugin.sendItems(itemsChan, fileList) 60 | close(errsChan) 61 | 62 | var items []ISourceItem 63 | for itm := range itemsChan { 64 | items = append(items, itm) 65 | } 66 | 67 | assert.Equal(t, 1, len(items), "should have one valid item") 68 | _, ok := items[0].(item) 69 | assert.True(t, ok, "item should be of type item") 70 | } 71 | 72 | func TestGetFiles(t *testing.T) { 73 | tests := []struct { 74 | name string 75 | nonExistent bool 76 | setup func(t *testing.T, baseDir string) (ignoredPatterns []string, expectedFiles []string) 77 | err error 78 | }{ 79 | { 80 | name: "All valid files", 81 | nonExistent: false, 82 | setup: func(t *testing.T, baseDir string) ([]string, []string) { 83 | file1 := filepath.Join(baseDir, "file1.txt") 84 | err := os.WriteFile(file1, []byte("content1"), 0644) 85 | assert.NoError(t, err) 86 | file2 := filepath.Join(baseDir, "file2.txt") 87 | err = os.WriteFile(file2, []byte("content2"), 0644) 88 | assert.NoError(t, err) 89 | return []string{}, []string{file1, file2} 90 | }, 91 | }, 92 | { 93 | name: "Skip empty files", 94 | nonExistent: false, 95 | setup: func(t *testing.T, baseDir string) ([]string, []string) { 96 | empty := filepath.Join(baseDir, "empty.txt") 97 | err := os.WriteFile(empty, []byte(""), 0644) 98 | assert.NoError(t, err) 99 | valid := filepath.Join(baseDir, "file.txt") 100 | err = os.WriteFile(valid, []byte("content"), 0644) 101 | assert.NoError(t, err) 102 | return []string{}, []string{valid} 103 | }, 104 | }, 105 | { 106 | name: "Ignore folder via global ignoredFolders", 107 | nonExistent: false, 108 | setup: func(t *testing.T, baseDir string) ([]string, []string) { 109 | ignoredDir := filepath.Join(baseDir, "ignoredFolder") 110 | err := os.Mkdir(ignoredDir, 0755) 111 | assert.NoError(t, err) 112 | 113 | ignoredFile := filepath.Join(ignoredDir, "file.txt") 114 | err = os.WriteFile(ignoredFile, []byte("content"), 0644) 115 | assert.NoError(t, err) 116 | 117 | valid := filepath.Join(baseDir, "file.txt") 118 | err = os.WriteFile(valid, []byte("content"), 0644) 119 | assert.NoError(t, err) 120 | return []string{}, []string{valid} 121 | }, 122 | }, 123 | { 124 | name: "Ignore files by pattern", 125 | nonExistent: false, 126 | setup: func(t *testing.T, baseDir string) ([]string, []string) { 127 | ignoreFile := filepath.Join(baseDir, "skip.ignore") 128 | err := os.WriteFile(ignoreFile, []byte("ignored content"), 0644) 129 | assert.NoError(t, err) 130 | valid := filepath.Join(baseDir, "file.txt") 131 | err = os.WriteFile(valid, []byte("content"), 0644) 132 | assert.NoError(t, err) 133 | return []string{"*.ignore"}, []string{valid} 134 | }, 135 | }, 136 | { 137 | name: "Non-existent directory", 138 | nonExistent: true, 139 | setup: func(t *testing.T, baseDir string) ([]string, []string) { 140 | return []string{}, []string{} 141 | }, 142 | err: fs.ErrNotExist, 143 | }, 144 | } 145 | 146 | for _, tc := range tests { 147 | t.Run(tc.name, func(t *testing.T) { 148 | var baseDir string 149 | var err error 150 | 151 | if !tc.nonExistent { 152 | baseDir, err = os.MkdirTemp("", "testfiles") 153 | assert.NoError(t, err) 154 | defer func(path string) { 155 | err := os.RemoveAll(path) 156 | assert.NoError(t, err) 157 | }(baseDir) 158 | } 159 | 160 | ignoredPatterns, expectedFiles := tc.setup(t, baseDir) 161 | 162 | plugin := &FileSystemPlugin{ 163 | ProjectName: "TestProject", 164 | Path: baseDir, 165 | Ignored: ignoredPatterns, 166 | } 167 | 168 | itemsChan := make(chan ISourceItem, 10) 169 | errsChan := make(chan error, 10) 170 | 171 | fileList, err := plugin.getFiles() 172 | if tc.err != nil { 173 | assert.ErrorIs(t, err, tc.err) 174 | return 175 | } 176 | 177 | plugin.sendItems(itemsChan, fileList) 178 | close(errsChan) 179 | 180 | var collectedFiles []string 181 | for itm := range itemsChan { 182 | it, ok := itm.(item) 183 | assert.True(t, ok, "item should be of type item") 184 | collectedFiles = append(collectedFiles, it.Source) 185 | } 186 | 187 | expectedMap := make(map[string]bool) 188 | for _, f := range expectedFiles { 189 | expectedMap[f] = true 190 | } 191 | for _, f := range collectedFiles { 192 | delete(expectedMap, f) 193 | } 194 | assert.Equal(t, 0, len(expectedMap), "not all expected files were returned") 195 | }) 196 | } 197 | } 198 | --------------------------------------------------------------------------------