├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── dependabot.yml └── workflows │ ├── codeql.yml │ ├── dependency-review.yml │ ├── release.yaml │ └── test-and-coverage.yml ├── .gitignore ├── .goreleaser.yaml ├── .pre-commit-config.yaml ├── .pre-commit-hooks.yaml ├── .talismanrc ├── CODE_OF_CONDUCT.md ├── LICENSE ├── README.md ├── checksumcalculator ├── checksumcalculator.go └── checksumcalculator_test.go ├── cmd ├── acceptance_test.go ├── checksum_cmd.go ├── checksum_cmd_test.go ├── pattern_cmd.go ├── pre_commit_hook.go ├── pre_push_hook.go ├── runner.go ├── scanner_cmd.go ├── scanner_cmd_test.go ├── talisman.go ├── talisman_internal_test.go └── talisman_test.go ├── contributing.md ├── detector ├── chain.go ├── chain_test.go ├── detector │ └── detector.go ├── filecontent │ ├── base64_aggressive_detector.go │ ├── base64_aggressive_detector_test.go │ ├── base64_detector.go │ ├── base64_detector_test.go │ ├── filecontent_credit_card_detector.go │ ├── filecontent_credit_card_detector_test.go │ ├── filecontent_detector.go │ ├── filecontent_detector_test.go │ ├── hex_detector.go │ ├── hex_detector_test.go │ ├── shannon_entropy.go │ ├── shannon_entropy_test.go │ ├── word_check.go │ ├── word_check_test.go │ └── word_check_words_list.go ├── filename │ ├── filename_detector.go │ └── filename_detector_test.go ├── filesize │ ├── filesize_detector.go │ └── filesize_detector_test.go ├── helpers │ ├── detection_results.go │ ├── detection_results_test.go │ ├── ignore_evaluator.go │ └── ignore_evaluator_test.go ├── pattern │ ├── match_pattern.go │ ├── match_pattern_test.go │ ├── pattern_detector.go │ └── pattern_detector_test.go └── severity │ ├── pattern_severity.go │ ├── severity.go │ ├── severity_config.go │ └── severity_test.go ├── examples └── schema-store-talismanrc.json ├── git_testing ├── git_testing.go └── git_testing_test.go ├── gitrepo ├── constants.go ├── git_readers.go ├── gitrepo.go ├── gitrepo_test.go └── pixel.jpg ├── global_install_scripts ├── add_to_talismanignore.bash ├── add_to_talismanignore_in_git_repo.bash ├── install.bash ├── remove_hooks.bash ├── setup_talisman.bash ├── setup_talisman_hook_in_repo.bash ├── talisman_hook_script.bash ├── uninstall.bash ├── uninstall_git_repo_hook.bash └── update_talisman.bash ├── go.mod ├── go.sum ├── install.sh ├── internal └── mock │ ├── checksumcalculator │ └── checksumcalculator.go │ ├── prompt │ └── survey.go │ └── utility │ └── sha_256_hasher.go ├── mock_scripts ├── prompt └── survey.go ├── report └── report.go ├── scanner ├── scanner.go └── scanner_test.go ├── talisman.png ├── talismanrc ├── rc_file.go ├── rc_file_test.go ├── scopes.go ├── talismanrc.go ├── talismanrc_test.go ├── types.go ├── types_test.go └── util.go ├── test-install.sh ├── third-party └── schema-store-talismanrc.json └── utility ├── progress_bar.go ├── progress_bar_test.go ├── sha_256_hasher.go ├── sha_256_hasher_test.go ├── utility.go └── utility_test.go /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea/enhancement for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directory: "/" 5 | schedule: 6 | interval: "monthly" 7 | - package-ecosystem: "github-actions" 8 | directory: "/" 9 | schedule: 10 | interval: "monthly" 11 | -------------------------------------------------------------------------------- /.github/workflows/codeql.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "main" ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "main" ] 20 | schedule: 21 | - cron: '17 22 * * 6' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'go' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v4 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v3 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | 52 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 53 | # queries: security-extended,security-and-quality 54 | 55 | 56 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 57 | # If this step fails, then you should remove it and run the build manually (see below) 58 | - name: Autobuild 59 | uses: github/codeql-action/autobuild@v3 60 | 61 | # ℹ️ Command-line programs to run using the OS shell. 62 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 63 | 64 | # If the Autobuild fails above, remove it and uncomment the following three lines. 65 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 66 | 67 | # - run: | 68 | # echo "Run, Build Application using script" 69 | # ./location_of_script_within_repo/buildscript.sh 70 | 71 | - name: Perform CodeQL Analysis 72 | uses: github/codeql-action/analyze@v3 73 | with: 74 | category: "/language:${{matrix.language}}" 75 | -------------------------------------------------------------------------------- /.github/workflows/dependency-review.yml: -------------------------------------------------------------------------------- 1 | # Dependency Review Action 2 | # 3 | # This Action will scan dependency manifest files that change as part of a Pull Request, surfacing known-vulnerable versions of the packages declared or updated in the PR. Once installed, if the workflow run is marked as required, PRs introducing known-vulnerable packages will be blocked from merging. 4 | # 5 | # Source repository: https://github.com/actions/dependency-review-action 6 | # Public documentation: https://docs.github.com/en/code-security/supply-chain-security/understanding-your-software-supply-chain/about-dependency-review#dependency-review-enforcement 7 | name: 'Dependency Review' 8 | on: [pull_request] 9 | 10 | permissions: 11 | contents: read 12 | 13 | jobs: 14 | dependency-review: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: 'Checkout Repository' 18 | uses: actions/checkout@v4 19 | - name: 'Dependency Review' 20 | uses: actions/dependency-review-action@v4 21 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | permissions: 9 | contents: write 10 | packages: write 11 | 12 | jobs: 13 | goreleaser: 14 | runs-on: ubuntu-latest 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v4 18 | with: 19 | fetch-depth: 0 20 | - name: Fetch all tags 21 | run: git fetch --force --tags 22 | - name: Set up Go 23 | uses: actions/setup-go@v5 24 | with: 25 | go-version-file: go.mod 26 | - name: Generate release notes 27 | id: release-notes 28 | uses: actions/github-script@v7 29 | with: 30 | result-encoding: string 31 | script: | 32 | return github.rest.repos.generateReleaseNotes({ 33 | owner: context.repo.owner, 34 | repo: context.repo.repo, 35 | tag_name: "${{ github.ref_name }}", 36 | }).then(response => response.data.body) 37 | - name: Write release notes to file 38 | run: | 39 | cat << EOF > release-notes.md 40 | ${{ steps.release-notes.outputs.result }} 41 | EOF 42 | - name: Run GoReleaser 43 | uses: goreleaser/goreleaser-action@v6 44 | with: 45 | version: '~> v2' 46 | args: release --clean --release-notes release-notes.md 47 | env: 48 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 49 | -------------------------------------------------------------------------------- /.github/workflows/test-and-coverage.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | push: 5 | branches: '**' 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | - name: Set up Go 15 | uses: actions/setup-go@v5 16 | with: 17 | go-version-file: go.mod 18 | - name: Format 19 | run: | 20 | go fmt ./... 21 | git diff --no-patch --exit-code 22 | - name: Test 23 | run: | 24 | git config user.email "talisman-maintainers@thoughtworks.com" 25 | git config user.name "Talisman Maintainers" 26 | go test -covermode=count -v ./... 27 | - name: Install bash_unit 28 | run: bash <(curl -s https://raw.githubusercontent.com/pgrange/bash_unit/master/install.sh) 29 | - name: Test install script 30 | run: ./bash_unit test-install.sh 31 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .vagrant 3 | _workspace 4 | *.swp 5 | talisman 6 | checksums 7 | vendor/ 8 | detector/.talismanrc 9 | .vscode/** 10 | coverage.out 11 | coverage.txt 12 | .talismanrc 13 | talisman_reports/ 14 | *.iml 15 | dist/ 16 | *.out 17 | *.pprof 18 | *.mprof 19 | .envrc 20 | talisman_html_report 21 | .vscode 22 | .idea 23 | 24 | dist/ 25 | out/ 26 | .goreleaser-schema.json 27 | 28 | ### Go template 29 | # If you prefer the allow list template instead of the deny list, see community template: 30 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 31 | # 32 | # Binaries for programs and plugins 33 | *.exe 34 | *.exe~ 35 | *.dll 36 | *.so 37 | *.dylib 38 | 39 | # Test binary, built with `go test -c` 40 | *.test 41 | 42 | # Output of the go coverage tool, specifically when used with LiteIDE 43 | 44 | # Dependency directories (remove the comment below to include it) 45 | # vendor/ 46 | 47 | # Go workspace file 48 | go.work 49 | 50 | release-notes.md 51 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | project_name: talisman 3 | 4 | before: 5 | hooks: 6 | - go mod tidy 7 | 8 | builds: 9 | - env: 10 | - CGO_ENABLED=0 11 | goos: 12 | - linux 13 | - windows 14 | - darwin 15 | 16 | # Custom ldflags templates. 17 | # Default is `-s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} -X main.builtBy=goreleaser`. 18 | ldflags: 19 | - -s -w -X main.Version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.Date}} 20 | main: ./cmd 21 | 22 | release: 23 | draft: true 24 | replace_existing_draft: true 25 | 26 | archives: 27 | - meta: false 28 | formats: binary 29 | wrap_in_directory: true 30 | name_template: "{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}" 31 | 32 | checksum: 33 | name_template: 'checksums' 34 | algorithm: sha256 35 | 36 | snapshot: 37 | version_template: "{{ incpatch .Version }}-next" 38 | 39 | changelog: 40 | sort: asc 41 | filters: 42 | exclude: 43 | - '^docs:' 44 | - '^test:' 45 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v5.0.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: check-yaml 8 | - id: check-added-large-files 9 | - repo: local 10 | hooks: 11 | - id: go-format 12 | name: Go Format 13 | language: system 14 | entry: go fmt ./... 15 | types: [go] 16 | pass_filenames: false 17 | - id: go-test 18 | name: Go Tests 19 | language: system 20 | entry: go test ./... 21 | types: [go] 22 | pass_filenames: false 23 | -------------------------------------------------------------------------------- /.pre-commit-hooks.yaml: -------------------------------------------------------------------------------- 1 | - id: talisman-commit 2 | name: talisman 3 | entry: cmd --githook pre-commit 4 | stages: [pre-commit] 5 | # talisman currently discovers files by itself and does not take them on the cli 6 | pass_filenames: false 7 | types: [text] 8 | language: golang 9 | language_version: 1.23.0 10 | minimum_pre_commit_version: 3.2.0 11 | 12 | - id: talisman-push 13 | name: talisman 14 | entry: cmd --githook pre-push 15 | stages: [pre-push] 16 | # talisman currently discovers files by itself and does not take them on the cli 17 | pass_filenames: false 18 | types: [text] 19 | language: golang 20 | language_version: 1.23.0 21 | minimum_pre_commit_version: 3.2.0 22 | -------------------------------------------------------------------------------- /.talismanrc: -------------------------------------------------------------------------------- 1 | fileignoreconfig: 2 | - filename: README.md 3 | ignore_detectors: 4 | - filecontent 5 | - filename: build_binaries.bash 6 | checksum: fdc955c83e408178fcccd834c6db0b5622edb876ea9e72d25da8ef8ab1e87192 7 | - filename: checksumcalculator/checksumcalculator_test.go 8 | checksum: d74273fe8cb37c2c4a7469785eb21b4f1d23a3437927612f8a5964f5011f0998 9 | - filename: detector/detection_results_test.go 10 | checksum: 69fed055782cddfe0f0d23ea440cef9f9dd0b9e8a3c8a73856741bb26257b223 11 | ignore_detectors: 12 | - filecontent 13 | - filename: detector/filecontent/filecontent_detector_test.go 14 | checksum: 05d00bb99452d37ab45de28c9e074357e5be22a80d95dad1b2cdd01074b25b2a 15 | - filename: detector/filename/filename_detector.go 16 | checksum: 5404565683a7e812fa98ff2d14237c4d1ba7dc5b4aca2dd3ba663b33dc8ddae7 17 | - filename: detector/filename/filename_detector_test.go 18 | checksum: d04b9a2cc51e6835c1b3c13fc0872837b34bde2a5f2c070b1047b055299d5cf9 19 | - filename: detector/match_pattern_test.go 20 | checksum: d68aa0e06355e3b848941727d1fcb32cf47e3d615f9921f0db39855325010446 21 | - filename: detector/pattern/match_pattern_test.go 22 | checksum: c95b8106ced5ad34ec1d00773a05f8789715034a734197c93cdaa4ed5036c177 23 | - filename: detector/pattern/pattern_detector.go 24 | checksum: 78cddc944d4092ae2e88535d04f05281784848a990fb55a9d38339f29080a239 25 | - filename: detector/pattern/pattern_detector_test.go 26 | checksum: 4d70b790f28f2d23d506f808d489aa43f1efd2514549ae6a83a535e1223382e3 27 | - filename: detector/pattern_detector_test.go 28 | checksum: 4d70b790f28f2d23d506f808d489aa43f1efd2514549ae6a83a535e1223382e3 29 | - filename: detector/severity/severity_config.go 30 | checksum: a9b041a6f5a5c93146a79dfc7b3d2d26c4a6cf621aa0aa92db7e8c343c6926a5 31 | - filename: global_install_scripts/install.bash 32 | checksum: bf0da9b8b6f779f502564166323a903a49b56b9d8df2597729a5b96c8f066074 33 | - filename: install.sh 34 | checksum: c909f6a1caefba3f196d489f9262608044be596a44793c2173ec55b98ecec649 35 | - filename: talismanrc/talismanrc_test.go 36 | checksum: eab40d0745dc215267da86ef7f926be77c4d46e9a248dadbc7e52aa186e82853 37 | scopeconfig: 38 | - scope: go 39 | version: "1.0" 40 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at talisman-maintainers@thoughtworks.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 ThoughtWorks Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /checksumcalculator/checksumcalculator.go: -------------------------------------------------------------------------------- 1 | package checksumcalculator 2 | 3 | import ( 4 | "strings" 5 | "talisman/gitrepo" 6 | "talisman/talismanrc" 7 | "talisman/utility" 8 | ) 9 | 10 | type ChecksumCalculator interface { 11 | SuggestTalismanRC(fileNamePatterns []string) string 12 | CalculateCollectiveChecksumForPattern(fileNamePattern string) string 13 | } 14 | 15 | type checksumCalculator struct { 16 | allTrackedFiles []gitrepo.Addition 17 | hasher utility.SHA256Hasher 18 | } 19 | 20 | // NewChecksumCalculator returns new instance of the CheckSumDetector 21 | func NewChecksumCalculator(hasher utility.SHA256Hasher, gitAdditions []gitrepo.Addition) ChecksumCalculator { 22 | return &checksumCalculator{hasher: hasher, allTrackedFiles: gitAdditions} 23 | } 24 | 25 | // SuggestTalismanRC returns the suggestion for .talismanrc format 26 | func (cc *checksumCalculator) SuggestTalismanRC(fileNamePatterns []string) string { 27 | var fileIgnoreConfigs []talismanrc.FileIgnoreConfig 28 | result := strings.Builder{} 29 | for _, pattern := range fileNamePatterns { 30 | collectiveChecksum := cc.CalculateCollectiveChecksumForPattern(pattern) 31 | if collectiveChecksum != "" { 32 | fileIgnoreConfigs = append(fileIgnoreConfigs, talismanrc.IgnoreFileWithChecksum(pattern, collectiveChecksum)) 33 | } 34 | } 35 | if len(fileIgnoreConfigs) != 0 { 36 | result.WriteString("\n\x1b[33m.talismanrc format for given file names / patterns\x1b[0m\n") 37 | result.Write([]byte(talismanrc.SuggestRCFor(fileIgnoreConfigs))) 38 | } 39 | return result.String() 40 | } 41 | 42 | // CalculateCollectiveChecksumForPattern calculates and returns the checksum for files matching the input pattern 43 | func (cc *checksumCalculator) CalculateCollectiveChecksumForPattern(fileNamePattern string) string { 44 | var patternPaths []string 45 | currentCollectiveChecksum := "" 46 | for _, file := range cc.allTrackedFiles { 47 | if file.Matches(fileNamePattern) { 48 | patternPaths = append(patternPaths, string(file.Path)) 49 | } 50 | } 51 | // Calculate current collective checksum 52 | patternPaths = utility.UniqueItems(patternPaths) 53 | if len(patternPaths) != 0 { 54 | currentCollectiveChecksum = cc.hasher.CollectiveSHA256Hash(patternPaths) 55 | } 56 | return currentCollectiveChecksum 57 | } 58 | -------------------------------------------------------------------------------- /checksumcalculator/checksumcalculator_test.go: -------------------------------------------------------------------------------- 1 | package checksumcalculator 2 | 3 | import ( 4 | "talisman/gitrepo" 5 | "talisman/utility" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | var defaultSHA256Hasher utility.SHA256Hasher 12 | 13 | func init() { 14 | defaultSHA256Hasher = utility.MakeHasher("default", ".") 15 | } 16 | 17 | func TestNewChecksumCalculator(t *testing.T) { 18 | t.Run("should return empty CollectiveChecksum when non existing file name pattern is sent", func(t *testing.T) { 19 | gitAdditions := []gitrepo.Addition{ 20 | { 21 | Path: "GitRepoPath1", 22 | Name: "GitRepoName1", 23 | }, 24 | } 25 | expectedCC := "" 26 | fileNamePattern := "*NonExistenceFileNamePattern" 27 | cc := NewChecksumCalculator(defaultSHA256Hasher, gitAdditions) 28 | 29 | actualCC := cc.CalculateCollectiveChecksumForPattern(fileNamePattern) 30 | 31 | assert.Equal(t, expectedCC, actualCC) 32 | }) 33 | 34 | t.Run("should return CollectiveChecksum when existing file name pattern is sent", func(t *testing.T) { 35 | gitAdditions := []gitrepo.Addition{ 36 | { 37 | Path: "GitRepoPath1/GitRepoName1", 38 | Name: "GitRepoName1", 39 | }, 40 | } 41 | expectedCC := "19250a996e1200d33e91454bc662efae7682410e5347cfc56b0ff386dfbc10ae" 42 | fileNamePattern := "GitRepoPath1/" 43 | cc := NewChecksumCalculator(defaultSHA256Hasher, gitAdditions) 44 | 45 | actualCC := cc.CalculateCollectiveChecksumForPattern(fileNamePattern) 46 | 47 | assert.Equal(t, expectedCC, actualCC) 48 | }) 49 | 50 | t.Run("should return the files own CollectiveChecksum when same file name is present in subfolders", func(t *testing.T) { 51 | gitAdditions := []gitrepo.Addition{ 52 | { 53 | Path: "hello.txt", 54 | Name: "hello.txt", 55 | }, 56 | { 57 | Path: "subfolder/hello.txt", 58 | Name: "hello.txt", 59 | }, 60 | } 61 | hello_expectedCC := "9d30c2e4bcf181bba07374cc416f1892d89918038bce5172776475347c4d2d69" 62 | fileNamePattern1 := "hello.txt" 63 | cc1 := NewChecksumCalculator(defaultSHA256Hasher, gitAdditions) 64 | actualCC := cc1.CalculateCollectiveChecksumForPattern(fileNamePattern1) 65 | assert.Equal(t, hello_expectedCC, actualCC) 66 | 67 | subfolder_hello_expectedCC := "6c779c16bcc2e63c659be7649a531650210d6b96ae590a146f9ccdca383587f6" 68 | fileNamePattern2 := "subfolder/hello.txt" 69 | cc2 := NewChecksumCalculator(defaultSHA256Hasher, gitAdditions) 70 | subfolder_actualCC := cc2.CalculateCollectiveChecksumForPattern(fileNamePattern2) 71 | assert.Equal(t, subfolder_hello_expectedCC, subfolder_actualCC) 72 | 73 | all_txt_expectedCC := "aba77c9077539130e21a8f275fc5f1f43b7f0589e392bc89e4b96c578f0a9184" 74 | fileNamePattern3 := "*.txt" 75 | cc3 := NewChecksumCalculator(defaultSHA256Hasher, gitAdditions) 76 | all_txt_actualCC := cc3.CalculateCollectiveChecksumForPattern(fileNamePattern3) 77 | assert.Equal(t, all_txt_expectedCC, all_txt_actualCC) 78 | }) 79 | } 80 | 81 | func TestDefaultChecksumCalculator_SuggestTalismanRC(t *testing.T) { 82 | t.Run("should return no suggestion for .talismanrc format when no matching file name patterns is sent", func(t *testing.T) { 83 | gitAdditions := []gitrepo.Addition{ 84 | { 85 | Path: "GitRepoPath1", 86 | Name: "GitRepoName1", 87 | }, { 88 | Path: "GitRepoPath2", 89 | Name: "GitRepoName2", 90 | }, 91 | } 92 | expectedCC := "" 93 | fileNamePatterns := []string{"*NonExistenceFileNamePattern1", "*NonExistenceFileNamePattern2", "*NonExistenceFileNamePattern3"} 94 | cc := NewChecksumCalculator(defaultSHA256Hasher, gitAdditions) 95 | 96 | actualCC := cc.SuggestTalismanRC(fileNamePatterns) 97 | 98 | assert.Equal(t, expectedCC, actualCC) 99 | }) 100 | 101 | t.Run("should return suggestion for .talismanrc format when matching file name patterns is sent", func(t *testing.T) { 102 | gitAdditions := []gitrepo.Addition{ 103 | { 104 | Path: "GitRepoPath1", 105 | Name: "GitRepoName1", 106 | }, { 107 | Path: "GitRepoPath2", 108 | Name: "GitRepoName2", 109 | }, 110 | } 111 | expectedCC := "\n\x1b[33m.talismanrc format for given file names / patterns\x1b[0m\nfileignoreconfig:\n- filename: '*1'\n checksum: 54bbf09e5c906e2d7cc0808729f8120cfa3c4bad3fb6a85689ae23ca00e5a3c8\n- filename: Git*2\n checksum: d2103425cbcc10118556d7cd63dd198b3e771bcc0359d6b196ddafcf5b87dac5\nversion: \"1.0\"\n" 112 | fileNamePatterns := []string{"*1", "Git*2", "*NonExistenceFileNamePattern3"} 113 | cc := NewChecksumCalculator(defaultSHA256Hasher, gitAdditions) 114 | 115 | actualCC := cc.SuggestTalismanRC(fileNamePatterns) 116 | 117 | assert.Equal(t, expectedCC, actualCC) 118 | }) 119 | } 120 | -------------------------------------------------------------------------------- /cmd/checksum_cmd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "talisman/checksumcalculator" 7 | "talisman/gitrepo" 8 | "talisman/utility" 9 | 10 | "github.com/sirupsen/logrus" 11 | ) 12 | 13 | type ChecksumCmd struct { 14 | fileNamePatterns []string 15 | hasher utility.SHA256Hasher 16 | repoRoot string 17 | } 18 | 19 | func NewChecksumCmd(fileNamePatterns []string) *ChecksumCmd { 20 | wd, _ := os.Getwd() 21 | hasher := utility.MakeHasher("checksum", wd) 22 | return &ChecksumCmd{fileNamePatterns: fileNamePatterns, hasher: hasher, repoRoot: wd} 23 | } 24 | 25 | func (s *ChecksumCmd) Run() int { 26 | repo := gitrepo.RepoLocatedAt(s.repoRoot) 27 | if s.hasher == nil { 28 | logrus.Errorf("unable to start hasher") 29 | return EXIT_FAILURE 30 | } 31 | 32 | gitTrackedFilesAsAdditions := repo.TrackedFilesAsAdditions() 33 | gitTrackedFilesAsAdditions = append(gitTrackedFilesAsAdditions, repo.StagedAdditions()...) 34 | 35 | cc := checksumcalculator.NewChecksumCalculator(s.hasher, gitTrackedFilesAsAdditions) 36 | rcSuggestion := cc.SuggestTalismanRC(s.fileNamePatterns) 37 | 38 | if rcSuggestion != "" { 39 | fmt.Print(rcSuggestion) 40 | return EXIT_SUCCESS 41 | } 42 | return EXIT_FAILURE 43 | } 44 | -------------------------------------------------------------------------------- /cmd/checksum_cmd_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "talisman/git_testing" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestChecksumCalculatorShouldExitSuccess(t *testing.T) { 12 | git_testing.DoInTempGitRepo(func(git *git_testing.GitTesting) { 13 | git.SetupBaselineFiles("simple-file.txt") 14 | git.CreateFileWithContents("private.pem", "secret") 15 | git.CreateFileWithContents("another/private.pem", "secret") 16 | git.CreateFileWithContents("sample.txt", "password") 17 | os.Chdir(git.Root()) 18 | 19 | checksumCmd := NewChecksumCmd([]string{"*.txt"}) 20 | assert.Equal(t, 0, checksumCmd.Run(), "Expected run() to return 0 as given patterns are found and .talsimanrc is suggested") 21 | options.Checksum = "" 22 | }) 23 | } 24 | 25 | func TestChecksumCalculatorShouldExitFailure(t *testing.T) { 26 | git_testing.DoInTempGitRepo(func(git *git_testing.GitTesting) { 27 | git.SetupBaselineFiles("simple-file.txt") 28 | git.CreateFileWithContents("private.pem", "secret") 29 | git.CreateFileWithContents("another/private.pem", "secret") 30 | git.CreateFileWithContents("sample.txt", "password") 31 | os.Chdir(git.Root()) 32 | 33 | checksumCmd := NewChecksumCmd([]string{"*.java"}) 34 | assert.Equal(t, 1, checksumCmd.Run(), "Expected run() to return 1 as given patterns are found and .talsimanrc is suggested") 35 | options.Checksum = "" 36 | }) 37 | } 38 | 39 | func TestChecksumCalculatorShouldExitFailureWhenHasherIsEmpty(t *testing.T) { 40 | git_testing.DoInTempGitRepo(func(git *git_testing.GitTesting) { 41 | checksumCmd := ChecksumCmd{[]string{"*.java"}, nil, git.Root()} 42 | assert.Equal(t, 1, checksumCmd.Run(), "Expected run() to return 1 because hasher failed to start") 43 | }) 44 | } 45 | -------------------------------------------------------------------------------- /cmd/pattern_cmd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "talisman/gitrepo" 5 | "talisman/utility" 6 | 7 | log "github.com/sirupsen/logrus" 8 | 9 | "github.com/bmatcuk/doublestar" 10 | ) 11 | 12 | type PatternCmd struct { 13 | *runner 14 | } 15 | 16 | func NewPatternCmd(pattern string) *PatternCmd { 17 | var additions []gitrepo.Addition 18 | 19 | files, _ := doublestar.Glob(pattern) 20 | for _, file := range files { 21 | log.Debugf("reading file %s", file) 22 | data, err := utility.SafeReadFile(file) 23 | 24 | if err != nil { 25 | log.Warnf("Error reading file: %s. Skipping", file) 26 | continue 27 | } 28 | 29 | newAddition := gitrepo.NewAddition(file, data) 30 | additions = append(additions, newAddition) 31 | } 32 | 33 | return &PatternCmd{NewRunner(additions, "pattern")} 34 | } 35 | -------------------------------------------------------------------------------- /cmd/pre_commit_hook.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "talisman/gitrepo" 7 | ) 8 | 9 | type PreCommitHook struct { 10 | runner 11 | } 12 | 13 | func NewPreCommitHook() *PreCommitHook { 14 | wd, _ := os.Getwd() 15 | repo := gitrepo.RepoLocatedAt(wd) 16 | 17 | return &PreCommitHook{*NewRunner(repo.GetDiffForStagedFiles(), PreCommit)} 18 | } 19 | -------------------------------------------------------------------------------- /cmd/pre_push_hook.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "os" 7 | "strings" 8 | 9 | log "github.com/sirupsen/logrus" 10 | "talisman/gitrepo" 11 | ) 12 | 13 | const ( 14 | //EmptySha represents the state of a brand new ref 15 | EmptySha string = "0000000000000000000000000000000000000000" 16 | //ShaId of the empty tree in Git 17 | EmptyTreeSha string = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" 18 | ) 19 | 20 | type PrePushHook struct { 21 | localRef, localCommit, remoteRef, remoteCommit string 22 | *runner 23 | } 24 | 25 | func NewPrePushHook(stdin io.Reader) *PrePushHook { 26 | localRef, localCommit, remoteRef, remoteCommit := readRefAndSha(stdin) 27 | prePushHook := &PrePushHook{ 28 | localRef, 29 | localCommit, 30 | remoteRef, 31 | remoteCommit, 32 | NewRunner(nil, PrePush)} 33 | prePushHook.additions = prePushHook.getRepoAdditions() 34 | return prePushHook 35 | } 36 | 37 | // If the outgoing ref does not exist on the remote, all commits on the local ref will be checked 38 | // If the outgoing ref already exists, all additions in the range between "localSha" and "remoteSha" will be validated 39 | func (p *PrePushHook) getRepoAdditions() []gitrepo.Addition { 40 | if p.runningOnDeletedRef() { 41 | log.WithFields(log.Fields{ 42 | "localRef": p.localRef, 43 | "localCommit": p.localCommit, 44 | "remoteRef": p.remoteRef, 45 | "remoteCommit": p.remoteCommit, 46 | }).Info("Running on a deleted ref. Nothing to verify as outgoing changes are all deletions.") 47 | 48 | return []gitrepo.Addition{} 49 | } 50 | 51 | if p.runningOnNewRef() { 52 | log.WithFields(log.Fields{ 53 | "localRef": p.localRef, 54 | "localCommit": p.localCommit, 55 | "remoteRef": p.remoteRef, 56 | "remoteCommit": p.remoteCommit, 57 | }).Info("Running on a new ref. All changes in the ref will be verified.") 58 | 59 | return p.getRepoAdditionsFrom(EmptyTreeSha, p.localCommit) 60 | } 61 | 62 | log.WithFields(log.Fields{ 63 | "localRef": p.localRef, 64 | "localCommit": p.localCommit, 65 | "remoteRef": p.remoteRef, 66 | "remoteCommit": p.remoteCommit, 67 | }).Info("Running on an existing ref. All changes in the commit range will be verified.") 68 | 69 | return p.getRepoAdditionsFrom(p.remoteCommit, p.localCommit) 70 | } 71 | 72 | func (p *PrePushHook) runningOnDeletedRef() bool { 73 | return p.localCommit == EmptySha 74 | } 75 | 76 | func (p *PrePushHook) runningOnNewRef() bool { 77 | return p.remoteCommit == EmptySha 78 | } 79 | 80 | func (p *PrePushHook) getRepoAdditionsFrom(oldCommit, newCommit string) []gitrepo.Addition { 81 | wd, _ := os.Getwd() 82 | repo := gitrepo.RepoLocatedAt(wd) 83 | return repo.AdditionsWithinRange(oldCommit, newCommit) 84 | } 85 | 86 | func readRefAndSha(file io.Reader) (string, string, string, string) { 87 | text, _ := bufio.NewReader(file).ReadString('\n') 88 | refsAndShas := strings.Split(strings.Trim(string(text), "\n"), " ") 89 | if len(refsAndShas) < 4 { 90 | return EmptySha, EmptySha, "", "" 91 | } 92 | return refsAndShas[0], refsAndShas[1], refsAndShas[2], refsAndShas[3] 93 | } 94 | -------------------------------------------------------------------------------- /cmd/runner.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "talisman/detector" 7 | "talisman/detector/helpers" 8 | "talisman/detector/severity" 9 | "talisman/gitrepo" 10 | "talisman/prompt" 11 | "talisman/talismanrc" 12 | ) 13 | 14 | // runner represents a single run of the validations for a given commit range 15 | type runner struct { 16 | additions []gitrepo.Addition 17 | results *helpers.DetectionResults 18 | mode string 19 | } 20 | 21 | // NewRunner returns a new runner. 22 | func NewRunner(additions []gitrepo.Addition, mode string) *runner { 23 | return &runner{ 24 | additions: additions, 25 | results: helpers.NewDetectionResults(), 26 | mode: mode, 27 | } 28 | } 29 | 30 | // Run will validate the commit range for errors and return either COMPLETED_SUCCESSFULLY or COMPLETED_WITH_ERRORS 31 | func (r *runner) Run(tRC *talismanrc.TalismanRC, promptContext prompt.PromptContext) int { 32 | wd, _ := os.Getwd() 33 | repo := gitrepo.RepoLocatedAt(wd) 34 | ie := helpers.BuildIgnoreEvaluator(r.mode, tRC, repo) 35 | 36 | setCustomSeverities(tRC) 37 | additionsToScan := tRC.RemoveScopedFiles(r.additions) 38 | 39 | detector.DefaultChain(tRC, ie).Test(additionsToScan, tRC, r.results) 40 | r.printReport(promptContext) 41 | exitStatus := r.exitStatus() 42 | return exitStatus 43 | } 44 | 45 | func setCustomSeverities(tRC *talismanrc.TalismanRC) { 46 | for _, cs := range tRC.CustomSeverities { 47 | severity.SeverityConfiguration[cs.Detector] = cs.Severity 48 | } 49 | } 50 | 51 | func (r *runner) printReport(promptContext prompt.PromptContext) { 52 | if r.results.HasWarnings() { 53 | fmt.Println(r.results.ReportWarnings()) 54 | } 55 | if r.results.HasIgnores() || r.results.HasFailures() { 56 | r.results.Report(promptContext, r.mode) 57 | } 58 | } 59 | 60 | func (r *runner) exitStatus() int { 61 | if r.results.HasFailures() { 62 | return EXIT_FAILURE 63 | } 64 | return EXIT_SUCCESS 65 | } 66 | -------------------------------------------------------------------------------- /cmd/scanner_cmd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "talisman/detector" 7 | "talisman/detector/helpers" 8 | "talisman/gitrepo" 9 | "talisman/report" 10 | "talisman/scanner" 11 | "talisman/talismanrc" 12 | "talisman/utility" 13 | 14 | logr "github.com/sirupsen/logrus" 15 | ) 16 | 17 | const ( 18 | SCAN_MODE = "scan" 19 | ) 20 | 21 | type ScannerCmd struct { 22 | additions []gitrepo.Addition 23 | results *helpers.DetectionResults 24 | reportDirectory string 25 | ignoreEvaluator helpers.IgnoreEvaluator 26 | tRC *talismanrc.TalismanRC 27 | } 28 | 29 | // Run scans git commit history for potential secrets and returns 0 or 1 as exit code 30 | func (s *ScannerCmd) Run() int { 31 | fmt.Printf("\n\n") 32 | utility.CreateArt("Running Scan..") 33 | 34 | additionsToScan := s.tRC.RemoveScopedFiles(s.additions) 35 | 36 | detector.DefaultChain(s.tRC, s.ignoreEvaluator).Test(additionsToScan, s.tRC, s.results) 37 | reportsPath, err := report.GenerateReport(s.results, s.reportDirectory) 38 | if err != nil { 39 | logr.Errorf("error while generating report: %v", err) 40 | return EXIT_FAILURE 41 | } 42 | 43 | fmt.Printf("\nPlease check '%s' folder for the talisman scan report\n\n", reportsPath) 44 | return s.exitStatus() 45 | } 46 | 47 | func (s *ScannerCmd) exitStatus() int { 48 | if s.results.HasFailures() { 49 | return EXIT_FAILURE 50 | } 51 | return EXIT_SUCCESS 52 | } 53 | 54 | // NewScannerCmd Returns a new scanner command 55 | func NewScannerCmd(ignoreHistory bool, tRC *talismanrc.TalismanRC, reportDirectory string) *ScannerCmd { 56 | repoRoot, _ := os.Getwd() 57 | reader := gitrepo.NewBatchGitObjectHashReader(repoRoot) 58 | additions := scanner.GetAdditions(ignoreHistory, reader) 59 | ignoreEvaluator := helpers.ScanHistoryEvaluator() 60 | if ignoreHistory { 61 | ignoreEvaluator = helpers.BuildIgnoreEvaluator("default", tRC, gitrepo.RepoLocatedAt(repoRoot)) 62 | } 63 | return &ScannerCmd{ 64 | additions: additions, 65 | results: helpers.NewDetectionResults(), 66 | reportDirectory: reportDirectory, 67 | ignoreEvaluator: ignoreEvaluator, 68 | tRC: tRC, 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /cmd/scanner_cmd_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "talisman/git_testing" 6 | "talisman/talismanrc" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestScannerCmdRunsSuccessfully(t *testing.T) { 13 | git_testing.DoInTempGitRepo(func(git *git_testing.GitTesting) { 14 | git.SetupBaselineFiles("simple-file") 15 | git.CreateFileWithContents("some-dir/should-be-included.txt", "safeContents") 16 | git.AddAndcommit("*", "Start of Scan") 17 | os.Chdir(git.Root()) 18 | 19 | scannerCmd := NewScannerCmd(true, &talismanrc.TalismanRC{}, git.Root()) 20 | scannerCmd.Run() 21 | assert.Equal(t, 0, scannerCmd.exitStatus(), "Expected ScannerCmd.exitStatus() to return 0 since no secret is found") 22 | }) 23 | } 24 | 25 | func TestScannerCmdDetectsSecretAndFails(t *testing.T) { 26 | git_testing.DoInTempGitRepo(func(git *git_testing.GitTesting) { 27 | git.SetupBaselineFiles("simple-file") 28 | git.CreateFileWithContents("some-dir/file-with-secret.txt", awsAccessKeyIDExample) 29 | git.AddAndcommit("*", "Initial Commit") 30 | git.RemoveFile("some-dir/file-with-secret.txt") 31 | git.AddAndcommit("*", "Removed secret") 32 | git.CreateFileWithContents("some-dir/safe-file.txt", "safeContents") 33 | git.AddAndcommit("*", "Start of Scan") 34 | os.Chdir(git.Root()) 35 | 36 | scannerCmd := NewScannerCmd(false, &talismanrc.TalismanRC{}, git.Root()) 37 | scannerCmd.Run() 38 | assert.Equal(t, 1, scannerCmd.exitStatus(), "Expected ScannerCmd.exitStatus() to return 1 since secret present in history") 39 | }) 40 | } 41 | 42 | func TestScannerCmdAddingSecretKeyShouldExitZeroIfFileIsWithinConfiguredScope(t *testing.T) { 43 | git_testing.DoInTempGitRepo(func(git *git_testing.GitTesting) { 44 | git.SetupBaselineFiles("simple-file") 45 | git.CreateFileWithContents("go.sum", awsAccessKeyIDExample) 46 | git.CreateFileWithContents("go.mod", awsAccessKeyIDExample) 47 | git.AddAndcommit("*", "go sum file") 48 | os.Chdir(git.Root()) 49 | 50 | tRC := &talismanrc.TalismanRC{ScopeConfig: []talismanrc.ScopeConfig{{ScopeName: "go"}}} 51 | scannerCmd := NewScannerCmd(false, tRC, git.Root()) 52 | scannerCmd.Run() 53 | assert.Equal(t, 0, scannerCmd.exitStatus(), "Expected ScannerCmd.exitStatus() to return 0 since no secret is found") 54 | }) 55 | } 56 | 57 | func TestScannerCmdDetectsSecretAndIgnoresWhileRunningInIgnoreHistoryModeWithValidIgnoreConf(t *testing.T) { 58 | git_testing.DoInTempGitRepo(func(git *git_testing.GitTesting) { 59 | git.SetupBaselineFiles("simple-file") 60 | git.CreateFileWithContents("go.sum", awsAccessKeyIDExample) 61 | git.CreateFileWithContents("go.mod", awsAccessKeyIDExample) 62 | git.AddAndcommit("*", "go sum file") 63 | os.Chdir(git.Root()) 64 | 65 | tRC := &talismanrc.TalismanRC{ 66 | FileIgnoreConfig: []talismanrc.FileIgnoreConfig{ 67 | {FileName: "go.sum", Checksum: "582093519ae682d5170aecc9b935af7e90ed528c577ecd2c9dd1fad8f4924ab9"}, 68 | {FileName: "go.mod", Checksum: "8a03b9b61c505ace06d590d2b9b4f4b6fa70136e14c26875ced149180e00d1af"}, 69 | }} 70 | scannerCmd := NewScannerCmd(true, tRC, git.Root()) 71 | scannerCmd.Run() 72 | assert.Equal(t, 0, scannerCmd.exitStatus(), "Expected ScannerCmd.exitStatus() to return 0 since secrets file ignore is enabled") 73 | }) 74 | } 75 | 76 | func TestScannerCmdDetectsSecretWhileRunningNormalScanMode(t *testing.T) { 77 | git_testing.DoInTempGitRepo(func(git *git_testing.GitTesting) { 78 | git.SetupBaselineFiles("simple-file") 79 | git.CreateFileWithContents("go.sum", awsAccessKeyIDExample) 80 | git.CreateFileWithContents("go.mod", awsAccessKeyIDExample) 81 | git.AddAndcommit("*", "go sum file") 82 | os.Chdir(git.Root()) 83 | 84 | tRC := &talismanrc.TalismanRC{ 85 | FileIgnoreConfig: []talismanrc.FileIgnoreConfig{ 86 | {FileName: "go.sum", Checksum: "582093519ae682d5170aecc9b935af7e90ed528c577ecd2c9dd1fad8f4924ab9"}, 87 | {FileName: "go.mod", Checksum: "8a03b9b61c505ace06d590d2b9b4f4b6fa70136e14c26875ced149180e00d1af"}, 88 | }} 89 | scannerCmd := NewScannerCmd(false, tRC, git.Root()) 90 | scannerCmd.Run() 91 | assert.Equal(t, 1, scannerCmd.exitStatus(), "Expected ScannerCmd.exitStatus() to return 1 because file ignore is disabled when scanning history") 92 | }) 93 | } 94 | -------------------------------------------------------------------------------- /cmd/talisman.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "os" 8 | "runtime/pprof" 9 | "strings" 10 | "talisman/utility" 11 | "time" 12 | 13 | "talisman/prompt" 14 | "talisman/talismanrc" 15 | 16 | "runtime" 17 | 18 | log "github.com/sirupsen/logrus" 19 | "github.com/spf13/afero" 20 | flag "github.com/spf13/pflag" 21 | ) 22 | 23 | var ( 24 | showVersion bool 25 | //Version : Version of talisman 26 | Version = "Development Build" 27 | interactive bool 28 | talismanInput io.Reader 29 | ) 30 | 31 | const ( 32 | //PrePush : Const for name of pre-push hook 33 | PrePush = "pre-push" 34 | //PreCommit : Const for name of of pre-commit hook 35 | PreCommit = "pre-commit" 36 | //EXIT_SUCCESS : Const to indicate successful talisman invocation 37 | EXIT_SUCCESS = 0 38 | //EXIT_FAILURE : Const to indicate failed successful invocation 39 | EXIT_FAILURE = 1 40 | ) 41 | 42 | var options struct { 43 | Debug bool 44 | LogLevel string 45 | GitHook string 46 | Pattern string 47 | Scan bool 48 | IgnoreHistory bool 49 | Checksum string 50 | ReportDirectory string 51 | ScanWithHtml bool 52 | ShouldProfile bool 53 | } 54 | 55 | //var options Options 56 | 57 | func init() { 58 | log.SetOutput(os.Stderr) 59 | talismanInput = os.Stdin 60 | flag.BoolVarP(&options.Debug, 61 | "debug", "d", false, 62 | "enable debug mode (warning: very verbose)") 63 | flag.StringVarP(&options.LogLevel, 64 | "loglevel", "l", "error", 65 | "set log level for talisman (allowed values: error|info|warn|debug, default: error)") 66 | flag.BoolVarP(&showVersion, 67 | "version", "v", false, 68 | "show current version of talisman") 69 | flag.StringVarP(&options.Pattern, 70 | "pattern", "p", "", 71 | "pattern (glob-like) of files to scan (ignores githooks)") 72 | flag.StringVarP(&options.GitHook, 73 | "githook", "g", PrePush, 74 | "either pre-push or pre-commit") 75 | flag.BoolVarP(&options.Scan, 76 | "scan", "s", false, 77 | "scanner scans the git commit history for potential secrets") 78 | flag.BoolVarP(&options.IgnoreHistory, 79 | "ignoreHistory", "^", false, 80 | "scanner scans all files on current head, will not scan through git commit history") 81 | flag.StringVarP(&options.Checksum, 82 | "checksum", "c", "", 83 | "checksum calculator calculates checksum and suggests .talismanrc entry") 84 | flag.StringVarP(&options.ReportDirectory, 85 | "reportDirectory", "r", "talisman_report", 86 | "directory where the scan report will be stored") 87 | flag.BoolVarP(&options.ScanWithHtml, 88 | "scanWithHtml", "w", false, 89 | "generate html report (**Make sure you have installed talisman_html_report to use this, as mentioned in talisman Readme**)") 90 | flag.BoolVarP(&interactive, 91 | "interactive", "i", false, 92 | "interactively update talismanrc (only makes sense with -g/--githook)") 93 | flag.BoolVarP(&options.ShouldProfile, 94 | "profile", "f", false, 95 | "profile cpu and memory usage of talisman") 96 | } 97 | 98 | func main() { 99 | flag.Parse() 100 | 101 | if flag.NFlag() == 0 { 102 | flag.PrintDefaults() 103 | os.Exit(EXIT_SUCCESS) 104 | } 105 | 106 | if showVersion { 107 | fmt.Printf("talisman %s\n", Version) 108 | os.Exit(EXIT_SUCCESS) 109 | } 110 | 111 | if options.GitHook != "" { 112 | if !(options.GitHook == PreCommit || options.GitHook == PrePush) { 113 | fmt.Println(fmt.Errorf("githook should be %s or %s, but got %s", PreCommit, PrePush, options.GitHook)) 114 | os.Exit(EXIT_FAILURE) 115 | } 116 | } 117 | 118 | if options.ShouldProfile { 119 | stopProfFunc := setupProfiling() 120 | defer stopProfFunc() 121 | } 122 | 123 | promptContext := prompt.NewPromptContext(interactive, prompt.NewPrompt()) 124 | os.Exit(run(promptContext)) 125 | } 126 | 127 | func run(promptContext prompt.PromptContext) (returnCode int) { 128 | start := time.Now() 129 | defer func() { fmt.Printf("Talisman done in %v\n", time.Since(start)) }() 130 | 131 | if err := validateGitExecutable(afero.NewOsFs(), runtime.GOOS); err != nil { 132 | log.Errorf("error validating git executable: %v", err) 133 | return 1 134 | } 135 | 136 | setLogLevel() 137 | 138 | if options.GitHook == "" { 139 | options.GitHook = PrePush 140 | } 141 | 142 | optionsBytes, _ := json.Marshal(options) 143 | fields := make(map[string]interface{}) 144 | _ = json.Unmarshal(optionsBytes, &fields) 145 | log.WithFields(fields).Debug("Talisman execution environment") 146 | defer utility.DestroyHashers() 147 | if options.Checksum != "" { 148 | log.Infof("Running %s patterns against checksum calculator", options.Checksum) 149 | return NewChecksumCmd(strings.Fields(options.Checksum)).Run() 150 | } else if options.Scan { 151 | log.Infof("Running scanner") 152 | talismanrc, err := talismanrc.Load() 153 | if err != nil { 154 | return EXIT_FAILURE 155 | } 156 | return NewScannerCmd(options.IgnoreHistory, talismanrc, options.ReportDirectory).Run() 157 | } else if options.ScanWithHtml { 158 | log.Infof("Running scanner with html report") 159 | talismanrc, err := talismanrc.Load() 160 | if err != nil { 161 | return EXIT_FAILURE 162 | } 163 | return NewScannerCmd(options.IgnoreHistory, talismanrc, "talisman_html_report").Run() 164 | } else if options.Pattern != "" { 165 | log.Infof("Running scan for %s", options.Pattern) 166 | talismanrc, err := talismanrc.Load() 167 | if err != nil { 168 | return EXIT_FAILURE 169 | } 170 | return NewPatternCmd(options.Pattern).Run(talismanrc, promptContext) 171 | } else if options.GitHook == PreCommit { 172 | log.Infof("Running %s hook", options.GitHook) 173 | talismanrc, err := talismanrc.Load() 174 | if err != nil { 175 | return EXIT_FAILURE 176 | } 177 | return NewPreCommitHook().Run(talismanrc, promptContext) 178 | } else { 179 | log.Infof("Running %s hook", options.GitHook) 180 | talismanrc, err := talismanrc.Load() 181 | if err != nil { 182 | return EXIT_FAILURE 183 | } 184 | return NewPrePushHook(talismanInput).Run(talismanrc, promptContext) 185 | } 186 | } 187 | 188 | func validateGitExecutable(fs afero.Fs, operatingSystem string) error { 189 | if operatingSystem == "windows" { 190 | extensions := strings.ToLower(os.Getenv("PATHEXT")) 191 | windowsExecutables := strings.Split(extensions, ";") 192 | for _, executable := range windowsExecutables { 193 | gitExecutable := fmt.Sprintf("git%s", executable) 194 | exists, err := afero.Exists(fs, gitExecutable) 195 | if err != nil { 196 | return err 197 | } 198 | if exists { 199 | return fmt.Errorf("not allowed to have git executable located in repository: %s", gitExecutable) 200 | } 201 | } 202 | } 203 | return nil 204 | } 205 | 206 | func setLogLevel() { 207 | switch options.LogLevel { 208 | case "info": 209 | log.SetLevel(log.InfoLevel) 210 | case "debug": 211 | log.SetLevel(log.DebugLevel) 212 | case "error": 213 | log.SetLevel(log.ErrorLevel) 214 | case "warn": 215 | log.SetLevel(log.WarnLevel) 216 | default: 217 | log.SetLevel(log.ErrorLevel) 218 | } 219 | if options.Debug { 220 | log.SetLevel(log.DebugLevel) 221 | } 222 | } 223 | 224 | func setupProfiling() func() { 225 | log.Info("Profiling initiated") 226 | 227 | cpuProfFile, err := os.Create("talisman.cpuprof") 228 | if err != nil { 229 | log.Fatalf("Unable to create cpu profiling output file talisman.cpuprof: %v", err) 230 | } 231 | 232 | memProfFile, err := os.Create("talisman.memprof") 233 | if err != nil { 234 | log.Fatalf("Unable to create memory profiling output file talisman.memprof: %v", err) 235 | } 236 | 237 | _ = pprof.StartCPUProfile(cpuProfFile) 238 | progEnded := false 239 | 240 | go func() { 241 | memProfTimer := time.NewTimer(500 * time.Millisecond) 242 | 243 | for !progEnded { 244 | <-memProfTimer.C 245 | _ = pprof.WriteHeapProfile(memProfFile) 246 | } 247 | _ = memProfFile.Close() 248 | }() 249 | 250 | return func() { 251 | progEnded = true 252 | pprof.StopCPUProfile() 253 | log.Info("Profiling completed") 254 | _ = cpuProfFile.Close() 255 | } 256 | } 257 | -------------------------------------------------------------------------------- /cmd/talisman_internal_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/spf13/afero" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestParsingShasFromStdIn(t *testing.T) { 13 | file, err := ioutil.TempFile(os.TempDir(), "mockStdin") 14 | if err != nil { 15 | panic(err) 16 | } 17 | defer os.Remove(file.Name()) 18 | defer file.Close() 19 | file.WriteString("localRef localSha remoteRef remoteSha") 20 | file.Seek(0, 0) 21 | 22 | _, oldSha, _, newSha := readRefAndSha(file) 23 | assert.Equal(t, "localSha", oldSha, "oldSha did not equal 'localSha', got: %s", oldSha) 24 | assert.Equal(t, "remoteSha", newSha, "newSha did not equal 'remoteSha', got: %s", newSha) 25 | } 26 | 27 | func Test_validateGitExecutable(t *testing.T) { 28 | t.Run("given operating systems is windows", func(t *testing.T) { 29 | 30 | operatingSystem := "windows" 31 | os.Setenv("PATHEXT", ".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC") 32 | 33 | t.Run("should return error if git executable exists in current directory", func(t *testing.T) { 34 | fs := afero.NewMemMapFs() 35 | gitExecutable := "git.exe" 36 | afero.WriteFile(fs, gitExecutable, []byte("git executable"), 0700) 37 | err := validateGitExecutable(fs, operatingSystem) 38 | assert.EqualError(t, err, "not allowed to have git executable located in repository: git.exe") 39 | }) 40 | 41 | t.Run("should return nil if git executable does not exist in current directory", func(t *testing.T) { 42 | err := validateGitExecutable(afero.NewMemMapFs(), operatingSystem) 43 | assert.NoError(t, err) 44 | }) 45 | 46 | }) 47 | 48 | t.Run("given operating systems is linux", func(t *testing.T) { 49 | 50 | operatingSystem := "linux" 51 | 52 | t.Run("should return nil if git executable exists in current directory", func(t *testing.T) { 53 | fs := afero.NewMemMapFs() 54 | gitExecutable := "git.exe" 55 | afero.WriteFile(fs, gitExecutable, []byte("git executable"), 0700) 56 | err := validateGitExecutable(fs, operatingSystem) 57 | assert.NoError(t, err) 58 | }) 59 | 60 | t.Run("should return nil if git executable does not exist in current directory", func(t *testing.T) { 61 | err := validateGitExecutable(afero.NewMemMapFs(), operatingSystem) 62 | assert.NoError(t, err) 63 | }) 64 | 65 | }) 66 | 67 | } 68 | -------------------------------------------------------------------------------- /cmd/talisman_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/sirupsen/logrus" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func Test_setLogLevel(t *testing.T) { 12 | levels := []string{"error", "warn", "info", "debug", "unknown"} 13 | expectedLogrusLevels := []logrus.Level{ 14 | logrus.ErrorLevel, logrus.WarnLevel, 15 | logrus.InfoLevel, logrus.DebugLevel, logrus.ErrorLevel} 16 | 17 | for idx, level := range levels { 18 | options.LogLevel = level 19 | setLogLevel() 20 | assert.True( 21 | t, 22 | logrus.IsLevelEnabled(expectedLogrusLevels[idx]), 23 | fmt.Sprintf("expected level to be %v for options.LogLevel = %s", expectedLogrusLevels[idx], level)) 24 | 25 | options.Debug = true 26 | setLogLevel() 27 | assert.True( 28 | t, 29 | logrus.IsLevelEnabled(logrus.DebugLevel), 30 | "expected level to be debug when options.Debug is set") 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /contributing.md: -------------------------------------------------------------------------------- 1 | # Contributing to Talisman 2 | 3 | By contributing to Talisman, you agree to abide by the [code of conduct](CODE_OF_CONDUCT.md). 4 | 5 | ## How to start contributing 6 | 7 | If you are not sure how to begin contributing to Talisman, have a look at the issues tagged under [good first issue](https://github.com/thoughtworks/talisman/labels/good%20first%20issue). 8 | 9 | ## Developing locally 10 | 11 | To contribute to Talisman, you need a working golang development environment. 12 | Check [this link](https://golang.org/doc/install) to help you get started. 13 | 14 | Once you have go installed and set up, clone the talisman repository. In your 15 | working copy, fetch the dependencies by having go mod fetch them for you: 16 | 17 | ``` 18 | go mod vendor 19 | ``` 20 | 21 | Run the tests: 22 | 23 | ``` 24 | go test ./... 25 | ``` 26 | 27 | Build talisman: 28 | 29 | ``` 30 | go build -o dist/talisman -ldflags="-s -w" talisman/cmd 31 | ``` 32 | 33 | To build for multiple platforms we use [GoReleaser](https://goreleaser.com/): 34 | 35 | ``` 36 | goreleaser build --snapshot --clean 37 | ``` 38 | 39 | ## Submitting a Pull Request 40 | 41 | To send in a pull request 42 | 43 | 1. Fork the repo. 44 | 2. Create a new feature branch based off the master branch. 45 | 3. Provide the commit message with the issue number and a proper description. 46 | 4. Ensure that all the tests pass. 47 | 5. Submit the pull request. 48 | 49 | ## Updating Talisman GitHub Pages 50 | 51 | 1. Checkout a new branch from gh-pages 52 | 2. Navigate to the docs/ folder and update the files 53 | 3. See instructions for checking locally [here](https://github.com/thoughtworks/talisman/blob/gh-pages/README.md). 54 | 4. Raise a pull request against the branch gh-pages 55 | 56 | ## Releasing 57 | 58 | 1. Tag the commit to be released with the next version according to 59 | [semver](https://semver.org/) conventions 60 | 2. Push the tag to trigger the GitHub Actions Release pipeline 61 | 3. Approve the [drafted GitHub Release](https://github.com/thoughtworks/talisman/releases) 62 | 63 | The latest version will now be accessible to anyone who builds their own 64 | binaries, downloads binaries directly from GitHub Releases or homebrew, or uses 65 | the install script from the website. 66 | -------------------------------------------------------------------------------- /detector/chain.go: -------------------------------------------------------------------------------- 1 | package detector 2 | 3 | import ( 4 | "os" 5 | "talisman/detector/detector" 6 | "talisman/detector/filecontent" 7 | "talisman/detector/filename" 8 | "talisman/detector/helpers" 9 | "talisman/detector/pattern" 10 | "talisman/gitrepo" 11 | "talisman/talismanrc" 12 | "talisman/utility" 13 | 14 | log "github.com/sirupsen/logrus" 15 | ) 16 | 17 | // Chain represents a chain of Detectors. 18 | // It is itself a detector. 19 | type Chain struct { 20 | detectors []detector.Detector 21 | ignoreEvaluator helpers.IgnoreEvaluator 22 | } 23 | 24 | // NewChain returns an empty DetectorChain 25 | // It is itself a detector, but it tests nothing. 26 | func NewChain(ignoreEvaluator helpers.IgnoreEvaluator) *Chain { 27 | result := Chain{[]detector.Detector{}, ignoreEvaluator} 28 | return &result 29 | } 30 | 31 | // DefaultChain returns a DetectorChain with pre-configured detectors 32 | func DefaultChain(tRC *talismanrc.TalismanRC, ignoreEvaluator helpers.IgnoreEvaluator) *Chain { 33 | chain := NewChain(ignoreEvaluator) 34 | chain.AddDetector(filename.DefaultFileNameDetector(tRC.Threshold)) 35 | chain.AddDetector(filecontent.NewFileContentDetector(tRC)) 36 | chain.AddDetector(pattern.NewPatternDetector(tRC.CustomPatterns)) 37 | return chain 38 | } 39 | 40 | // AddDetector adds the detector that is passed in to the chain 41 | func (dc *Chain) AddDetector(d detector.Detector) *Chain { 42 | dc.detectors = append(dc.detectors, d) 43 | return dc 44 | } 45 | 46 | // Test validates the additions against each detector in the chain. 47 | // The results are passed in from detector to detector and thus collect all errors from all detectors 48 | func (dc *Chain) Test(additions []gitrepo.Addition, talismanRC *talismanrc.TalismanRC, result *helpers.DetectionResults) { 49 | log.Printf("Number of files to scan: %d\n", len(additions)) 50 | log.Printf("Number of detectors: %d\n", len(dc.detectors)) 51 | total := len(additions) * len(dc.detectors) 52 | progressBar := utility.GetProgressBar(os.Stdout, "Talisman Scan") 53 | progressBar.Start(total) 54 | for _, v := range dc.detectors { 55 | v.Test(dc.ignoreEvaluator, additions, talismanRC, result, func() { 56 | progressBar.Increment() 57 | }) 58 | } 59 | progressBar.Finish() 60 | } 61 | -------------------------------------------------------------------------------- /detector/chain_test.go: -------------------------------------------------------------------------------- 1 | package detector 2 | 3 | import ( 4 | "io/ioutil" 5 | "talisman/detector/filecontent" 6 | "talisman/detector/filename" 7 | "talisman/detector/helpers" 8 | "talisman/detector/pattern" 9 | "talisman/detector/severity" 10 | "talisman/gitrepo" 11 | "talisman/talismanrc" 12 | "testing" 13 | 14 | logr "github.com/sirupsen/logrus" 15 | "github.com/stretchr/testify/assert" 16 | ) 17 | 18 | func init() { 19 | logr.SetOutput(ioutil.Discard) 20 | } 21 | 22 | type FailingDetection struct{} 23 | 24 | func (v FailingDetection) Test(comparator helpers.IgnoreEvaluator, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) { 25 | result.Fail("some_file", "filecontent", "FAILED BY DESIGN", []string{}, severity.Low) 26 | } 27 | 28 | type PassingDetection struct{} 29 | 30 | func (p PassingDetection) Test(comparator helpers.IgnoreEvaluator, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) { 31 | } 32 | 33 | func TestEmptyValidationChainPassesAllValidations(t *testing.T) { 34 | ie := helpers.BuildIgnoreEvaluator("pre-push", nil, gitrepo.RepoLocatedAt(".")) 35 | v := NewChain(ie) 36 | results := helpers.NewDetectionResults() 37 | v.Test(nil, &talismanrc.TalismanRC{}, results) 38 | assert.False(t, results.HasFailures(), "Empty validation chain is expected to always pass") 39 | } 40 | 41 | func TestValidationChainWithFailingValidationAlwaysFails(t *testing.T) { 42 | ie := helpers.BuildIgnoreEvaluator("pre-push", nil, gitrepo.RepoLocatedAt(".")) 43 | v := NewChain(ie) 44 | v.AddDetector(PassingDetection{}) 45 | v.AddDetector(FailingDetection{}) 46 | results := helpers.NewDetectionResults() 47 | v.Test(nil, &talismanrc.TalismanRC{}, results) 48 | 49 | assert.False(t, results.Successful(), "Expected validation chain with a failure to fail.") 50 | } 51 | 52 | func TestDefaultChainShouldCreateChainSpecifiedModeAndPresetDetectors(t *testing.T) { 53 | talismanRC := &talismanrc.TalismanRC{ 54 | Threshold: severity.Medium, 55 | CustomPatterns: []talismanrc.PatternString{"AKIA*"}, 56 | } 57 | ie := helpers.BuildIgnoreEvaluator("pre-push", talismanRC, gitrepo.RepoLocatedAt(".")) 58 | v := DefaultChain(talismanRC, ie) 59 | assert.Equal(t, 3, len(v.detectors)) 60 | 61 | defaultFileNameDetector := filename.DefaultFileNameDetector(talismanRC.Threshold) 62 | assert.Equal(t, defaultFileNameDetector, v.detectors[0]) 63 | 64 | expectedFileContentDetector := filecontent.NewFileContentDetector(talismanRC) 65 | assert.Equal(t, expectedFileContentDetector, v.detectors[1]) 66 | 67 | expectedPatternDetector := pattern.NewPatternDetector(talismanRC.CustomPatterns) 68 | assert.Equal(t, expectedPatternDetector, v.detectors[2]) 69 | } 70 | -------------------------------------------------------------------------------- /detector/detector/detector.go: -------------------------------------------------------------------------------- 1 | package detector 2 | 3 | import ( 4 | "talisman/detector/helpers" 5 | "talisman/gitrepo" 6 | "talisman/talismanrc" 7 | ) 8 | 9 | // Detector represents a single kind of test to be performed against a set of Additions 10 | // Detectors are expected to honor the ignores that are passed in and log them in the results 11 | // Detectors are expected to signal any errors to the results 12 | type Detector interface { 13 | Test(comparator helpers.IgnoreEvaluator, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) 14 | } 15 | -------------------------------------------------------------------------------- /detector/filecontent/base64_aggressive_detector.go: -------------------------------------------------------------------------------- 1 | package filecontent 2 | 3 | import ( 4 | "encoding/base64" 5 | "strings" 6 | ) 7 | 8 | var delimiters = []string{".", "-", "="} 9 | 10 | const aggressivenessThreshold = 15 //decreasing makes it more aggressive 11 | 12 | type Base64AggressiveDetector struct { 13 | } 14 | 15 | func (ac *Base64AggressiveDetector) Test(s string) string { 16 | for _, d := range delimiters { 17 | subStrings := strings.Split(s, d) 18 | res := checkEachSubString(subStrings) 19 | if res != "" { 20 | return res 21 | } 22 | } 23 | return "" 24 | } 25 | 26 | func decodeBase64(s string) string { 27 | if len(s) <= aggressivenessThreshold { 28 | return "" 29 | } 30 | _, err := base64.StdEncoding.DecodeString(s) 31 | if err != nil { 32 | return "" 33 | } 34 | return s 35 | } 36 | 37 | func checkEachSubString(subStrings []string) string { 38 | for _, sub := range subStrings { 39 | suspicious := decodeBase64(sub) 40 | if suspicious != "" { 41 | return suspicious 42 | } 43 | } 44 | return "" 45 | } 46 | -------------------------------------------------------------------------------- /detector/filecontent/base64_aggressive_detector_test.go: -------------------------------------------------------------------------------- 1 | package filecontent 2 | 3 | import ( 4 | "talisman/detector/helpers" 5 | "testing" 6 | 7 | "talisman/gitrepo" 8 | "talisman/talismanrc" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | var _blankTalismanRC = &talismanrc.TalismanRC{} 14 | var dummyCompletionCallbackFunc = func() {} 15 | var aggressiveModeFileContentDetector = NewFileContentDetector(_blankTalismanRC).AggressiveMode() 16 | 17 | func TestShouldFlagPotentialAWSAccessKeysInAggressiveMode(t *testing.T) { 18 | const awsAccessKeyIDExample string = "AKIAIOSFODNN7EXAMPLE\n" 19 | results := helpers.NewDetectionResults() 20 | filename := "filename" 21 | additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(awsAccessKeyIDExample))} 22 | 23 | aggressiveModeFileContentDetector. 24 | Test( 25 | defaultIgnoreEvaluator, 26 | additions, 27 | _blankTalismanRC, 28 | results, 29 | dummyCompletionCallbackFunc) 30 | 31 | assert.True(t, results.HasFailures(), "Expected file to not contain base64 encoded texts.") 32 | } 33 | 34 | func TestShouldFlagPotentialAWSAccessKeysAtPropertyDefinitionInAggressiveMode(t *testing.T) { 35 | const awsAccessKeyIDExample string = "accessKey=AKIAIOSFODNN7EXAMPLE" 36 | results := helpers.NewDetectionResults() 37 | filename := "filename" 38 | additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(awsAccessKeyIDExample))} 39 | 40 | aggressiveModeFileContentDetector. 41 | Test( 42 | defaultIgnoreEvaluator, 43 | additions, 44 | _blankTalismanRC, 45 | results, 46 | dummyCompletionCallbackFunc) 47 | 48 | assert.True(t, results.HasFailures(), "Expected file to not contain base64 encoded texts.") 49 | } 50 | 51 | func TestShouldNotFlagPotentialSecretsWithinSafeJavaCodeEvenInAggressiveMode(t *testing.T) { 52 | const awsAccessKeyIDExample string = "public class HelloWorld {\r\n\r\n" + 53 | " public static void main(String[] args) {\r\n " + 54 | " // Prints \"Hello, World\" to the terminal window.\r\n " + 55 | " System.out.println(\"Hello, World\");\r\n " + 56 | " }\r\n\r\n" + 57 | "}" 58 | results := helpers.NewDetectionResults() 59 | filename := "filename" 60 | additions := []gitrepo.Addition{gitrepo.NewAddition(filename, []byte(awsAccessKeyIDExample))} 61 | 62 | aggressiveModeFileContentDetector. 63 | Test( 64 | defaultIgnoreEvaluator, 65 | additions, 66 | _blankTalismanRC, 67 | results, 68 | dummyCompletionCallbackFunc) 69 | 70 | assert.False(t, results.HasFailures(), "Expected file to not contain base64 encoded texts.") 71 | } 72 | -------------------------------------------------------------------------------- /detector/filecontent/base64_detector.go: -------------------------------------------------------------------------------- 1 | package filecontent 2 | 3 | import ( 4 | "math" 5 | "talisman/talismanrc" 6 | 7 | log "github.com/sirupsen/logrus" 8 | ) 9 | 10 | const BASE64_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=" 11 | const BASE64_ENTROPY_THRESHOLD = 4.5 12 | const MIN_BASE64_SECRET_LENGTH = 20 13 | 14 | type Base64Detector struct { 15 | base64Map map[string]bool 16 | AggressiveDetector *Base64AggressiveDetector 17 | entropy *Entropy 18 | wordCheck *WordCheck 19 | base64EntropyThreshold float64 20 | } 21 | 22 | func NewBase64Detector(tRC *talismanrc.TalismanRC) *Base64Detector { 23 | bd := Base64Detector{} 24 | bd.initBase64Map() 25 | bd.AggressiveDetector = nil 26 | 27 | bd.base64EntropyThreshold = BASE64_ENTROPY_THRESHOLD 28 | if tRC.Experimental.Base64EntropyThreshold > 0.0 { 29 | bd.base64EntropyThreshold = tRC.Experimental.Base64EntropyThreshold 30 | log.Debugf("Setting b64 entropy threshold to %f", bd.base64EntropyThreshold) 31 | } 32 | 33 | bd.entropy = &Entropy{} 34 | return &bd 35 | } 36 | 37 | func (bd *Base64Detector) initBase64Map() { 38 | bd.base64Map = map[string]bool{} 39 | for i := 0; i < len(BASE64_CHARS); i++ { 40 | bd.base64Map[string(BASE64_CHARS[i])] = true 41 | } 42 | } 43 | 44 | func (bd *Base64Detector) CheckBase64Encoding(word string) string { 45 | entropyCandidates := bd.entropy.GetEntropyCandidatesWithinWord(word, MIN_BASE64_SECRET_LENGTH, bd.base64Map) 46 | for _, candidate := range entropyCandidates { 47 | entropy := bd.entropy.GetShannonEntropy(candidate, BASE64_CHARS) 48 | sliceLimit := int(math.Min(50, float64(len(candidate)))) 49 | log.Debugf("Detected entropy for word %s = %f", candidate[0:sliceLimit], entropy) 50 | if entropy > bd.base64EntropyThreshold && !bd.wordCheck.containsWordsOnly(candidate) { 51 | return word 52 | } 53 | } 54 | if bd.AggressiveDetector != nil { 55 | return bd.AggressiveDetector.Test(word) 56 | } 57 | return "" 58 | } 59 | -------------------------------------------------------------------------------- /detector/filecontent/base64_detector_test.go: -------------------------------------------------------------------------------- 1 | package filecontent 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestBase64DetectorShouldNotDetectSafeText(t *testing.T) { 10 | s := "pretty safe" 11 | bd := Base64Detector{} 12 | bd.initBase64Map() 13 | 14 | res := bd.CheckBase64Encoding(s) 15 | assert.Equal(t, "", res) 16 | } 17 | 18 | func TestBase64DetectorShouldDetectBase64Text(t *testing.T) { 19 | s := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" 20 | bd := Base64Detector{} 21 | bd.initBase64Map() 22 | 23 | res := bd.CheckBase64Encoding(s) 24 | assert.Equal(t, s, res) 25 | } 26 | 27 | func TestBase64DetectorShouldNotDetectLongMethodNamesEvenWithHighEntropy(t *testing.T) { 28 | s := "TestBase64DetectorShouldNotDetectLongMethodNamesEvenWithRidiculousHighEntropyWordsMightExist" 29 | bd := Base64Detector{} 30 | bd.initBase64Map() 31 | 32 | res := bd.CheckBase64Encoding(s) 33 | assert.Equal(t, "", res) 34 | } 35 | -------------------------------------------------------------------------------- /detector/filecontent/filecontent_credit_card_detector.go: -------------------------------------------------------------------------------- 1 | package filecontent 2 | 3 | import "regexp" 4 | 5 | type CreditCardDetector struct { 6 | creditCardRegex []*regexp.Regexp 7 | } 8 | 9 | var ( 10 | creditCardPatterns = []*regexp.Regexp{ 11 | regexp.MustCompile(`(?:3[47][0-9]{13})`), 12 | regexp.MustCompile(`(?:3(?:0[0-5]|[68][0-9])[0-9]{11})`), 13 | regexp.MustCompile(`^65[4-9][0-9]{13}|64[4-9][0-9]{13}|6011[0-9]{12}|(622(?:12[6-9]|1[3-9][0-9]|[2-8][0-9][0-9]|9[01][0-9]|92[0-5])[0-9]{10})$`), 14 | regexp.MustCompile(`^(?:2131|1800|35\d{3})\d{11}$`), 15 | regexp.MustCompile(`^(5018|5020|5038|6304|6759|6761|6763)[0-9]{8,15}$`), 16 | regexp.MustCompile(`(?:(?:5[1-5][0-9]{2}|222[1-9]|22[3-9][0-9]|2[3-6][0-9]{2}|27[01][0-9]|2720)[0-9]{12})`), 17 | regexp.MustCompile(`((?:4[0-9]{12})(?:[0-9]{3})?)`), 18 | } 19 | ) 20 | 21 | func (detector CreditCardDetector) checkCreditCardNumber(content string) string { 22 | if !isLuhnNumber(content) { 23 | return "" 24 | } 25 | for _, regex := range detector.creditCardRegex { 26 | if regex.MatchString(content) { 27 | return content 28 | } 29 | } 30 | return "" 31 | } 32 | 33 | func NewCreditCardDetector() *CreditCardDetector { 34 | return &CreditCardDetector{creditCardPatterns} 35 | } 36 | 37 | func isLuhnNumber(content string) bool { 38 | var isAlternate bool 39 | var checksum int 40 | 41 | for position := len(content) - 1; position > -1; position-- { 42 | const ASCII_INDEX = 48 43 | digit := int(content[position] - ASCII_INDEX) 44 | if isAlternate { 45 | digit = digit * 2 46 | if digit > 9 { 47 | digit = (digit % 10) + 1 48 | } 49 | } 50 | isAlternate = !isAlternate 51 | checksum += digit 52 | } 53 | if checksum%10 == 0 { 54 | return true 55 | } 56 | return false 57 | } 58 | -------------------------------------------------------------------------------- /detector/filecontent/filecontent_credit_card_detector_test.go: -------------------------------------------------------------------------------- 1 | package filecontent 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestShouldReturnEmptyStringWhenCreditCardNumberDoesNotMatchAnyRegex(t *testing.T) { 10 | assert.Equal(t, "", NewCreditCardDetector().checkCreditCardNumber("1234123412341234")) 11 | } 12 | 13 | func TestShouldReturnEmptyStringWhenValidMasterCardPatternIsDetectedButNotValidLuhnNumber(t *testing.T) { 14 | assert.Equal(t, "", NewCreditCardDetector().checkCreditCardNumber("52222111111111111")) 15 | } 16 | 17 | func TestShouldReturnCardNumberWhenAmericanExpressCardNumberIsGiven(t *testing.T) { 18 | assert.Equal(t, "340000000000009", NewCreditCardDetector().checkCreditCardNumber("340000000000009")) 19 | } 20 | 21 | func TestShouldReturnCardNumberWhenDinersClubCardNumberIsGiven(t *testing.T) { 22 | assert.Equal(t, "30000000000004", NewCreditCardDetector().checkCreditCardNumber("30000000000004")) 23 | } 24 | 25 | func TestShouldReturnCardNumberWhenDiscoverCardNumberIsGiven(t *testing.T) { 26 | assert.Equal(t, "6011000000000004", NewCreditCardDetector().checkCreditCardNumber("6011000000000004")) 27 | } 28 | 29 | func TestShouldReturnCardNumberWhenJCBCardNumberIsGiven(t *testing.T) { 30 | assert.Equal(t, "3530111333300000", NewCreditCardDetector().checkCreditCardNumber("3530111333300000")) 31 | } 32 | 33 | func TestShouldReturnCardNumberWhenMaestroCardNumberIsGiven(t *testing.T) { 34 | assert.Equal(t, "6759649826438453", NewCreditCardDetector().checkCreditCardNumber("6759649826438453")) 35 | } 36 | 37 | func TestShouldReturnCardNumberWhenVisaCardNumberIsGiven(t *testing.T) { 38 | assert.Equal(t, "4111111111111111", NewCreditCardDetector().checkCreditCardNumber("4111111111111111")) 39 | } 40 | -------------------------------------------------------------------------------- /detector/filecontent/filecontent_detector.go: -------------------------------------------------------------------------------- 1 | package filecontent 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | "sync" 8 | "talisman/detector/helpers" 9 | "talisman/detector/severity" 10 | "talisman/gitrepo" 11 | "talisman/talismanrc" 12 | 13 | log "github.com/sirupsen/logrus" 14 | ) 15 | 16 | type fn func(fc *FileContentDetector, word string) string 17 | 18 | type FileContentDetector struct { 19 | base64Detector *Base64Detector 20 | hexDetector *HexDetector 21 | creditCardDetector *CreditCardDetector 22 | base64EntropyThreshold float64 23 | } 24 | 25 | func NewFileContentDetector(tRC *talismanrc.TalismanRC) *FileContentDetector { 26 | fc := FileContentDetector{} 27 | fc.base64Detector = NewBase64Detector(tRC) 28 | fc.hexDetector = NewHexDetector() 29 | fc.creditCardDetector = NewCreditCardDetector() 30 | return &fc 31 | } 32 | 33 | func (fc *FileContentDetector) AggressiveMode() *FileContentDetector { 34 | fc.base64Detector.AggressiveDetector = &Base64AggressiveDetector{} 35 | return fc 36 | } 37 | 38 | type contentType int 39 | 40 | const ( 41 | base64Content contentType = iota 42 | hexContent 43 | creditCardContent 44 | ) 45 | 46 | func (ct contentType) getInfo() string { 47 | switch ct { 48 | case base64Content: 49 | return "Base64Detector: Failing file as it contains a base64 encoded text." 50 | case hexContent: 51 | return "HexDetector: Failing file as it contains a hex encoded text." 52 | case creditCardContent: 53 | return "CreditCardDetector: Failing file as it contains a potential credit card number." 54 | } 55 | return "" 56 | } 57 | 58 | func (ct contentType) getMessageFormat() string { 59 | switch ct { 60 | case base64Content: 61 | return "Expected file to not contain base64 encoded texts such as: %s" 62 | case hexContent: 63 | return "Expected file to not contain hex encoded texts such as: %s" 64 | case creditCardContent: 65 | return "Expected file to not contain credit card numbers such as: %s" 66 | } 67 | 68 | return "" 69 | } 70 | 71 | type content struct { 72 | name gitrepo.FileName 73 | path gitrepo.FilePath 74 | contentType contentType 75 | results []string 76 | severity severity.Severity 77 | } 78 | 79 | func (fc *FileContentDetector) Test(comparator helpers.IgnoreEvaluator, currentAdditions []gitrepo.Addition, talismanRC *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) { 80 | contentTypes := []struct { 81 | contentType 82 | fn 83 | severity severity.Severity 84 | }{ 85 | { 86 | contentType: base64Content, 87 | fn: checkBase64, 88 | severity: severity.SeverityConfiguration["Base64Content"], 89 | }, 90 | { 91 | contentType: hexContent, 92 | fn: checkHex, 93 | severity: severity.SeverityConfiguration["HexContent"], 94 | }, 95 | { 96 | contentType: creditCardContent, 97 | fn: checkCreditCardNumber, 98 | severity: severity.SeverityConfiguration["CreditCardContent"], 99 | }, 100 | } 101 | re := regexp.MustCompile(`(?i)checksum[ \t]*:[ \t]*[0-9a-fA-F]+`) 102 | 103 | contents := make(chan content, 512) 104 | ignoredFilePaths := make(chan gitrepo.FilePath, len(currentAdditions)) 105 | 106 | waitGroup := &sync.WaitGroup{} 107 | waitGroup.Add(len(currentAdditions)) 108 | for _, addition := range currentAdditions { 109 | go func(addition gitrepo.Addition) { 110 | defer waitGroup.Done() 111 | defer additionCompletionCallback() 112 | if comparator.ShouldIgnore(addition, "filecontent") { 113 | ignoredFilePaths <- addition.Path 114 | return 115 | } 116 | 117 | if string(addition.Name) == talismanrc.RCFileName { 118 | content := re.ReplaceAllString(string(addition.Data), "") 119 | data := []byte(content) 120 | addition.Data = data 121 | } 122 | addition.Data = []byte(talismanRC.RemoveAllowedPatterns(addition)) 123 | for _, ct := range contentTypes { 124 | contents <- content{ 125 | name: addition.Name, 126 | path: addition.Path, 127 | contentType: ct.contentType, 128 | results: fc.detectFile(addition.Data, ct.fn), 129 | severity: ct.severity, 130 | } 131 | } 132 | }(addition) 133 | } 134 | go func() { 135 | waitGroup.Wait() 136 | close(ignoredFilePaths) 137 | close(contents) 138 | }() 139 | for ignoredChanHasMore, contentChanHasMore := true, true; ignoredChanHasMore || contentChanHasMore; { 140 | select { 141 | case ignoredFilePath, hasMore := <-ignoredFilePaths: 142 | log.Debugf("Processing results for ignored file %v", ignoredFilePath) 143 | if !hasMore { 144 | ignoredChanHasMore = false 145 | continue 146 | } 147 | processIgnoredFilepath(ignoredFilePath, result) 148 | case c, hasMore := <-contents: 149 | log.Debugf("Processing results for file %v", c.path) 150 | if !hasMore { 151 | contentChanHasMore = false 152 | continue 153 | } 154 | processContent(c, talismanRC.Threshold, result) 155 | } 156 | } 157 | } 158 | 159 | func processIgnoredFilepath(path gitrepo.FilePath, result *helpers.DetectionResults) { 160 | log.WithFields(log.Fields{ 161 | "filePath": path, 162 | }).Info("Ignoring addition as it was specified to be ignored.") 163 | result.Ignore(path, "filecontent") 164 | } 165 | 166 | func processContent(c content, threshold severity.Severity, result *helpers.DetectionResults) { 167 | for _, res := range c.results { 168 | if res != "" { 169 | log.WithFields(log.Fields{ 170 | "filePath": c.path, 171 | }).Info(c.contentType.getInfo()) 172 | if string(c.name) == talismanrc.RCFileName || !c.severity.ExceedsThreshold(threshold) { 173 | result.Warn(c.path, "filecontent", fmt.Sprintf(c.contentType.getMessageFormat(), formatForReporting(res)), []string{}, c.severity) 174 | } else { 175 | result.Fail(c.path, "filecontent", fmt.Sprintf(c.contentType.getMessageFormat(), formatForReporting(res)), []string{}, c.severity) 176 | } 177 | } 178 | } 179 | } 180 | 181 | func formatForReporting(input string) string { 182 | if len(input) > 50 { 183 | return input[:47] + "..." 184 | } 185 | return input 186 | } 187 | 188 | func (fc *FileContentDetector) detectFile(data []byte, getResult fn) []string { 189 | content := string(data) 190 | return fc.checkEachLine(content, getResult) 191 | } 192 | 193 | func (fc *FileContentDetector) checkEachLine(content string, getResult fn) []string { 194 | lines := strings.Split(content, "\n") 195 | res := []string{} 196 | for _, line := range lines { 197 | lineResult := fc.checkEachWord(line, getResult) 198 | if len(lineResult) > 0 { 199 | res = append(res, lineResult...) 200 | } 201 | } 202 | return res 203 | } 204 | 205 | func (fc *FileContentDetector) checkEachWord(line string, getResult fn) []string { 206 | words := strings.Fields(line) 207 | res := []string{} 208 | for _, word := range words { 209 | wordResult := getResult(fc, word) 210 | if wordResult != "" { 211 | res = append(res, wordResult) 212 | } 213 | } 214 | return res 215 | } 216 | 217 | func checkBase64(fc *FileContentDetector, word string) string { 218 | return fc.base64Detector.CheckBase64Encoding(word) 219 | } 220 | 221 | func checkCreditCardNumber(fc *FileContentDetector, word string) string { 222 | return fc.creditCardDetector.checkCreditCardNumber(word) 223 | } 224 | 225 | func checkHex(fc *FileContentDetector, word string) string { 226 | return fc.hexDetector.CheckHexEncoding(word) 227 | } 228 | -------------------------------------------------------------------------------- /detector/filecontent/hex_detector.go: -------------------------------------------------------------------------------- 1 | package filecontent 2 | 3 | const HEX_CHARS = "1234567890abcdefABCDEF" 4 | const HEX_ENTROPY_THRESHOLD = 2.7 5 | const MIN_HEX_SECRET_LENGTH = 20 6 | 7 | type HexDetector struct { 8 | hexMap map[string]bool 9 | entropy *Entropy 10 | } 11 | 12 | func NewHexDetector() *HexDetector { 13 | bd := HexDetector{} 14 | bd.initHexMap() 15 | bd.entropy = &Entropy{} 16 | return &bd 17 | } 18 | 19 | func (hd *HexDetector) initHexMap() { 20 | hd.hexMap = map[string]bool{} 21 | for i := 0; i < len(HEX_CHARS); i++ { 22 | hd.hexMap[string(HEX_CHARS[i])] = true 23 | } 24 | } 25 | 26 | func (hd *HexDetector) CheckHexEncoding(word string) string { 27 | entropyCandidates := hd.entropy.GetEntropyCandidatesWithinWord(word, MIN_HEX_SECRET_LENGTH, hd.hexMap) 28 | for _, candidate := range entropyCandidates { 29 | entropy := hd.entropy.GetShannonEntropy(candidate, HEX_CHARS) 30 | if entropy > HEX_ENTROPY_THRESHOLD { 31 | return word 32 | } 33 | } 34 | return "" 35 | } 36 | -------------------------------------------------------------------------------- /detector/filecontent/hex_detector_test.go: -------------------------------------------------------------------------------- 1 | package filecontent 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestHexDetectorShouldNotDetectSafeText(t *testing.T) { 10 | s := "pretty safe" 11 | hd := HexDetector{} 12 | hd.initHexMap() 13 | 14 | res := hd.CheckHexEncoding(s) 15 | assert.Equal(t, "", res) 16 | } 17 | 18 | func TestHexDetectorShouldDetectBase64Text(t *testing.T) { 19 | s := "6A6176617375636B73676F726F636B7368616861" 20 | hd := HexDetector{} 21 | hd.initHexMap() 22 | 23 | res := hd.CheckHexEncoding(s) 24 | assert.Equal(t, s, res) 25 | } 26 | -------------------------------------------------------------------------------- /detector/filecontent/shannon_entropy.go: -------------------------------------------------------------------------------- 1 | package filecontent 2 | 3 | import ( 4 | "math" 5 | "strings" 6 | ) 7 | 8 | type Entropy struct { 9 | } 10 | 11 | func (en *Entropy) GetShannonEntropy(str string, superSet string) float64 { 12 | if str == "" { 13 | return 0 14 | } 15 | entropy := 0.0 16 | for _, c := range superSet { 17 | p := float64(strings.Count(str, string(c))) / float64(len(str)) 18 | if p > 0 { 19 | entropy -= p * math.Log2(p) 20 | } 21 | } 22 | return entropy 23 | } 24 | 25 | func (en *Entropy) GetEntropyCandidatesWithinWord(word string, minCandidateLength int, superSet map[string]bool) []string { 26 | candidates := []string{} 27 | count := 0 28 | subSet := "" 29 | if len(word) < minCandidateLength { 30 | return candidates 31 | } 32 | for _, c := range word { 33 | char := string(c) 34 | if superSet[char] { 35 | subSet += char 36 | count++ 37 | } else { 38 | if count > minCandidateLength { 39 | candidates = append(candidates, subSet) 40 | } 41 | subSet = "" 42 | count = 0 43 | } 44 | } 45 | if count > minCandidateLength { 46 | candidates = append(candidates, subSet) 47 | } 48 | return candidates 49 | } 50 | -------------------------------------------------------------------------------- /detector/filecontent/shannon_entropy_test.go: -------------------------------------------------------------------------------- 1 | package filecontent 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestEntropyCandidatesShouldBeFoundForGivenSet(t *testing.T) { 10 | entropy := Entropy{} 11 | dc := Base64Detector{} 12 | dc.initBase64Map() 13 | candidatesWithinWord := entropy.GetEntropyCandidatesWithinWord("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", 20, dc.base64Map) 14 | assert.Equal(t, 1, len(candidatesWithinWord)) 15 | } 16 | 17 | func TestEntropyCandidatesShouldBeEmptyForShorterWords(t *testing.T) { 18 | entropy := Entropy{} 19 | dc := Base64Detector{} 20 | dc.initBase64Map() 21 | candidatesWithinWord := entropy.GetEntropyCandidatesWithinWord("abc", 4, dc.base64Map) 22 | assert.Equal(t, 0, len(candidatesWithinWord)) 23 | } 24 | 25 | func TestEntropyValueOfSecretShouldBeHigherThanFour(t *testing.T) { 26 | entropy := Entropy{} 27 | shannonEntropy := entropy.GetShannonEntropy("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", BASE64_CHARS) 28 | assert.True(t, 4 < shannonEntropy) 29 | } 30 | 31 | func TestEntropyValueOfEmptyStringShouldBeZero(t *testing.T) { 32 | entropy := Entropy{} 33 | shannonEntropy := entropy.GetShannonEntropy("", BASE64_CHARS) 34 | assert.Equal(t, float64(0), shannonEntropy) 35 | } 36 | -------------------------------------------------------------------------------- /detector/filecontent/word_check.go: -------------------------------------------------------------------------------- 1 | package filecontent 2 | 3 | import ( 4 | "bufio" 5 | log "github.com/sirupsen/logrus" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | type WordCheck struct { 11 | } 12 | 13 | const AVERAGE_LENGTH_OF_WORDS_IN_ENGLISH = 5 //See http://bit.ly/2qYFzFf for reference 14 | 15 | func (en *WordCheck) containsWordsOnly(text string) bool { 16 | text = strings.ToLower(text) 17 | file := &os.File{} 18 | defer file.Close() 19 | reader := bufio.NewReader(strings.NewReader(DictionaryWordsString)) 20 | if reader == nil { 21 | return false 22 | } 23 | wordCount := howManyWordsExistInText(reader, text) 24 | if wordCount >= (len(text) / (AVERAGE_LENGTH_OF_WORDS_IN_ENGLISH)) { 25 | return true 26 | } 27 | return false 28 | } 29 | 30 | func howManyWordsExistInText(reader *bufio.Reader, text string) int { 31 | wordCount := 0 32 | for { 33 | word, err := reader.ReadString('\n') 34 | word = strings.Trim(word, "\n") 35 | 36 | if word != "" && len(word) > 2 && strings.Contains(text, word) { 37 | text = strings.Replace(text, word, "", 1) //already matched 38 | wordCount++ 39 | } 40 | 41 | if err != nil { //EOF 42 | log.Debugf("[WordChecker]: Found %d words", wordCount) 43 | break 44 | } 45 | } 46 | return wordCount 47 | } 48 | -------------------------------------------------------------------------------- /detector/filecontent/word_check_test.go: -------------------------------------------------------------------------------- 1 | package filecontent 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestWordCheckWithWordsOnlyText(t *testing.T) { 10 | wc := WordCheck{} 11 | isWordsOnly := wc.containsWordsOnly("helloWorldGreetingsFromThoughtWorks") 12 | assert.True(t, isWordsOnly) 13 | } 14 | 15 | func TestWordCheckWithWordsOnlyLongText(t *testing.T) { 16 | wc := WordCheck{} 17 | isWordsOnly := wc.containsWordsOnly("TestBase64DetectorShouldNotDetectLongMethodNamesEvenWithRidiculousHighEntropyWordsMightExist") 18 | assert.True(t, isWordsOnly) 19 | } 20 | 21 | func TestWordCheckWithSingleWord(t *testing.T) { 22 | wc := WordCheck{} 23 | isWordsOnly := wc.containsWordsOnly("exception") 24 | assert.True(t, isWordsOnly) 25 | } 26 | 27 | func TestWordCheckWithSingleShortWord(t *testing.T) { 28 | wc := WordCheck{} 29 | isWordsOnly := wc.containsWordsOnly("for") 30 | assert.True(t, isWordsOnly) 31 | } 32 | 33 | func TestWordCheckWithSecret(t *testing.T) { 34 | wc := WordCheck{} 35 | isWordsOnly := wc.containsWordsOnly("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY") 36 | assert.False(t, isWordsOnly) 37 | } 38 | 39 | func TestWordCheckWithHex(t *testing.T) { 40 | wc := WordCheck{} 41 | isWordsOnly := wc.containsWordsOnly("68656C6C6F20776F726C6421") 42 | assert.False(t, isWordsOnly) 43 | } 44 | 45 | func TestWordCheckWithHalfWordsHalfHex(t *testing.T) { 46 | wc := WordCheck{} 47 | isWordsOnly := wc.containsWordsOnly("68656C6C6F20776F726C6421helloWorldGreetingsFromThoughtWorks") 48 | assert.False(t, isWordsOnly) 49 | } 50 | 51 | func TestWordCheckWithHalfWordsHalfSecret(t *testing.T) { 52 | wc := WordCheck{} 53 | isWordsOnly := wc.containsWordsOnly("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEYhelloWorldGreetingsFromThoughtWorks") 54 | assert.False(t, isWordsOnly) 55 | } 56 | -------------------------------------------------------------------------------- /detector/filesize/filesize_detector.go: -------------------------------------------------------------------------------- 1 | package filesize 2 | 3 | import ( 4 | "fmt" 5 | "talisman/detector/detector" 6 | "talisman/detector/helpers" 7 | "talisman/detector/severity" 8 | "talisman/gitrepo" 9 | "talisman/talismanrc" 10 | 11 | log "github.com/sirupsen/logrus" 12 | ) 13 | 14 | type FileSizeDetector struct { 15 | size int 16 | } 17 | 18 | func NewFileSizeDetector(size int) detector.Detector { 19 | return FileSizeDetector{size} 20 | } 21 | 22 | func (fd FileSizeDetector) Test(comparator helpers.IgnoreEvaluator, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) { 23 | largeFileSizeSeverity := severity.SeverityConfiguration["LargeFileSize"] 24 | for _, addition := range currentAdditions { 25 | if comparator.ShouldIgnore(addition, "filesize") { 26 | log.WithFields(log.Fields{ 27 | "filePath": addition.Path, 28 | }).Info("Ignoring addition as it was specified to be ignored.") 29 | result.Ignore(addition.Path, "filesize") 30 | additionCompletionCallback() 31 | continue 32 | } 33 | size := len(addition.Data) 34 | if size > fd.size { 35 | log.WithFields(log.Fields{ 36 | "filePath": addition.Path, 37 | "fileSize": size, 38 | "maxSize": fd.size, 39 | }).Info("Failing file as it is larger than max allowed file size.") 40 | if largeFileSizeSeverity.ExceedsThreshold(ignoreConfig.Threshold) { 41 | result.Fail(addition.Path, "filesize", fmt.Sprintf("The file name %q with file size %d is larger than max allowed file size(%d)", addition.Path, size, fd.size), addition.Commits, largeFileSizeSeverity) 42 | } else { 43 | result.Warn(addition.Path, "filesize", fmt.Sprintf("The file name %q with file size %d is larger than max allowed file size(%d)", addition.Path, size, fd.size), addition.Commits, largeFileSizeSeverity) 44 | } 45 | } 46 | additionCompletionCallback() 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /detector/filesize/filesize_detector_test.go: -------------------------------------------------------------------------------- 1 | package filesize 2 | 3 | import ( 4 | "talisman/detector/helpers" 5 | "talisman/detector/severity" 6 | "testing" 7 | 8 | "talisman/gitrepo" 9 | "talisman/talismanrc" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | var talismanRC = &talismanrc.TalismanRC{} 15 | var defaultIgnoreEvaluator = helpers.BuildIgnoreEvaluator("default", talismanRC, gitrepo.RepoLocatedAt(".")) 16 | 17 | func ignoreEvaluatorWithTalismanRC(tRC *talismanrc.TalismanRC) helpers.IgnoreEvaluator { 18 | return helpers.BuildIgnoreEvaluator("default", tRC, gitrepo.RepoLocatedAt(".")) 19 | } 20 | 21 | func TestShouldFlagLargeFiles(t *testing.T) { 22 | results := helpers.NewDetectionResults() 23 | content := []byte("more than one byte") 24 | additions := []gitrepo.Addition{gitrepo.NewAddition("filename", content)} 25 | NewFileSizeDetector(2).Test(defaultIgnoreEvaluator, additions, talismanRC, results, func() {}) 26 | assert.True(t, results.HasFailures(), "Expected file to fail the check against file size detector.") 27 | } 28 | 29 | func TestShouldNotFlagLargeFilesIfThresholdIsBelowSeverity(t *testing.T) { 30 | results := helpers.NewDetectionResults() 31 | content := []byte("more than one byte") 32 | talismanRCWithThreshold := &talismanrc.TalismanRC{Threshold: severity.High} 33 | additions := []gitrepo.Addition{gitrepo.NewAddition("filename", content)} 34 | NewFileSizeDetector(2).Test(defaultIgnoreEvaluator, additions, talismanRCWithThreshold, results, func() {}) 35 | assert.False(t, results.HasFailures(), "Expected file to not fail the check against file size detector.") 36 | assert.True(t, results.HasWarnings(), "Expected file to have warnings against file size detector.") 37 | } 38 | 39 | func TestShouldNotFlagSmallFiles(t *testing.T) { 40 | results := helpers.NewDetectionResults() 41 | content := []byte("m") 42 | additions := []gitrepo.Addition{gitrepo.NewAddition("filename", content)} 43 | NewFileSizeDetector(2).Test(defaultIgnoreEvaluator, additions, talismanRC, results, func() {}) 44 | assert.False(t, results.HasFailures(), "Expected file to not fail the check against file size detector.") 45 | } 46 | 47 | func TestShouldNotFlagIgnoredLargeFiles(t *testing.T) { 48 | results := helpers.NewDetectionResults() 49 | content := []byte("more than one byte") 50 | 51 | filename := "filename" 52 | fileIgnoreConfig := talismanrc.FileIgnoreConfig{ 53 | FileName: filename, 54 | IgnoreDetectors: []string{"filesize"}, 55 | } 56 | talismanRC := &talismanrc.TalismanRC{ 57 | FileIgnoreConfig: []talismanrc.FileIgnoreConfig{fileIgnoreConfig}, 58 | } 59 | 60 | additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} 61 | NewFileSizeDetector(2).Test(ignoreEvaluatorWithTalismanRC(talismanRC), additions, talismanRC, results, func() {}) 62 | assert.True(t, results.Successful(), "expected file %s to be ignored by file size detector", filename) 63 | } 64 | -------------------------------------------------------------------------------- /detector/helpers/ignore_evaluator.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "os" 5 | "talisman/checksumcalculator" 6 | "talisman/gitrepo" 7 | "talisman/talismanrc" 8 | "talisman/utility" 9 | ) 10 | 11 | type IgnoreEvaluator interface { 12 | ShouldIgnore(addition gitrepo.Addition, detectorType string) bool 13 | } 14 | 15 | type scanAllAdditions struct{} 16 | 17 | // Returns an IgnoreEvaluator that forces all files to be scanned, such as when scanning the history of a repo 18 | func ScanHistoryEvaluator() IgnoreEvaluator { 19 | return &scanAllAdditions{} 20 | } 21 | 22 | // Returns false so that all additions are scanned 23 | func (ie *scanAllAdditions) ShouldIgnore(gitrepo.Addition, string) bool { 24 | return false 25 | } 26 | 27 | type ignoreEvaluator struct { 28 | calculator checksumcalculator.ChecksumCalculator 29 | talismanRC *talismanrc.TalismanRC 30 | } 31 | 32 | // Returns an IgnoreEvaluator around the rules defined in the current .talismanrc file 33 | func BuildIgnoreEvaluator(hasherMode string, talismanRC *talismanrc.TalismanRC, repo gitrepo.GitRepo) IgnoreEvaluator { 34 | wd, _ := os.Getwd() 35 | hasher := utility.MakeHasher(hasherMode, wd) 36 | allTrackedFiles := append(repo.TrackedFilesAsAdditions(), repo.StagedAdditions()...) 37 | calculator := checksumcalculator.NewChecksumCalculator(hasher, allTrackedFiles) 38 | return &ignoreEvaluator{calculator: calculator, talismanRC: talismanRC} 39 | } 40 | 41 | // ShouldIgnore returns true if the talismanRC indicates that a Detector should ignore an Addition 42 | func (ie *ignoreEvaluator) ShouldIgnore(addition gitrepo.Addition, detectorType string) bool { 43 | return ie.talismanRC.Deny(addition, detectorType) || ie.isScanNotRequired(addition) 44 | } 45 | 46 | // isScanNotRequired returns true if an Addition's checksum matches one ignored by the .talismanrc file 47 | func (ie *ignoreEvaluator) isScanNotRequired(addition gitrepo.Addition) bool { 48 | for _, ignore := range ie.talismanRC.FileIgnoreConfig { 49 | if addition.Matches(ignore.GetFileName()) { 50 | currentCollectiveChecksum := ie.calculator.CalculateCollectiveChecksumForPattern(ignore.GetFileName()) 51 | return ignore.ChecksumMatches(currentCollectiveChecksum) 52 | } 53 | } 54 | return false 55 | } 56 | -------------------------------------------------------------------------------- /detector/helpers/ignore_evaluator_test.go: -------------------------------------------------------------------------------- 1 | package helpers 2 | 3 | import ( 4 | "io" 5 | "talisman/gitrepo" 6 | mockchecksumcalculator "talisman/internal/mock/checksumcalculator" 7 | "talisman/talismanrc" 8 | 9 | "github.com/golang/mock/gomock" 10 | logr "github.com/sirupsen/logrus" 11 | "github.com/stretchr/testify/assert" 12 | 13 | "testing" 14 | ) 15 | 16 | func init() { 17 | logr.SetOutput(io.Discard) 18 | } 19 | func TestIsScanNotRequired(t *testing.T) { 20 | 21 | t.Run("should return false if talismanrc is empty", func(t *testing.T) { 22 | ignoreConfig := &talismanrc.TalismanRC{ 23 | FileIgnoreConfig: []talismanrc.FileIgnoreConfig{}, 24 | } 25 | ie := ignoreEvaluator{nil, ignoreConfig} 26 | addition := gitrepo.Addition{Path: "some.txt"} 27 | 28 | required := ie.isScanNotRequired(addition) 29 | 30 | assert.False(t, required) 31 | }) 32 | 33 | t.Run("should loop through talismanrc configs", func(t *testing.T) { 34 | ctrl := gomock.NewController(t) 35 | defer ctrl.Finish() 36 | checksumCalculator := mockchecksumcalculator.NewMockChecksumCalculator(ctrl) 37 | ignoreConfig := talismanrc.TalismanRC{ 38 | FileIgnoreConfig: []talismanrc.FileIgnoreConfig{ 39 | { 40 | FileName: "some.txt", 41 | Checksum: "sha1", 42 | }, 43 | }, 44 | } 45 | ie := ignoreEvaluator{calculator: checksumCalculator, talismanRC: &ignoreConfig} 46 | addition := gitrepo.Addition{Name: "some.txt", Path: "some.txt"} 47 | checksumCalculator.EXPECT().CalculateCollectiveChecksumForPattern("some.txt").Return("sha1") 48 | 49 | required := ie.isScanNotRequired(addition) 50 | 51 | assert.True(t, required) 52 | }) 53 | 54 | } 55 | 56 | type sillyChecksumCalculator struct{} 57 | 58 | func (scc *sillyChecksumCalculator) CalculateCollectiveChecksumForPattern(fileNamePattern string) string { 59 | return "silly" 60 | } 61 | func (scc *sillyChecksumCalculator) SuggestTalismanRC(fileNamePatterns []string) string { 62 | return "" 63 | } 64 | 65 | func TestDeterminingFilesToIgnore(t *testing.T) { 66 | tRC := talismanrc.TalismanRC{ 67 | FileIgnoreConfig: []talismanrc.FileIgnoreConfig{ 68 | { 69 | FileName: "some.txt", 70 | Checksum: "silly", 71 | }, 72 | { 73 | FileName: "other.txt", 74 | Checksum: "serious", 75 | }, 76 | { 77 | FileName: "ignore-contents", 78 | IgnoreDetectors: []string{"filecontent"}, 79 | }, 80 | }, 81 | } 82 | ie := ignoreEvaluator{&sillyChecksumCalculator{}, &tRC} 83 | 84 | t.Run("Should ignore file based on checksum", func(t *testing.T) { 85 | assert.True(t, ie.ShouldIgnore(gitrepo.Addition{Path: "some.txt"}, "")) 86 | }) 87 | 88 | t.Run("Should not ignore file if checksum doesn't match", func(t *testing.T) { 89 | assert.False(t, ie.ShouldIgnore(gitrepo.Addition{Path: "other.txt"}, "")) 90 | }) 91 | 92 | t.Run("Should ignore if detector is disabled for file", func(t *testing.T) { 93 | assert.True(t, ie.ShouldIgnore(gitrepo.Addition{Path: "ignore-contents"}, "filecontent")) 94 | }) 95 | 96 | t.Run("Should not ignore if a different detector is disabled for file", func(t *testing.T) { 97 | assert.False(t, ie.ShouldIgnore(gitrepo.Addition{Path: "ignore-contents"}, "filename")) 98 | }) 99 | } 100 | 101 | func TestNeverIgnoreFilesForHistory(t *testing.T) { 102 | scanAllEvaluator := ScanHistoryEvaluator() 103 | assert.False(t, scanAllEvaluator.ShouldIgnore(gitrepo.Addition{Name: "any-file"}, "any_detector")) 104 | } 105 | -------------------------------------------------------------------------------- /detector/pattern/match_pattern.go: -------------------------------------------------------------------------------- 1 | package pattern 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "talisman/detector/severity" 7 | "talisman/talismanrc" 8 | 9 | "github.com/sirupsen/logrus" 10 | ) 11 | 12 | type PatternMatcher struct { 13 | regexes []*severity.PatternSeverity 14 | } 15 | 16 | type DetectionsWithSeverity struct { 17 | detections []string 18 | severity severity.Severity 19 | } 20 | 21 | func (pm *PatternMatcher) check(content string, thresholdValue severity.Severity) []DetectionsWithSeverity { 22 | var detectionsWithSeverity []DetectionsWithSeverity 23 | for _, pattern := range pm.regexes { 24 | var detected []string 25 | regex := pattern.Pattern 26 | logrus.Debugf("checking for pattern %v", regex) 27 | matches := regex.FindAllString(content, -1) 28 | if matches != nil { 29 | detected = append(detected, matches...) 30 | detectionsWithSeverity = append(detectionsWithSeverity, DetectionsWithSeverity{detections: detected, severity: pattern.Severity}) 31 | } 32 | } 33 | return detectionsWithSeverity 34 | } 35 | 36 | func (pm *PatternMatcher) add(ps talismanrc.PatternString) { 37 | re, err := regexp.Compile(fmt.Sprintf("(%s)", string(ps))) 38 | if err != nil { 39 | logrus.Warnf("ignoring invalid pattern '%s'", ps) 40 | return 41 | } 42 | logrus.Infof("added custom pattern '%s' with high severity", ps) 43 | pm.regexes = append(pm.regexes, &severity.PatternSeverity{Pattern: re, Severity: severity.SeverityConfiguration["CustomPattern"]}) 44 | } 45 | 46 | func NewPatternMatcher(patterns []*severity.PatternSeverity) *PatternMatcher { 47 | return &PatternMatcher{patterns} 48 | } 49 | -------------------------------------------------------------------------------- /detector/pattern/match_pattern_test.go: -------------------------------------------------------------------------------- 1 | package pattern 2 | 3 | import ( 4 | "io/ioutil" 5 | "regexp" 6 | "talisman/detector/severity" 7 | "talisman/talismanrc" 8 | "testing" 9 | 10 | logr "github.com/sirupsen/logrus" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func init() { 15 | logr.SetOutput(ioutil.Discard) 16 | } 17 | 18 | var ( 19 | testRegexpPasswordPattern = `(?i)(['"_]?password['"]? *[:=][^,;]{8,})` 20 | testRegexpPassword = regexp.MustCompile(testRegexpPasswordPattern) 21 | testRegexpPwPattern = `(?i)(['"_]?pw['"]? *[:=][^,;]{8,})` 22 | testRegexpPw = regexp.MustCompile(testRegexpPwPattern) 23 | ) 24 | 25 | func TestShouldReturnEmptyStringWhenDoesNotMatchAnyRegex(t *testing.T) { 26 | detections := NewPatternMatcher([]*severity.PatternSeverity{{Pattern: testRegexpPassword, Severity: severity.Low}}).check("safeString", severity.Low) 27 | assert.Equal(t, []DetectionsWithSeverity(nil), detections) 28 | } 29 | 30 | func TestShouldReturnStringWhenMatchedPasswordPattern(t *testing.T) { 31 | detections1 := NewPatternMatcher([]*severity.PatternSeverity{{Pattern: testRegexpPassword, Severity: severity.Low}}).check("password\" : 123456789", severity.Low) 32 | detections2 := NewPatternMatcher([]*severity.PatternSeverity{{Pattern: testRegexpPw, Severity: severity.Medium}}).check("pw\" : 123456789", severity.Low) 33 | assert.Equal(t, []DetectionsWithSeverity{{detections: []string{"password\" : 123456789"}, severity: severity.Low}}, detections1) 34 | assert.Equal(t, []DetectionsWithSeverity{{detections: []string{"pw\" : 123456789"}, severity: severity.Medium}}, detections2) 35 | } 36 | 37 | func TestShouldAddGoodPatternWithHighToMatcher(t *testing.T) { 38 | pm := NewPatternMatcher([]*severity.PatternSeverity{}) 39 | pm.add(talismanrc.PatternString(testRegexpPwPattern)) 40 | detections := pm.check("pw\" : 123456789", severity.Low) 41 | assert.Equal(t, []DetectionsWithSeverity{{detections: []string{"pw\" : 123456789"}, severity: severity.High}}, detections) 42 | } 43 | 44 | func TestShouldNotAddBadPatternToMatcher(t *testing.T) { 45 | pm := NewPatternMatcher([]*severity.PatternSeverity{}) 46 | pm.add(`*a(crappy|regex`) 47 | assert.Equal(t, 0, len(pm.regexes)) 48 | } 49 | -------------------------------------------------------------------------------- /detector/pattern/pattern_detector.go: -------------------------------------------------------------------------------- 1 | package pattern 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "sync" 7 | "talisman/detector/helpers" 8 | "talisman/detector/severity" 9 | "talisman/gitrepo" 10 | "talisman/talismanrc" 11 | 12 | log "github.com/sirupsen/logrus" 13 | ) 14 | 15 | type PatternDetector struct { 16 | secretsPattern *PatternMatcher 17 | } 18 | 19 | var ( 20 | detectorPatterns = []*severity.PatternSeverity{ 21 | {Pattern: regexp.MustCompile(`(?i)((.*)(password|passphrase|secret|key|pwd|pword|pass)(.*) *[:=>,][^,;\n]{8,})`), Severity: severity.SeverityConfiguration["PasswordPhrasePattern"]}, 22 | {Pattern: regexp.MustCompile(`(?i)((:)(password|passphrase|secret|key|pwd|pword|pass)(.*) *[ ][^,;\n]{8,})`), Severity: severity.SeverityConfiguration["PasswordPhrasePattern"]}, 23 | {Pattern: regexp.MustCompile(`(?i)(['"_]?pw['"]? *[:=][^,;\n]{8,})`), Severity: severity.SeverityConfiguration["PasswordPhrasePattern"]}, 24 | {Pattern: regexp.MustCompile(`(?i)(\S*)`), Severity: severity.SeverityConfiguration["ConsumerKeyPattern"]}, 25 | {Pattern: regexp.MustCompile(`(?i)(\S*)`), Severity: severity.SeverityConfiguration["ConsumerSecretParrern"]}, 26 | {Pattern: regexp.MustCompile(`(?i)(AWS[ \w]+key[ \w]+[:=])`), Severity: severity.SeverityConfiguration["AWSKeyPattern"]}, 27 | {Pattern: regexp.MustCompile(`(?i)(AWS[ \w]+secret[ \w]+[:=])`), Severity: severity.SeverityConfiguration["AWSSecretPattern"]}, 28 | {Pattern: regexp.MustCompile(`(?s)(BEGIN RSA PRIVATE KEY.*END RSA PRIVATE KEY)`), Severity: severity.SeverityConfiguration["RSAKeyPattern"]}, 29 | } 30 | ) 31 | 32 | type match struct { 33 | name gitrepo.FileName 34 | path gitrepo.FilePath 35 | commits []string 36 | detections []DetectionsWithSeverity 37 | } 38 | 39 | // Test tests the contents of the Additions to ensure that they don't look suspicious 40 | func (detector PatternDetector) Test(comparator helpers.IgnoreEvaluator, currentAdditions []gitrepo.Addition, ignoreConfig *talismanrc.TalismanRC, result *helpers.DetectionResults, additionCompletionCallback func()) { 41 | matches := make(chan match, 512) 42 | ignoredFilePaths := make(chan gitrepo.FilePath, 512) 43 | waitGroup := &sync.WaitGroup{} 44 | waitGroup.Add(len(currentAdditions)) 45 | for _, addition := range currentAdditions { 46 | go func(addition gitrepo.Addition) { 47 | defer waitGroup.Done() 48 | defer additionCompletionCallback() 49 | if comparator.ShouldIgnore(addition, "filecontent") { 50 | ignoredFilePaths <- addition.Path 51 | return 52 | } 53 | detections := detector.secretsPattern.check(ignoreConfig.RemoveAllowedPatterns(addition), ignoreConfig.Threshold) 54 | matches <- match{name: addition.Name, path: addition.Path, detections: detections, commits: addition.Commits} 55 | }(addition) 56 | } 57 | go func() { 58 | waitGroup.Wait() 59 | close(matches) 60 | close(ignoredFilePaths) 61 | }() 62 | for ignoredChanHasMore, matchChanHasMore := true, true; ignoredChanHasMore || matchChanHasMore; { 63 | select { 64 | case match, hasMore := <-matches: 65 | if !hasMore { 66 | matchChanHasMore = false 67 | continue 68 | } 69 | detector.processMatch(match, result, ignoreConfig.Threshold) 70 | case ignore, hasMore := <-ignoredFilePaths: 71 | if !hasMore { 72 | ignoredChanHasMore = false 73 | continue 74 | } 75 | detector.processIgnore(ignore, result) 76 | } 77 | } 78 | } 79 | 80 | func (detector PatternDetector) processIgnore(ignoredFilePath gitrepo.FilePath, result *helpers.DetectionResults) { 81 | log.WithFields(log.Fields{ 82 | "filePath": ignoredFilePath, 83 | }).Info("Ignoring addition as it was specified to be ignored.") 84 | result.Ignore(ignoredFilePath, "filecontent") 85 | } 86 | 87 | func (detector PatternDetector) processMatch(match match, result *helpers.DetectionResults, threshold severity.Severity) { 88 | for _, detectionWithSeverity := range match.detections { 89 | for _, detection := range detectionWithSeverity.detections { 90 | if detection != "" { 91 | if string(match.name) == talismanrc.RCFileName || !detectionWithSeverity.severity.ExceedsThreshold(threshold) { 92 | log.WithFields(log.Fields{ 93 | "filePath": match.path, 94 | "pattern": detection, 95 | }).Warn("Warning file as it matched pattern.") 96 | result.Warn(match.path, "filecontent", fmt.Sprintf("Potential secret pattern : %s", detection), match.commits, detectionWithSeverity.severity) 97 | } else { 98 | log.WithFields(log.Fields{ 99 | "filePath": match.path, 100 | "pattern": detection, 101 | }).Info("Failing file as it matched pattern.") 102 | result.Fail(match.path, "filecontent", fmt.Sprintf("Potential secret pattern : %s", detection), match.commits, detectionWithSeverity.severity) 103 | } 104 | } 105 | } 106 | } 107 | } 108 | 109 | // NewPatternDetector returns a PatternDetector that tests Additions against the pre-configured patterns 110 | func NewPatternDetector(custom []talismanrc.PatternString) *PatternDetector { 111 | matcher := NewPatternMatcher(detectorPatterns) 112 | for _, pattern := range custom { 113 | matcher.add(pattern) 114 | } 115 | return &PatternDetector{matcher} 116 | } 117 | -------------------------------------------------------------------------------- /detector/pattern/pattern_detector_test.go: -------------------------------------------------------------------------------- 1 | package pattern 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | "talisman/detector/helpers" 7 | "talisman/detector/severity" 8 | "talisman/gitrepo" 9 | "talisman/talismanrc" 10 | "testing" 11 | 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | var talismanRC = &talismanrc.TalismanRC{} 16 | var defaultIgnoreEvaluator = helpers.BuildIgnoreEvaluator("default", talismanRC, gitrepo.RepoLocatedAt(".")) 17 | var dummyCallback = func() {} 18 | 19 | var ( 20 | customPatterns []talismanrc.PatternString 21 | ) 22 | 23 | func ignoreEvaluatorWithTalismanRC(tRC *talismanrc.TalismanRC) helpers.IgnoreEvaluator { 24 | return helpers.BuildIgnoreEvaluator("default", tRC, gitrepo.RepoLocatedAt(".")) 25 | } 26 | 27 | func TestShouldDetectPasswordPatterns(t *testing.T) { 28 | filename := "secret.txt" 29 | values := [7]string{"password", "secret", "key", "pwd", "pass", "pword", "passphrase"} 30 | for i := 0; i < len(values); i++ { 31 | shouldPassDetectionOfSecretPattern(filename, []byte(strings.ToTitle(values[i])+":UnsafeString"), t) 32 | shouldPassDetectionOfSecretPattern(filename, []byte(values[i]+"=UnsafeString"), t) 33 | shouldPassDetectionOfSecretPattern(filename, []byte("."+values[i]+"=randomStringGoesHere}"), t) 34 | shouldPassDetectionOfSecretPattern(filename, []byte(":"+values[i]+" randomStringGoesHere"), t) 35 | shouldPassDetectionOfSecretPattern(filename, []byte(values[i]+" ,\"randomStringGoesHere\""), t) 36 | shouldPassDetectionOfSecretPattern(filename, []byte("'"+values[i]+"' ,\"randomStringGoesHere\""), t) 37 | shouldPassDetectionOfSecretPattern(filename, []byte("\""+values[i]+"\" ,\"randomStringGoesHere\""), t) 38 | shouldPassDetectionOfSecretPattern(filename, 39 | []byte("\"SERVER_"+strings.ToUpper(values[i])+"\" : UnsafeString"), 40 | t) 41 | shouldPassDetectionOfSecretPattern(filename, []byte(values[i]+"2-string : UnsafeString"), t) 42 | shouldPassDetectionOfSecretPattern(filename, 43 | []byte("<"+values[i]+" data=123> randomStringGoesHere "), 44 | t) 45 | shouldPassDetectionOfSecretPattern(filename, 46 | []byte(" randomStringGoesHere "), 47 | t) 48 | } 49 | 50 | shouldPassDetectionOfSecretPattern(filename, []byte("\"pw\" : UnsafeString"), t) 51 | shouldPassDetectionOfSecretPattern(filename, []byte("Pw=UnsafeString"), t) 52 | 53 | shouldPassDetectionOfSecretPattern(filename, []byte("alksjdhfkjaklsdhflk12345adskjf"), t) 54 | shouldPassDetectionOfSecretPattern(filename, []byte("AWS key :"), t) 55 | shouldPassDetectionOfSecretPattern(filename, []byte(`BEGIN RSA PRIVATE KEY----- 56 | aghjdjadslgjagsfjlsgjalsgjaghjldasja 57 | -----END RSA PRIVATE KEY`), t) 58 | shouldPassDetectionOfSecretPattern(filename, []byte(`PWD=appropriate`), t) 59 | shouldPassDetectionOfSecretPattern(filename, []byte(`pass=appropriate`), t) 60 | shouldPassDetectionOfSecretPattern(filename, []byte(`adminpwd=appropriate`), t) 61 | 62 | shouldFailDetectionOfSecretPattern(filename, []byte("\"pAsSWoRD\" :1234567"), t) 63 | shouldFailDetectionOfSecretPattern(filename, []byte(`setPassword("12345678")`), t) 64 | shouldFailDetectionOfSecretPattern(filename, []byte(`setenv(password,123456)`), t) 65 | shouldFailDetectionOfSecretPattern(filename, []byte(`random=12345678)`), t) 66 | } 67 | 68 | func TestShouldIgnorePasswordPatternsIfChecksumMatches(t *testing.T) { 69 | results := helpers.NewDetectionResults() 70 | content := []byte("\"password\" : UnsafePassword") 71 | filename := "secret.txt" 72 | additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} 73 | fileIgnoreConfig := talismanrc.FileIgnoreConfig{ 74 | FileName: filename, 75 | Checksum: "833b6c24c8c2c5c7e1663226dc401b29c005492dc76a1150fc0e0f07f29d4cc3", 76 | IgnoreDetectors: []string{"filecontent"}, 77 | AllowedPatterns: []string{}} 78 | ignores := &talismanrc.TalismanRC{FileIgnoreConfig: []talismanrc.FileIgnoreConfig{fileIgnoreConfig}} 79 | 80 | NewPatternDetector(customPatterns).Test(ignoreEvaluatorWithTalismanRC(ignores), additions, ignores, results, dummyCallback) 81 | 82 | assert.True(t, results.Successful(), "Expected file %s to be ignored because checksum matches", filename) 83 | } 84 | 85 | func TestShouldIgnoreAllowedPattern(t *testing.T) { 86 | results := helpers.NewDetectionResults() 87 | content := []byte("\"key\" : \"This is an allowed keyword\"\npassword=y0uw1lln3v3rgu3ssmyP@55w0rd") 88 | filename := "allowed_pattern.txt" 89 | additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} 90 | fileIgnoreConfig := talismanrc.FileIgnoreConfig{ 91 | FileName: filename, Checksum: "", 92 | IgnoreDetectors: []string{}, 93 | AllowedPatterns: []string{"key"}} 94 | ignores := &talismanrc.TalismanRC{ 95 | FileIgnoreConfig: []talismanrc.FileIgnoreConfig{fileIgnoreConfig}, 96 | AllowedPatterns: []*talismanrc.Pattern{{Regexp: regexp.MustCompile("password")}}} 97 | 98 | NewPatternDetector(customPatterns).Test(defaultIgnoreEvaluator, additions, ignores, results, dummyCallback) 99 | 100 | assert.True(t, 101 | results.Successful(), 102 | "Expected keywords %v %v to be ignored by Talisman", fileIgnoreConfig.AllowedPatterns, ignores.AllowedPatterns) 103 | } 104 | func TestShouldOnlyWarnSecretPatternIfBelowThreshold(t *testing.T) { 105 | results := helpers.NewDetectionResults() 106 | content := []byte(`password=UnsafeString`) 107 | filename := "secret.txt" 108 | additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} 109 | talismanRCWithThreshold := &talismanrc.TalismanRC{Threshold: severity.High} 110 | 111 | NewPatternDetector(customPatterns).Test(ignoreEvaluatorWithTalismanRC(talismanRCWithThreshold), additions, talismanRCWithThreshold, results, dummyCallback) 112 | 113 | assert.False(t, results.HasFailures(), "Expected file %s to not have failures", filename) 114 | assert.True(t, results.HasWarnings(), "Expected file %s to have warnings", filename) 115 | } 116 | 117 | func DetectionOfSecretPattern(filename string, content []byte) (*helpers.DetectionResults, []gitrepo.Addition, string) { 118 | results := helpers.NewDetectionResults() 119 | additions := []gitrepo.Addition{gitrepo.NewAddition(filename, content)} 120 | NewPatternDetector(customPatterns).Test(defaultIgnoreEvaluator, additions, talismanRC, results, dummyCallback) 121 | expected := "Potential secret pattern : " + string(content) 122 | return results, additions, expected 123 | } 124 | 125 | func shouldPassDetectionOfSecretPattern(filename string, content []byte, t *testing.T) { 126 | results, additions, expected := DetectionOfSecretPattern(filename, content) 127 | assert.Equal(t, expected, getFailureMessage(results, additions)) 128 | assert.Len(t, results.Results, 1) 129 | } 130 | 131 | func shouldFailDetectionOfSecretPattern(filename string, content []byte, t *testing.T) { 132 | results, additions, expected := DetectionOfSecretPattern(filename, content) 133 | assert.NotEqual(t, expected, getFailureMessage(results, additions)) 134 | assert.Len(t, results.Results, 0) 135 | } 136 | 137 | func getFailureMessage(results *helpers.DetectionResults, additions []gitrepo.Addition) string { 138 | failureMessages := []string{} 139 | for _, failureDetails := range results.GetFailures(additions[0].Path) { 140 | failureMessages = append(failureMessages, failureDetails.Message) 141 | } 142 | if len(failureMessages) == 0 { 143 | return "" 144 | } 145 | return failureMessages[0] 146 | } 147 | -------------------------------------------------------------------------------- /detector/severity/pattern_severity.go: -------------------------------------------------------------------------------- 1 | package severity 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | type PatternSeverity struct { 8 | Pattern *regexp.Regexp 9 | Severity Severity 10 | } 11 | -------------------------------------------------------------------------------- /detector/severity/severity.go: -------------------------------------------------------------------------------- 1 | package severity 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | var severityMap = map[Severity]string{ 9 | Low: "low", 10 | Medium: "medium", 11 | High: "high", 12 | } 13 | 14 | func String(severity Severity) string { 15 | return severityMap[severity] 16 | } 17 | 18 | func FromString(severity string) (Severity, error) { 19 | severityInLowerCase := strings.ToLower(severity) 20 | for k, v := range severityMap { 21 | if v == severityInLowerCase { 22 | return k, nil 23 | } 24 | } 25 | return 0, fmt.Errorf("unknown severity %v", severity) 26 | } 27 | 28 | type Severity int 29 | 30 | func (s Severity) String() string { 31 | return String(s) 32 | } 33 | 34 | func (s Severity) ExceedsThreshold(threshold Severity) bool { 35 | return s >= threshold 36 | } 37 | 38 | func (s *Severity) UnmarshalYAML(get func(interface{}) error) error { 39 | in := "" 40 | err := get(&in) 41 | if err != nil { 42 | return fmt.Errorf("Severity.Umarshal error: %v\n", err) 43 | } 44 | *s, err = FromString(in) 45 | return err 46 | } 47 | 48 | func (s Severity) MarshalYAML() (interface{}, error) { 49 | return String(s), nil 50 | } 51 | 52 | func (s *Severity) UnmarshalJSON(input []byte) error { 53 | v, err := FromString(string(input)) 54 | if err != nil { 55 | return err 56 | } 57 | *s = v 58 | return nil 59 | } 60 | 61 | func (s Severity) MarshalJSON() ([]byte, error) { 62 | bytes := []byte(fmt.Sprintf("\"%s\"", String(s))) 63 | return bytes, nil 64 | } 65 | 66 | const ( 67 | Low = Severity(iota + 1) 68 | Medium 69 | High 70 | ) 71 | -------------------------------------------------------------------------------- /detector/severity/severity_config.go: -------------------------------------------------------------------------------- 1 | package severity 2 | 3 | var SeverityConfiguration = map[string]Severity{ 4 | "ConsumerKeyPattern": High, 5 | "ConsumerSecretParrern": High, 6 | "AWSKeyPattern": High, 7 | "AWSSecretPattern": High, 8 | "RSAKeyPattern": High, 9 | "DSAFile": High, 10 | "PrivateKeyFile": High, 11 | "PemFile": High, 12 | "PpkFile": High, 13 | "SecretToken": High, 14 | "KeyPairFile": High, 15 | "CustomPattern": High, 16 | "PKCSFile": High, 17 | "PFXFile": High, 18 | "P12File": High, 19 | "NetrcFile": High, 20 | "RSAFile": High, 21 | "KeyChainFile": High, 22 | "KeyStoreFile": High, 23 | "OauthTokenFile": High, 24 | "HTPASSWDFile": High, 25 | "TunnelBlockFile": High, 26 | "CredentialsXML": High, 27 | "JenkinsPublishOverSSHFile": High, 28 | "Base64Content": High, 29 | "HexContent": High, 30 | "s3Config": Medium, 31 | "OpenVPNFile": Medium, 32 | "DatabaseYml": Medium, 33 | "ShellHistory": Low, 34 | "ASCFile": Low, 35 | "KDBFile": Low, 36 | "AgileKeyChainFile": Low, 37 | "PubXML": Low, 38 | "GitRobRC": Low, 39 | "ShellRC": Low, 40 | "CreditCardContent": Low, 41 | "ShellProfile": Low, 42 | "ShellAlias": Low, 43 | "OmniAuth": Low, 44 | "CarrierWaveRB": Low, 45 | "SchemaRB": Low, 46 | "PythonSettings": Low, 47 | "PhpConfig": Low, 48 | "PhpLocalSettings": Low, 49 | "EnvFile": Low, 50 | "BDumpFile": Low, 51 | "BSQLFile": Low, 52 | "PasswordFile": Low, 53 | "BackupFile": Low, 54 | "LogFile": Low, 55 | "KWallet": Low, 56 | "GNUCash": Low, 57 | "PasswordPhrasePattern": Low, 58 | "LargeFileSize": Low, 59 | } 60 | -------------------------------------------------------------------------------- /detector/severity/severity_test.go: -------------------------------------------------------------------------------- 1 | package severity 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestShouldReturnSeverityStringForDefinedSeverity(t *testing.T) { 10 | assert.Equal(t, String(Low), "low") 11 | assert.Equal(t, String(Medium), "medium") 12 | assert.Equal(t, String(High), "high") 13 | } 14 | func TestShouldReturnEmptyForInvalidSeverity(t *testing.T) { 15 | assert.Equal(t, String(10), "") 16 | } 17 | 18 | func TestShouldReturnSeverityForDefinedStrings(t *testing.T) { 19 | severityValue, _ := FromString("low") 20 | assert.Equal(t, severityValue, Low) 21 | severityValue, _ = FromString("mEDIum") 22 | assert.Equal(t, severityValue, Medium) 23 | severityValue, _ = FromString("HIGH") 24 | assert.Equal(t, severityValue, High) 25 | } 26 | 27 | func TestShouldReturnSeverityZeroWithErrorForUnknownStrings(t *testing.T) { 28 | severityValue, err := FromString("FakeSeverity") 29 | assert.Equal(t, Severity(0), severityValue) 30 | assert.Error(t, err) 31 | assert.Equal(t, "unknown severity FakeSeverity", err.Error()) 32 | } 33 | -------------------------------------------------------------------------------- /examples/schema-store-talismanrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$id": "https://github.com/thoughtworks/talisman/talismanrc", 4 | "title": "schema for .talismanrc", 5 | "type": "object", 6 | "additionalProperties": false, 7 | "fileMatch": [ 8 | ".talismanrc" 9 | ], 10 | "properties": { 11 | "fileignoreconfig": { 12 | "type": "array", 13 | "items": { 14 | "type": "object", 15 | "properties": { 16 | "filename": { 17 | "type": "string", 18 | "description": "Fully qualified filename" 19 | }, 20 | "checksum": { 21 | "type": "string", 22 | "description": "This field should always have the value specified by Talisman message" 23 | }, 24 | "ignore_detectors": { 25 | "type": "array", 26 | "description": "Disable specific detectors for a particular file", 27 | "items": { 28 | "type": "string", 29 | "enum": ["filecontent", "filename", "filesize"] 30 | } 31 | }, 32 | "allowed_patterns": { 33 | "type": "array", 34 | "description": "Keywords to ignore to reduce the number of false positives", 35 | "items": { 36 | "type": "string" 37 | } 38 | } 39 | }, 40 | "required": ["filename"] 41 | } 42 | }, 43 | "scopeconfig": { 44 | "type": "array", 45 | "description": "Talisman is configured to ignore certain files based on the specified scopes", 46 | "items": { 47 | "type": "object", 48 | "properties": { 49 | "scope": { 50 | "type": "string" 51 | } 52 | }, 53 | "required": ["scope"] 54 | } 55 | }, 56 | "allowed_patterns": { 57 | "type": "array", 58 | "description": "Keywords to ignore to reduce the number of false positives", 59 | "items": { 60 | "type": "string" 61 | } 62 | }, 63 | "custom_patterns": { 64 | "type": "array", 65 | "description": "You can specify custom regex patterns to look for in the current repository", 66 | "items": { 67 | "type": "string" 68 | } 69 | }, 70 | "custom_severities": { 71 | "type": "array", 72 | "description": "Custom detectors severities", 73 | "items": { 74 | "type": "object", 75 | "properties": { 76 | "detector": { 77 | "type": "string" 78 | }, 79 | "severity": { 80 | "type": "string", 81 | "enum": ["low", "medium", "high"] 82 | } 83 | }, 84 | "required": ["detector", "severity"] 85 | } 86 | }, 87 | "threshold": { 88 | "type": "string", 89 | "description": "Default minimal threshold", 90 | "enum": ["low", "medium", "high"] 91 | }, 92 | "version": { 93 | "type": "string", 94 | "description": ".talismanrc version" 95 | } 96 | }, 97 | "required": [] 98 | } 99 | -------------------------------------------------------------------------------- /git_testing/git_testing.go: -------------------------------------------------------------------------------- 1 | package git_testing 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "path/filepath" 8 | "strings" 9 | 10 | lorem "github.com/drhodes/golorem" 11 | "github.com/sirupsen/logrus" 12 | "github.com/spf13/afero" 13 | ) 14 | 15 | var Logger *logrus.Entry 16 | 17 | // GitTesting provides an API for manipulating a git repository during tests 18 | type GitTesting struct { 19 | root string 20 | } 21 | 22 | type GitOperation func(*GitTesting) 23 | 24 | // DoInTempGitRepo initializes a temporary git repository and executes the provided GitOperation in it 25 | func DoInTempGitRepo(gitOperation GitOperation) { 26 | gt := Init() 27 | defer gt.Clean() 28 | gitOperation(gt) 29 | } 30 | 31 | // Init creates a GitTesting based in a temporary directory 32 | func Init() *GitTesting { 33 | fs := afero.NewMemMapFs() 34 | path, _ := afero.TempDir(fs, afero.GetTempDir(fs, "talisman-test"), "") 35 | return initAt(path) 36 | } 37 | 38 | // initAt creates a GitTesting based at the specified path 39 | func initAt(gitRoot string) *GitTesting { 40 | os.MkdirAll(gitRoot, 0777) 41 | testingRepo := &GitTesting{gitRoot} 42 | output := testingRepo.execCommand("git", "init", ".") 43 | logrus.Debugf("Git init result %v", string(output)) 44 | testingRepo.execCommand("git", "config", "user.email", "talisman-test-user@example.com") 45 | testingRepo.execCommand("git", "config", "user.name", "Talisman Test User") 46 | testingRepo.execCommand("git", "config", "commit.gpgsign", "false") 47 | testingRepo.removeHooks() 48 | return testingRepo 49 | } 50 | 51 | // Clean removes the directory containing the git repository represented by a GitTesting 52 | func (git *GitTesting) Clean() { 53 | os.RemoveAll(git.root) 54 | } 55 | 56 | func (git *GitTesting) SetupBaselineFiles(filenames ...string) { 57 | Logger.Debugf("Creating %v in %s\n", filenames, git.root) 58 | for _, filename := range filenames { 59 | git.CreateFileWithContents(filename, lorem.Sentence(8, 10), lorem.Sentence(8, 10)) 60 | } 61 | git.AddAndcommit("*", "initial commit") 62 | } 63 | 64 | func (git *GitTesting) EarliestCommit() string { 65 | return git.execCommand("git", "rev-list", "--max-parents=0", "HEAD") 66 | } 67 | 68 | func (git *GitTesting) LatestCommit() string { 69 | return git.execCommand("git", "rev-parse", "HEAD") 70 | } 71 | 72 | func (git *GitTesting) CreateFileWithContents(filePath string, contents ...string) string { 73 | git.doInGitRoot(func() { 74 | os.MkdirAll(filepath.Dir(filePath), 0700) 75 | f, err := os.Create(filePath) 76 | git.die(fmt.Sprintf("when creating file %s", filePath), err) 77 | defer f.Close() 78 | for _, line := range contents { 79 | f.WriteString(line) 80 | } 81 | }) 82 | return filePath 83 | } 84 | 85 | func (git *GitTesting) OverwriteFileContent(filePath string, contents ...string) { 86 | git.doInGitRoot(func() { 87 | os.MkdirAll(filepath.Dir(filePath), 0770) 88 | f, err := os.Create(filePath) 89 | git.die(fmt.Sprintf("when overwriting file %s", filePath), err) 90 | defer f.Close() 91 | for _, line := range contents { 92 | f.WriteString(line) 93 | } 94 | f.Sync() 95 | }) 96 | } 97 | 98 | func (git *GitTesting) AppendFileContent(filePath string, contents ...string) { 99 | git.doInGitRoot(func() { 100 | os.MkdirAll(filepath.Dir(filePath), 0770) 101 | f, err := os.OpenFile(filePath, os.O_RDWR|os.O_APPEND, 0660) 102 | git.die(fmt.Sprintf("when appending file %s", filePath), err) 103 | defer f.Close() 104 | for _, line := range contents { 105 | f.WriteString(line) 106 | } 107 | f.Sync() 108 | }) 109 | } 110 | 111 | func (git *GitTesting) RemoveFile(filename string) { 112 | git.doInGitRoot(func() { 113 | os.Remove(filename) 114 | }) 115 | } 116 | 117 | func (git *GitTesting) FileContents(filePath string) []byte { 118 | var result []byte 119 | var err error 120 | git.doInGitRoot(func() { 121 | result, err = os.ReadFile(filePath) 122 | git.die(fmt.Sprintf("when reading file %s", filePath), err) 123 | }) 124 | return result 125 | } 126 | 127 | func (git *GitTesting) AddAndcommit(fileName string, message string) { 128 | git.Add(fileName) 129 | git.Commit(fileName, message) 130 | } 131 | 132 | func (git *GitTesting) Add(fileName string) { 133 | git.execCommand("git", "add", fileName) 134 | } 135 | 136 | func (git *GitTesting) Commit(fileName string, message string) { 137 | git.execCommand("git", "commit", "-m", message) 138 | } 139 | 140 | // GetBlobDetails returns git blob details for a path 141 | func (git *GitTesting) GetBlobDetails(fileName string) string { 142 | var output []byte 143 | objectHashAndFilename := "" 144 | git.doInGitRoot(func() { 145 | fmt.Println("hello") 146 | result := exec.Command("git", "rev-list", "--objects", "--all") 147 | output, _ = result.Output() 148 | objects := strings.Split(string(output), "\n") 149 | for _, object := range objects { 150 | objectDetails := strings.Split(object, " ") 151 | if len(objectDetails) == 2 && objectDetails[1] == fileName { 152 | objectHashAndFilename = object 153 | return 154 | } 155 | } 156 | }) 157 | return objectHashAndFilename 158 | } 159 | 160 | // execCommand executes a command with given arguments in the git repo directory 161 | func (git *GitTesting) execCommand(commandName string, args ...string) string { 162 | var output []byte 163 | git.doInGitRoot(func() { 164 | result := exec.Command(commandName, args...) 165 | var err error 166 | output, err = result.Output() 167 | summaryMessage := fmt.Sprintf("Command: %s %v\nWorkingDirectory: %s\nOutput %s\nError: %v", commandName, args, git.root, string(output), err) 168 | git.die(summaryMessage, err) 169 | Logger.Debug(summaryMessage) 170 | }) 171 | if len(output) > 0 { 172 | return strings.Trim(string(output), "\n") 173 | } 174 | return "" 175 | } 176 | 177 | func (git *GitTesting) die(msg string, err error) { 178 | if err != nil { 179 | Logger.Debugf(msg) 180 | panic(msg) 181 | } 182 | } 183 | 184 | func (git *GitTesting) doInGitRoot(operation func()) { 185 | wd, _ := os.Getwd() 186 | os.Chdir(git.root) 187 | defer func() { os.Chdir(wd) }() 188 | operation() 189 | } 190 | 191 | // Root returns the root directory of the git-testing repo 192 | func (git *GitTesting) Root() string { 193 | return git.root 194 | } 195 | 196 | // removeHooks removes all file-system hooks from git-test repo. 197 | // We do this to prevent any user-installed hooks from interfering with tests. 198 | func (git *GitTesting) removeHooks() { 199 | git.execCommand("rm", "-rf", ".git/hooks/") 200 | } 201 | -------------------------------------------------------------------------------- /git_testing/git_testing_test.go: -------------------------------------------------------------------------------- 1 | package git_testing 2 | 3 | import ( 4 | "os" 5 | "os/exec" 6 | "path" 7 | "path/filepath" 8 | "regexp" 9 | "strings" 10 | "testing" 11 | 12 | "github.com/sirupsen/logrus" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | var logger *logrus.Entry 17 | 18 | func init() { 19 | Logger = logrus.WithField("Environment", "Debug") 20 | Logger.Debug("GitTesting test started") 21 | logrus.SetOutput(os.Stderr) 22 | logger = Logger 23 | } 24 | 25 | func TestInitializingANewRepoSetsUpFolderAndGitStructures(t *testing.T) { 26 | DoInTempGitRepo(func(repo *GitTesting) { 27 | assert.True(t, exists(repo.root), "GitTesting initialization should create the directory structure required") 28 | assert.True(t, isGitRepo(repo.root), "Repo root does not contain the .git folder") 29 | }) 30 | } 31 | 32 | func TestSettingUpBaselineFilesSetsUpACommitInRepo(t *testing.T) { 33 | DoInTempGitRepo(func(repo *GitTesting) { 34 | repo.SetupBaselineFiles("a.txt", filepath.Join("alice", "bob", "b.txt")) 35 | verifyPresenceOfGitRepoWithCommits(t, 1, repo.root) 36 | }) 37 | } 38 | 39 | func TestEditingFilesInARepoWorks(t *testing.T) { 40 | DoInTempGitRepo(func(repo *GitTesting) { 41 | repo.SetupBaselineFiles("a.txt", filepath.Join("alice", "bob", "b.txt")) 42 | repo.AppendFileContent("a.txt", "\nmonkey see.\n", "monkey do.") 43 | content := repo.FileContents("a.txt") 44 | assert.True(t, strings.HasSuffix(string(content), "monkey see.\nmonkey do.")) 45 | repo.AddAndcommit("a.txt", "modified content") 46 | verifyPresenceOfGitRepoWithCommits(t, 2, repo.root) 47 | }) 48 | } 49 | 50 | func TestRemovingFilesInARepoWorks(t *testing.T) { 51 | DoInTempGitRepo(func(repo *GitTesting) { 52 | repo.SetupBaselineFiles("a.txt", filepath.Join("alice", "bob", "b.txt")) 53 | repo.RemoveFile("a.txt") 54 | assert.False(t, exists(filepath.Join("data", "testLocation1", "a.txt")), "Unexpected. Deleted file a.txt still exists inside the repo") 55 | repo.AddAndcommit("a.txt", "removed it") 56 | verifyPresenceOfGitRepoWithCommits(t, 2, repo.root) 57 | }) 58 | } 59 | 60 | func TestEarliestCommits(t *testing.T) { 61 | DoInTempGitRepo(func(repo *GitTesting) { 62 | repo.SetupBaselineFiles("a.txt") 63 | initialCommit := repo.EarliestCommit() 64 | repo.AppendFileContent("a.txt", "\nmonkey see.\n", "monkey do.") 65 | repo.AddAndcommit("a.txt", "modified content") 66 | assert.Equal(t, initialCommit, repo.EarliestCommit(), "First commit is not expected to change on repo modifications") 67 | }) 68 | } 69 | 70 | func TestLatestCommits(t *testing.T) { 71 | DoInTempGitRepo(func(repo *GitTesting) { 72 | repo.SetupBaselineFiles("a.txt") 73 | repo.AppendFileContent("a.txt", "\nmonkey see.\n", "monkey do.") 74 | repo.AddAndcommit("a.txt", "modified content") 75 | repo.AppendFileContent("a.txt", "\nline n-1.\n", "line n.") 76 | repo.AddAndcommit("a.txt", "more modified content") 77 | assert.NotEqual(t, repo.EarliestCommit(), repo.LatestCommit()) //bad test. 78 | }) 79 | } 80 | 81 | func verifyPresenceOfGitRepoWithCommits(t *testing.T, expectedCommitCount int, repoLocation string) { 82 | wd, _ := os.Getwd() 83 | os.Chdir(repoLocation) 84 | defer func() { os.Chdir(wd) }() 85 | 86 | cmd := exec.Command("git", "log", "--pretty=short") 87 | o, err := cmd.CombinedOutput() 88 | dieOnError(err) 89 | matches := regExp("(?m)^commit\\s[a-z0-9]+\\s+.*$").FindAllString(string(o), -1) 90 | assert.Len(t, matches, expectedCommitCount, "Repo root does not contain exactly %d commits.", expectedCommitCount) 91 | } 92 | 93 | func regExp(pattern string) *regexp.Regexp { 94 | return regexp.MustCompile(pattern) 95 | } 96 | 97 | func isGitRepo(loc string) bool { 98 | return exists(path.Join(loc, ".git")) 99 | } 100 | 101 | func exists(path string) bool { 102 | _, err := os.Stat(path) 103 | if (err != nil) && (os.IsNotExist(err)) { 104 | return false 105 | } else if err != nil { 106 | dieOnError(err) 107 | return true 108 | } else { 109 | return true 110 | } 111 | } 112 | 113 | func dieOnError(err error) { 114 | if err != nil { 115 | panic(err) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /gitrepo/constants.go: -------------------------------------------------------------------------------- 1 | package gitrepo 2 | 3 | const ( 4 | //GIT_STAGED_PREFIX is used to read a staged git file 5 | GIT_STAGED_PREFIX = "" 6 | 7 | //GIT_STAGED_PREFIX is used to read a commited git file 8 | GIT_HEAD_PREFIX = "HEAD" 9 | ) 10 | -------------------------------------------------------------------------------- /gitrepo/git_readers.go: -------------------------------------------------------------------------------- 1 | package gitrepo 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "os/exec" 8 | "strconv" 9 | 10 | "github.com/sirupsen/logrus" 11 | ) 12 | 13 | type ReadFunc func(string) ([]byte, error) 14 | type BatchReader interface { 15 | Start() error 16 | Read(string) ([]byte, error) 17 | Shutdown() error 18 | } 19 | 20 | type BatchGitObjectReader struct { 21 | repo *GitRepo 22 | cmd *exec.Cmd 23 | inputWriter *bufio.Writer 24 | outputReader *bufio.Reader 25 | read ReadFunc 26 | } 27 | 28 | func (bgor *BatchGitObjectReader) Start() error { 29 | return bgor.cmd.Start() 30 | } 31 | 32 | func (bgor *BatchGitObjectReader) Shutdown() error { 33 | return bgor.cmd.Process.Kill() 34 | } 35 | 36 | func (bgor *BatchGitObjectReader) Read(expr string) ([]byte, error) { 37 | return bgor.read(expr) 38 | } 39 | 40 | func newBatchGitObjectReader(root string) *BatchGitObjectReader { 41 | repo := &GitRepo{root} 42 | cmd := repo.makeRepoCommand("git", "cat-file", "--batch=%(objectsize)") 43 | inputPipe, err := cmd.StdinPipe() 44 | if err != nil { 45 | logrus.Fatalf("error creating stdin pipe for batch git file reader subprocess: %v", err) 46 | } 47 | outputPipe, err := cmd.StdoutPipe() 48 | if err != nil { 49 | logrus.Fatalf("error creating stdout pipe for batch git file reader subprocess: %v", err) 50 | } 51 | batchReader := BatchGitObjectReader{ 52 | repo: repo, 53 | cmd: cmd, 54 | inputWriter: bufio.NewWriter(inputPipe), 55 | outputReader: bufio.NewReader(outputPipe), 56 | } 57 | return &batchReader 58 | } 59 | 60 | func NewBatchGitHeadPathReader(root string) BatchReader { 61 | bgor := newBatchGitObjectReader(root) 62 | bgor.read = bgor.makePathReader(GIT_HEAD_PREFIX) 63 | return bgor 64 | } 65 | 66 | func NewBatchGitStagedPathReader(root string) BatchReader { 67 | bgor := newBatchGitObjectReader(root) 68 | bgor.read = bgor.makePathReader(GIT_STAGED_PREFIX) 69 | return bgor 70 | } 71 | 72 | func NewBatchGitObjectHashReader(root string) BatchReader { 73 | bgor := newBatchGitObjectReader(root) 74 | bgor.read = bgor.makeObjectHashReader() 75 | return bgor 76 | } 77 | 78 | type gitCatFileReadResult struct { 79 | contents []byte 80 | err error 81 | } 82 | 83 | func (bgor *BatchGitObjectReader) makePathReader(prefix string) ReadFunc { 84 | pathChan := make(chan ([]byte)) 85 | resultsChan := make(chan (gitCatFileReadResult)) 86 | go bgor.doCatFile(pathChan, resultsChan) 87 | return func(path string) ([]byte, error) { 88 | pathChan <- []byte(prefix + ":" + path) 89 | result := <-resultsChan 90 | return result.contents, result.err 91 | } 92 | } 93 | 94 | func (bgor *BatchGitObjectReader) makeObjectHashReader() ReadFunc { 95 | objectHashChan := make(chan ([]byte)) 96 | resultsChan := make(chan (gitCatFileReadResult)) 97 | go bgor.doCatFile(objectHashChan, resultsChan) 98 | return func(objectHash string) ([]byte, error) { 99 | objectHashChan <- []byte(objectHash) 100 | result := <-resultsChan 101 | return result.contents, result.err 102 | } 103 | } 104 | 105 | func (bgor *BatchGitObjectReader) doCatFile(gitExpressionChan chan ([]byte), resultsChan chan (gitCatFileReadResult)) { 106 | for { 107 | gitExpression := <-gitExpressionChan 108 | gitExpression = append(gitExpression, '\n') 109 | //Write file-path expression to process input 110 | bgor.inputWriter.Write(gitExpression) 111 | bgor.inputWriter.Flush() 112 | 113 | //Read line containing filesize from process output 114 | filesizeBytes, err := bgor.outputReader.ReadBytes('\n') 115 | if err != nil { 116 | logrus.Errorf("error reading filesize: %v", err) 117 | resultsChan <- gitCatFileReadResult{[]byte{}, err} 118 | continue 119 | } 120 | filesize, err := strconv.Atoi(string(filesizeBytes[:len(filesizeBytes)-1])) 121 | if err != nil { 122 | logrus.Errorf("error parsing filesize: %v", err) 123 | resultsChan <- gitCatFileReadResult{[]byte{}, err} 124 | continue 125 | } 126 | logrus.Debugf("Git Batch Reader: FilePath: %v, Size:%v", gitExpression, filesize) 127 | 128 | //Read file contents upto filesize bytes from process output 129 | fileBytes := make([]byte, filesize) 130 | n, err := io.ReadFull(bgor.outputReader, fileBytes) 131 | if n != filesize || err != nil { 132 | logrus.Errorf("error reading exactly %v bytes of %v: %v (read %v bytes)", filesize, gitExpression, err, n) 133 | resultsChan <- gitCatFileReadResult{[]byte{}, err} 134 | continue 135 | } 136 | 137 | //Read and discard trailing newline (not doing this will cause errors going forward) 138 | b, err := bgor.outputReader.ReadByte() 139 | if err != nil || b != '\n' { 140 | resultsChan <- gitCatFileReadResult{[]byte{}, fmt.Errorf("error discarding trailing newline : %v trailing byte: %v", err, b)} 141 | continue 142 | } 143 | 144 | resultsChan <- gitCatFileReadResult{fileBytes, nil} 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /gitrepo/pixel.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thoughtworks/talisman/12fab7055d7e640d5dae43209bafc48bf5ef1fd7/gitrepo/pixel.jpg -------------------------------------------------------------------------------- /global_install_scripts/add_to_talismanignore.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | shopt -s extglob 4 | 5 | DEBUG=${DEBUG:-''} 6 | 7 | function run() { 8 | function echo_debug() { 9 | [[ -z "${DEBUG}" ]] && return 10 | echo -ne $(tput setaf 3) >&2 11 | echo "$1" >&2 12 | echo -ne $(tput sgr0) >&2 13 | } 14 | export -f echo_debug 15 | 16 | INSTALL_ORG_REPO=${INSTALL_ORG_REPO:-'thoughtworks/talisman'} 17 | SCRIPT_BASE="https://raw.githubusercontent.com/${INSTALL_ORG_REPO}/master/global_install_scripts" 18 | 19 | TEMP_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'talisman_uninstall') 20 | trap "rm -r ${TEMP_DIR}" EXIT 21 | chmod 0700 ${TEMP_DIR} 22 | 23 | ADD_AN_IGNORE_SCRIPT=${TEMP_DIR}/add_to_talismanignore_in_git_repo.bash 24 | 25 | function get_dependent_scripts() { 26 | echo_debug "getting ${SCRIPT_BASE}/add_to_talismanignore_in_git_repo.bash via curl" 27 | curl --silent "${SCRIPT_BASE}/add_to_talismanignore_in_git_repo.bash" >${ADD_AN_IGNORE_SCRIPT} 28 | chmod +x ${ADD_AN_IGNORE_SCRIPT} 29 | } 30 | 31 | get_dependent_scripts 32 | 33 | echo "Adding pattern to .talismanignore recursively in git repos" 34 | read -p "Please enter pattern to add to .talismanignore (enter to abort): " IGNORE_PATTERN 35 | [[ -n $IGNORE_PATTERN ]] || exit 1 36 | 37 | read -e -p "Please enter root directory to search for git repos (Default: ${HOME}): " SEARCH_ROOT 38 | SEARCH_ROOT=${SEARCH_ROOT:-$HOME} 39 | SEARCH_CMD="find" 40 | EXTRA_SEARCH_OPTS="" 41 | echo -e "\tSearching ${SEARCH_ROOT} for git repositories" 42 | 43 | SUDO_PREFIX="" 44 | if [[ "${SEARCH_ROOT}" == "/" ]]; then 45 | echo -e "\tPlease enter your password when prompted to enable script to search as root user:" 46 | SUDO_PREFIX="sudo" 47 | EXTRA_SEARCH_OPTS="-xdev \( -path '/private/var' -prune \) -o" 48 | fi 49 | 50 | CMD_STRING="${SUDO_PREFIX} ${SEARCH_CMD} ${SEARCH_ROOT} ${EXTRA_SEARCH_OPTS} -name .git -type d -exec ${ADD_AN_IGNORE_SCRIPT} {} ${IGNORE_PATTERN} \;" 51 | echo_debug "EXECUTING: ${CMD_STRING}" 52 | eval "${CMD_STRING}" 53 | } 54 | 55 | run $0 $@ 56 | -------------------------------------------------------------------------------- /global_install_scripts/add_to_talismanignore_in_git_repo.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | GIT_REPO_DOT_GIT=$1 5 | IGNORE_PATTERN=$2 6 | 7 | function echo_debug() { 8 | [[ -z "${DEBUG}" ]] && return 9 | echo -ne $(tput setaf 3) >&2 10 | echo "$1" >&2 11 | echo -ne $(tput sgr0) >&2 12 | } 13 | 14 | function echo_success { 15 | echo -ne $(tput setaf 2) 16 | echo "$1" >&2 17 | echo -ne $(tput sgr0) 18 | } 19 | 20 | echo_debug "Adding $IGNORE_PATTERN to ${GIT_REPO_DOT_GIT}/../.talismanignore" 21 | echo $IGNORE_PATTERN >>${GIT_REPO_DOT_GIT}/../.talismanignore && echo_success "Added $IGNORE_PATTERN to ${GIT_REPO_DOT_GIT}/../.talismanignore" 22 | -------------------------------------------------------------------------------- /global_install_scripts/remove_hooks.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | shopt -s extglob 4 | 5 | DEBUG=${DEBUG:-''} 6 | 7 | declare HOOK_SCRIPT='pre-commit' # TODO: need ability to uninstall pre-push hook as well. 8 | if [[ $# -gt 0 && $1 =~ pre-push.* ]]; then 9 | HOOK_SCRIPT='pre-push' 10 | fi 11 | 12 | function run() { 13 | # Arguments: $1 = 'pre-commit' or 'pre-push'. whether to set talisman up as pre-commit or pre-push hook : TODO: not implemented yet 14 | # Environment variables: 15 | # DEBUG="any-non-emply-value": verbose output for debugging the script 16 | # INSTALL_ORG_REPO="..." : the github org/repo to install from (default thoughtworks/talisman) 17 | # 18 | # Download the script needed for uninstalling the repo level hooks 19 | # For each git repo found in the search root (default $HOME) 20 | # Run the repo level uninstall hook. This will remove the symlink from .git/hooks/pre- to the central $TALISMAN_SETUP_DIR 21 | # The script will only remove talisman hook, not a pre-commit.com script or some other non-talisman hook 22 | # Write exceptions to a file for manual action 23 | # Look in the uninstall_git_repo_hook.bash script for more details on what it does 24 | # Remove the talisman_hook_script in .git-template/hooks/pre- 25 | # Remove talisman binary and talisman_hook_script from $TALISMAN_SETUP_DIR ($HOME/.talisman/bin) 26 | 27 | function echo_error() { 28 | echo -ne $(tput setaf 1) >&2 29 | echo "$1" >&2 30 | echo -ne $(tput sgr0) >&2 31 | } 32 | export -f echo_error 33 | 34 | function echo_debug() { 35 | [[ -z "${DEBUG}" ]] && return 36 | echo -ne $(tput setaf 3) >&2 37 | echo "$1" >&2 38 | echo -ne $(tput sgr0) >&2 39 | } 40 | export -f echo_debug 41 | 42 | function echo_success { 43 | echo -ne $(tput setaf 2) 44 | echo "$1" >&2 45 | echo -ne $(tput sgr0) 46 | } 47 | export -f echo_success 48 | 49 | TALISMAN_SETUP_DIR="/usr/local/Cellar/talisman" 50 | TALISMAN_HOOK_SCRIPT_DIR=${HOME}/".talisman" 51 | TEMPLATE_DIR=$(git config --global init.templatedir) || true 52 | INSTALL_ORG_REPO=${INSTALL_ORG_REPO:-'thoughtworks/talisman'} 53 | SCRIPT_BASE="https://raw.githubusercontent.com/${INSTALL_ORG_REPO}/master/global_install_scripts" 54 | 55 | TEMP_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'talisman_uninstall') 56 | trap "rm -r ${TEMP_DIR}" EXIT 57 | chmod 0700 ${TEMP_DIR} 58 | 59 | DELETE_REPO_HOOK_SCRIPT=${TEMP_DIR}/uninstall_git_repo_hook.bash 60 | function get_dependent_scripts() { 61 | curl --silent "${SCRIPT_BASE}/uninstall_git_repo_hook.bash" >${DELETE_REPO_HOOK_SCRIPT} 62 | chmod +x ${DELETE_REPO_HOOK_SCRIPT} 63 | } 64 | 65 | function remove_git_talisman_hooks() { 66 | if [[ ! -x ${DELETE_REPO_HOOK_SCRIPT} ]]; then 67 | echo_error "Couldn't find executable script ${DELETE_REPO_HOOK_SCRIPT}" 68 | exit 1 69 | fi 70 | 71 | echo "Removing talisman hooks recursively in git repos" 72 | read -e -p "Please enter root directory to search for git repos (Default: ${HOME}): " SEARCH_ROOT 73 | SEARCH_ROOT=${SEARCH_ROOT:-$HOME} 74 | SEARCH_CMD="find" 75 | EXTRA_SEARCH_OPTS="" 76 | echo -e "\tSearching ${SEARCH_ROOT} for git repositories" 77 | 78 | SUDO_PREFIX="" 79 | if [[ "${SEARCH_ROOT}" == "/" ]]; then 80 | echo -e "\tPlease enter your password when prompted to enable script to search as root user:" 81 | SUDO_PREFIX="sudo" 82 | EXTRA_SEARCH_OPTS="-xdev \( -path '/private/var' -prune \) -o" 83 | fi 84 | EXCEPTIONS_FILE=${TEMP_DIR}/repos_with_multiple_hooks.paths 85 | touch ${EXCEPTIONS_FILE} 86 | 87 | TALISMAN_HOOK_SCRIPT_PATH=${TALISMAN_HOOK_SCRIPT_DIR}/talisman_hook_script 88 | CMD_STRING="${SUDO_PREFIX} ${SEARCH_CMD} ${SEARCH_ROOT} ${EXTRA_SEARCH_OPTS} -name .git -type d -exec ${DELETE_REPO_HOOK_SCRIPT} ${TALISMAN_HOOK_SCRIPT_PATH} ${EXCEPTIONS_FILE} {} ${HOOK_SCRIPT} \;" 89 | echo_debug "EXECUTING: ${CMD_STRING}" 90 | eval "${CMD_STRING}" || true 91 | 92 | NUMBER_OF_EXCEPTION_REPOS=$(cat ${EXCEPTIONS_FILE} | wc -l) 93 | 94 | if [ ${NUMBER_OF_EXCEPTION_REPOS} -gt 0 ]; then 95 | EXCEPTIONS_FILE_HOME_PATH="${HOME}/repos_to_remove_talisman_from.paths" 96 | mv ${EXCEPTIONS_FILE} ${EXCEPTIONS_FILE_HOME_PATH} 97 | echo_error "" 98 | echo_error "Please see ${EXCEPTIONS_FILE_HOME_PATH} for a list of repositories" 99 | echo_error "that talisman couldn't be automatically removed from" 100 | echo_error "This is likely because these repos are using pre-commit (https://pre-commit.com)" 101 | echo_error "Remove lines related to talisman from the .pre-commit-config.yaml manually" 102 | fi 103 | } 104 | 105 | get_dependent_scripts 106 | remove_git_talisman_hooks 107 | 108 | echo_debug "Removing talisman hooks from .git-template" 109 | echo_debug "${TEMPLATE_DIR}/hooks/${HOOK_SCRIPT}" 110 | if [[ -n $TEMPLATE_DIR && -e ${TEMPLATE_DIR}/hooks/${HOOK_SCRIPT} && 111 | ${TALISMAN_HOOK_SCRIPT_DIR}/talisman_hook_script -ef ${TEMPLATE_DIR}/hooks/${HOOK_SCRIPT} ]]; then 112 | rm -f "${TEMPLATE_DIR}/hooks/${HOOK_SCRIPT}" && 113 | echo_success "Removed ${HOOK_SCRIPT} from ${TEMPLATE_DIR}" 114 | fi 115 | 116 | echo_debug "Removing talisman hook script from ${TALISMAN_HOOK_SCRIPT_DIR}" 117 | rm -rf $TALISMAN_HOOK_SCRIPT_DIR && 118 | echo_success "Removed talisman hook script from ${TALISMAN_HOOK_SCRIPT_DIR}" 119 | 120 | if [ -n "${TALISMAN_HOME:-}" ]; then 121 | echo_error "Pleasee run \"brew uninstall talisman\" to remove the talisman binary" 122 | echo "Please remember to remove TALISMAN_HOME from your environment variables" 123 | fi 124 | } 125 | 126 | run $0 $@ 127 | -------------------------------------------------------------------------------- /global_install_scripts/setup_talisman_hook_in_repo.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | IFS=$'\n' 5 | function run() { 6 | 7 | TALISMAN_HOOK_SCRIPT_PATH=$1 8 | EXCEPTIONS_FILE=$2 9 | DOT_GIT_DIR=$3 10 | HOOK_SCRIPT=$4 11 | 12 | function echo_error() { 13 | echo -ne $(tput setaf 1) >&2 14 | echo "$1" >&2 15 | echo -ne $(tput sgr0) >&2 16 | } 17 | 18 | function echo_debug() { 19 | [[ -z "${DEBUG}" ]] && return 20 | echo -ne $(tput setaf 3) >&2 21 | echo "$1" >&2 22 | echo -ne $(tput sgr0) >&2 23 | } 24 | 25 | function echo_success { 26 | echo -ne $(tput setaf 2) 27 | echo "$1" >&2 28 | echo -ne $(tput sgr0) 29 | } 30 | 31 | REPO_HOOK_SCRIPT=${DOT_GIT_DIR}/hooks/${HOOK_SCRIPT} 32 | #check if a hook already exists 33 | if [ -e "${REPO_HOOK_SCRIPT}" ]; then 34 | #check if already hooked up to talisman 35 | if [ "${REPO_HOOK_SCRIPT}" -ef "${TALISMAN_HOOK_SCRIPT_PATH}" ]; then 36 | echo_success "Talisman already setup in ${REPO_HOOK_SCRIPT}" 37 | else 38 | if [ -e "${DOT_GIT_DIR}/../.pre-commit-config.yaml" ]; then 39 | echo_error "Pre-existing pre-commit.com hook detected in ${DOT_GIT_DIR}/hooks" 40 | fi 41 | echo ${DOT_GIT_DIR} | sed 's#/.git$##' >>${EXCEPTIONS_FILE} 42 | fi 43 | else 44 | echo "Setting up ${HOOK_SCRIPT} hook in ${DOT_GIT_DIR}/hooks" 45 | mkdir -p ${DOT_GIT_DIR}/hooks || (echo_error "Could not create hooks directory" && return) 46 | LN_FLAGS="-sf" 47 | [ -n "true" ] && LN_FLAGS="${LN_FLAGS}v" 48 | OS=$(uname -s) 49 | case $OS in 50 | "MINGW32_NT-10.0-WOW" | "MINGW64_NT-10.0") 51 | DOT_GIT_DIR_WIN=$(sed -e 's/\/\([a-z]\)\//\1:\\/' -e 's/\//\\/g' <<<"$DOT_GIT_DIR") 52 | TALISMAN_HOOK_SCRIPT_PATH_WIN=$(sed -e 's/\/\([a-z]\)\//\1:\\/' -e 's/\//\\/g' <<<"$TALISMAN_HOOK_SCRIPT_PATH") 53 | cmd <<<"mklink /H "${DOT_GIT_DIR_WIN}\\hooks\\${HOOK_SCRIPT}" "${TALISMAN_HOOK_SCRIPT_PATH_WIN}"" >/dev/null 54 | ;; 55 | *) 56 | ln ${LN_FLAGS} ${TALISMAN_HOOK_SCRIPT_PATH} ${DOT_GIT_DIR}/hooks/${HOOK_SCRIPT} 57 | ;; 58 | esac 59 | 60 | echo_success "DONE" 61 | fi 62 | } 63 | 64 | run $@ 65 | -------------------------------------------------------------------------------- /global_install_scripts/talisman_hook_script.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | shopt -s extglob 3 | 4 | # set TALISMAN_DEBUG="some-non-empty-value" in the env to get verbose output when the hook or talisman is running 5 | function echo_debug() { 6 | MSG="$@" 7 | [[ -n "${TALISMAN_DEBUG}" ]] && echo "${MSG}" 8 | } 9 | 10 | function echo_warning() { 11 | echo -ne $(tput setaf 3) >&2 12 | echo "$1" >&2 13 | echo -ne $(tput sgr0) >&2 14 | } 15 | 16 | function echo_error() { 17 | echo -ne $(tput setaf 1) >&2 18 | echo "$1" >&2 19 | echo -ne $(tput sgr0) >&2 20 | } 21 | 22 | function echo_success() { 23 | echo -ne $(tput setaf 2) 24 | echo "$1" >&2 25 | echo -ne $(tput sgr0) 26 | } 27 | 28 | function toLower() { 29 | echo "$1" | awk '{print tolower($0)}' 30 | } 31 | 32 | declare HOOKNAME="pre-commit" 33 | NAME=$(basename $0) 34 | ORG_REPO=${ORG_REPO:-'thoughtworks/talisman'} 35 | 36 | # given the various symlinks, this script may be invoked as 37 | # 'pre-commit', 'pre-push', 'talisman_hook_script pre-commit' or 'talisman_hook_script pre-push' 38 | case "$NAME" in 39 | pre-commit* | pre-push*) HOOKNAME="${NAME}" ;; 40 | talisman_hook_script) 41 | if [[ $# -gt 0 && $1 =~ pre-push.* ]]; then 42 | HOOKNAME="pre-push" 43 | fi 44 | ;; 45 | *) 46 | echo "Unexpected invocation. Please check invocation name and parameters" 47 | exit 1 48 | ;; 49 | esac 50 | 51 | TALISMAN_UPGRADE_CONNECT_TIMEOUT=${TALISMAN_UPGRADE_CONNECT_TIMEOUT:-10} 52 | function check_and_upgrade_talisman_binary() { 53 | if [[ -n "${TALISMAN_HOME:-}" && "$TALISMAN_SKIP_UPGRADE" != "true" ]]; then 54 | LATEST_VERSION=$(curl --connect-timeout $TALISMAN_UPGRADE_CONNECT_TIMEOUT -Is https://github.com/${ORG_REPO}/releases/latest | grep -iE "^location:" | grep -o '[^/]\+$' | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+') 55 | CURRENT_VERSION=$(${TALISMAN_BINARY} --version | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+') 56 | if [ -z "$LATEST_VERSION" ]; then 57 | echo_warning "Failed to retrieve latest Talisman version, skipping update." 58 | elif [ "$LATEST_VERSION" != "$CURRENT_VERSION" ]; then 59 | echo "" 60 | echo_warning "Your version of Talisman is outdated. Updating Talisman to v${LATEST_VERSION}" 61 | curl --silent https://raw.githubusercontent.com/${ORG_REPO}/master/global_install_scripts/update_talisman.bash >/tmp/update_talisman.bash && /bin/bash /tmp/update_talisman.bash 62 | else 63 | echo_debug "Talisman version up-to-date, skipping update" 64 | fi 65 | fi 66 | } 67 | 68 | check_and_upgrade_talisman_binary 69 | # Here HOOKNAME should be either 'pre-commit' (default) or 'pre-push' 70 | echo_debug "Firing ${HOOKNAME} hook" 71 | 72 | # Don't run talisman checks in a git repo, if we find a .talisman_skip or .talisman_skip.pre- file in the repo 73 | if [[ -f .talisman_skip || -f .talisman_skip.${HOOKNAME} ]]; then 74 | echo_debug "Found skip file. Not performing checks" 75 | exit 0 76 | fi 77 | 78 | TALISMAN_DEBUG="$(toLower "${TALISMAN_DEBUG}")" 79 | DEBUG_OPTS="" 80 | [[ "${TALISMAN_DEBUG}" == "true" ]] && DEBUG_OPTS="-d" 81 | 82 | TALISMAN_INTERACTIVE="$(toLower "${TALISMAN_INTERACTIVE}")" 83 | INTERACTIVE="" 84 | if [ "${TALISMAN_INTERACTIVE}" == "true" ]; then 85 | INTERACTIVE="-i" 86 | [[ "${HOOKNAME}" == "pre-commit" ]] && exec to the central $TALISMAN_SETUP_DIR 21 | # The script will only remove talisman hook, not a pre-commit.com script or some other non-talisman hook 22 | # Write exceptions to a file for manual action 23 | # Look in the uninstall_git_repo_hook.bash script for more details on what it does 24 | # Remove the talisman_hook_script in .git-template/hooks/pre- 25 | # Remove talisman binary and talisman_hook_script from $TALISMAN_SETUP_DIR ($HOME/.talisman/bin) 26 | 27 | function echo_error() { 28 | echo -ne $(tput setaf 1) >&2 29 | echo "$1" >&2 30 | echo -ne $(tput sgr0) >&2 31 | } 32 | export -f echo_error 33 | 34 | function echo_debug() { 35 | [[ -z "${DEBUG}" ]] && return 36 | echo -ne $(tput setaf 3) >&2 37 | echo "$1" >&2 38 | echo -ne $(tput sgr0) >&2 39 | } 40 | export -f echo_debug 41 | 42 | function echo_success() { 43 | echo -ne $(tput setaf 2) 44 | echo "$1" >&2 45 | echo -ne $(tput sgr0) 46 | } 47 | export -f echo_success 48 | 49 | TALISMAN_SETUP_DIR=${HOME}/.talisman/bin 50 | TEMPLATE_DIR=$(git config --global init.templatedir) || true 51 | INSTALL_ORG_REPO=${INSTALL_ORG_REPO:-'thoughtworks/talisman'} 52 | SCRIPT_BASE="https://raw.githubusercontent.com/${INSTALL_ORG_REPO}/master/global_install_scripts" 53 | 54 | TEMP_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'talisman_uninstall') 55 | trap "rm -r ${TEMP_DIR}" EXIT 56 | chmod 0700 ${TEMP_DIR} 57 | 58 | DELETE_REPO_HOOK_SCRIPT=${TEMP_DIR}/uninstall_git_repo_hook.bash 59 | function get_dependent_scripts() { 60 | curl --silent "${SCRIPT_BASE}/uninstall_git_repo_hook.bash" >${DELETE_REPO_HOOK_SCRIPT} 61 | chmod +x ${DELETE_REPO_HOOK_SCRIPT} 62 | } 63 | 64 | function remove_git_talisman_hooks() { 65 | if [[ ! -x ${DELETE_REPO_HOOK_SCRIPT} ]]; then 66 | echo_error "Couldn't find executable script ${DELETE_REPO_HOOK_SCRIPT}" 67 | exit 1 68 | fi 69 | 70 | echo "Removing talisman hooks recursively in git repos" 71 | read -e -p "Please enter root directory to search for git repos (Default: ${HOME}): " SEARCH_ROOT 72 | SEARCH_ROOT=${SEARCH_ROOT:-$HOME} 73 | SEARCH_CMD="find" 74 | EXTRA_SEARCH_OPTS="" 75 | echo -e "\tSearching ${SEARCH_ROOT} for git repositories" 76 | 77 | SUDO_PREFIX="" 78 | if [[ "${SEARCH_ROOT}" == "/" ]]; then 79 | echo -e "\tPlease enter your password when prompted to enable script to search as root user:" 80 | SUDO_PREFIX="sudo" 81 | EXTRA_SEARCH_OPTS="-xdev \( -path '/private/var' -prune \) -o" 82 | fi 83 | EXCEPTIONS_FILE=${TEMP_DIR}/repos_with_multiple_hooks.paths 84 | touch ${EXCEPTIONS_FILE} 85 | 86 | TALISMAN_PATH=${TALISMAN_SETUP_DIR}/talisman_hook_script 87 | CMD_STRING="${SUDO_PREFIX} ${SEARCH_CMD} ${SEARCH_ROOT} ${EXTRA_SEARCH_OPTS} -name .git -type d -exec ${DELETE_REPO_HOOK_SCRIPT} ${TALISMAN_PATH} ${EXCEPTIONS_FILE} {} ${HOOK_SCRIPT} \;" 88 | echo_debug "EXECUTING: ${CMD_STRING}" 89 | eval "${CMD_STRING}" || true 90 | 91 | NUMBER_OF_EXCEPTION_REPOS=$(cat ${EXCEPTIONS_FILE} | wc -l) 92 | 93 | if [ ${NUMBER_OF_EXCEPTION_REPOS} -gt 0 ]; then 94 | EXCEPTIONS_FILE_HOME_PATH="${HOME}/repos_to_remove_talisman_from.paths" 95 | mv ${EXCEPTIONS_FILE} ${EXCEPTIONS_FILE_HOME_PATH} 96 | echo_error "" 97 | echo_error "Please see ${EXCEPTIONS_FILE_HOME_PATH} for a list of repositories" 98 | echo_error "that talisman couldn't be automatically removed from" 99 | echo_error "This is likely because these repos are using pre-commit (https://pre-commit.com)" 100 | echo_error "Remove lines related to talisman from the .pre-commit-config.yaml manually" 101 | fi 102 | } 103 | 104 | function remove_talisman_env_variables() { 105 | FILE_PATH="$1" 106 | if [ -f $FILE_PATH ] && grep -q "TALISMAN_HOME" $FILE_PATH; then 107 | sed -i'-talisman.bak' '/# >>> talisman >>>/,/# <<< talisman <<&2 6 | echo "$1" >&2 7 | echo -ne $(tput sgr0) >&2 8 | } 9 | 10 | function echo_debug() { 11 | [[ -z "${DEBUG}" ]] && return 12 | echo -ne $(tput setaf 3) >&2 13 | echo "$1" >&2 14 | echo -ne $(tput sgr0) >&2 15 | } 16 | 17 | function echo_success { 18 | echo -ne $(tput setaf 2) 19 | echo "$1" >&2 20 | echo -ne $(tput sgr0) 21 | } 22 | 23 | TALISMAN_PATH=$1 24 | EXCEPTIONS_FILE=$2 25 | DOT_GIT_DIR=$3 26 | HOOK_SCRIPT=$4 27 | 28 | REPO_HOOK_SCRIPT=${DOT_GIT_DIR}/hooks/${HOOK_SCRIPT} 29 | echo_debug "Processing hook: ${REPO_HOOK_SCRIPT}" 30 | 31 | if [[ ! -e "${REPO_HOOK_SCRIPT}" ]]; then 32 | echo_success "No ${REPO_HOOK_SCRIPT}, nothing to do" 33 | exit 0 34 | fi 35 | 36 | # remove script if it is symlinked to talisman 37 | if [ "${REPO_HOOK_SCRIPT}" -ef "${TALISMAN_PATH}" ]; then 38 | rm ${REPO_HOOK_SCRIPT} && echo_success "Removed ${REPO_HOOK_SCRIPT}" 39 | exit 0 40 | fi 41 | 42 | if [ -e "${DOT_GIT_DIR}/../.pre-commit-config.yaml" ]; then 43 | # check if the .pre-commit-config contains "talisman", if so ask them to remove it manually 44 | echo_error "Pre-existing pre-commit.com hook detected in ${DOT_GIT_DIR}/hooks" 45 | fi 46 | echo ${DOT_GIT_DIR} | sed 's#/.git$##' >>$EXCEPTIONS_FILE 47 | } 48 | 49 | run $@ 50 | -------------------------------------------------------------------------------- /global_install_scripts/update_talisman.bash: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | shopt -s extglob 4 | 5 | DEBUG=${DEBUG:-''} 6 | FORCE_DOWNLOAD=${FORCE_DOWNLOAD:-''} 7 | declare UPDATE_TYPE="" 8 | if [[ $# -gt 0 && $1 =~ talisman-binary.* ]]; then 9 | UPDATE_TYPE='talisman-binary' 10 | fi 11 | 12 | function run() { 13 | 14 | # Download appropriate (appropriate = based on OS and ARCH) talisman binary from github 15 | # Copy the talisman binary to $TALISMAN_SETUP_DIR ($HOME/.talisman/bin) 16 | 17 | declare TALISMAN_BINARY_NAME 18 | 19 | IFS=$'\n' 20 | VERSION=${VERSION:-'latest'} 21 | INSTALL_ORG_REPO=${INSTALL_ORG_REPO:-'thoughtworks/talisman'} 22 | 23 | TALISMAN_SETUP_DIR=${HOME}/.talisman/bin # location of central install: talisman binary and hook script 24 | 25 | TEMP_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'talisman_setup') 26 | #trap "rm -r ${TEMP_DIR}" EXIT 27 | chmod 0700 ${TEMP_DIR} 28 | 29 | function echo_error() { 30 | echo -ne $(tput setaf 1) >&2 31 | echo "$1" >&2 32 | echo -ne $(tput sgr0) >&2 33 | } 34 | export -f echo_error 35 | 36 | function echo_debug() { 37 | [[ -z "${DEBUG}" ]] && return 38 | echo -ne $(tput setaf 3) >&2 39 | echo "$1" >&2 40 | echo -ne $(tput sgr0) >&2 41 | } 42 | export -f echo_debug 43 | 44 | function echo_success() { 45 | echo -ne $(tput setaf 2) 46 | echo "$1" >&2 47 | echo -ne $(tput sgr0) 48 | } 49 | export -f echo_success 50 | 51 | 52 | function operating_system() { 53 | OS=$(uname -s) 54 | case $OS in 55 | "Linux") 56 | echo "linux" 57 | ;; 58 | "Darwin") 59 | echo "darwin" 60 | ;; 61 | MINGW32_NT-10.0-WOW*) 62 | echo "windows" 63 | ;; 64 | MINGW64_NT-10.0*) 65 | echo "windows" 66 | ;; 67 | MSYS_NT-*) 68 | echo "windows" 69 | ;; 70 | *) 71 | echo_error "Talisman currently only supports Windows, Linux and MacOS(darwin) systems." 72 | echo_error "If this is a problem for you, please open an issue: https://github.com/${INSTALL_ORG_REPO}/issues/new" 73 | exit $E_UNSUPPORTED_ARCH 74 | ;; 75 | esac 76 | } 77 | 78 | function set_talisman_binary_name() { 79 | # based on OS (linux/darwin) and ARCH(32/64 bit) 80 | echo_debug "Running set_talisman_binary_name" 81 | declare OS 82 | OS=$(operating_system) 83 | ARCH=$(uname -m) 84 | case $ARCH in 85 | "x86_64") 86 | OS="${OS}_amd64" 87 | ;; 88 | "i686" | "i386") 89 | OS="${OS}_386" 90 | ;; 91 | "arm64") 92 | OS="${OS}_arm64" 93 | ;; 94 | *) 95 | echo_error "Talisman currently only supports x86 and x86_64 architectures." 96 | echo_error "If this is a problem for you, please open an issue: https://github.com/${INSTALL_ORG_REPO}/issues/new" 97 | exit $E_UNSUPPORTED_ARCH 98 | ;; 99 | esac 100 | 101 | TALISMAN_BINARY_NAME="talisman_${OS}" 102 | if [[ $OS == *"windows"* ]]; then 103 | TALISMAN_BINARY_NAME="${TALISMAN_BINARY_NAME}.exe" 104 | fi 105 | } 106 | 107 | function download() { 108 | echo_debug "Running download()" 109 | OBJECT=$1 110 | DOWNLOAD_URL=$(curl -Ls https://api.github.com/repos/"$INSTALL_ORG_REPO"/releases/latest | 111 | grep download_url | awk '{print $2}' | tr -d '"' | grep "$OBJECT") 112 | 113 | echo_debug "Downloading ${OBJECT} from ${DOWNLOAD_URL}" 114 | curl --location --silent ${DOWNLOAD_URL} >${TEMP_DIR}/${OBJECT} 115 | } 116 | 117 | function verify_checksum() { 118 | FILE_NAME=$1 119 | CHECKSUM_FILE_NAME='checksums' 120 | echo_debug "Verifying checksum for ${FILE_NAME}" 121 | download ${CHECKSUM_FILE_NAME} 122 | 123 | pushd ${TEMP_DIR} >/dev/null 2>&1 124 | grep ${TALISMAN_BINARY_NAME} ${CHECKSUM_FILE_NAME} >${CHECKSUM_FILE_NAME}.single 125 | 126 | if ! command -v shasum &> /dev/null; then 127 | sha256sum -c ${CHECKSUM_FILE_NAME}.single 128 | else 129 | shasum -a 256 -c ${CHECKSUM_FILE_NAME}.single 130 | fi 131 | popd >/dev/null 2>&1 132 | echo_debug "Checksum verification successful!" 133 | echo 134 | } 135 | 136 | function download_talisman_binary() { 137 | #download talisman binary 138 | echo_debug "Running download_talisman_binary" 139 | download ${TALISMAN_BINARY_NAME} 140 | verify_checksum ${TALISMAN_BINARY_NAME} 141 | } 142 | 143 | function verify_binary_is_working() { 144 | chmod +x ${TEMP_DIR}/${TALISMAN_BINARY_NAME} 145 | if ! ${TEMP_DIR}/${TALISMAN_BINARY_NAME} --version; then 146 | echo_error "Binary is not working, SKIPPING UPGRADE, Please open issue on github with your OS name and version" 147 | exit 0 148 | fi 149 | } 150 | 151 | function download_talisman_hook_script() { 152 | echo_debug "Running download_talisman_hook_script" 153 | curl --silent https://raw.githubusercontent.com/${INSTALL_ORG_REPO}/master/global_install_scripts/talisman_hook_script.bash >${TEMP_DIR}/talisman_hook_script 154 | } 155 | 156 | function setup_talisman() { 157 | # copy talisman binary from TEMP folder to the central location 158 | rm -f ${TALISMAN_SETUP_DIR}/${TALISMAN_BINARY_NAME} 159 | cp ${TEMP_DIR}/${TALISMAN_BINARY_NAME} ${TALISMAN_SETUP_DIR} 160 | chmod +x ${TALISMAN_SETUP_DIR}/${TALISMAN_BINARY_NAME} 161 | echo_success "Talisman binary updated successfully!" 162 | } 163 | 164 | function setup_talisman_hook_script() { 165 | BINARY_PATH=${TALISMAN_SETUP_DIR}/${TALISMAN_BINARY_NAME} 166 | rm -f ${TALISMAN_SETUP_DIR}/talisman_hook_script 167 | sed "s@\${TALISMAN_BINARY}@$BINARY_PATH@g" ${TEMP_DIR}/talisman_hook_script >${TALISMAN_SETUP_DIR}/talisman_hook_script 168 | chmod +x ${TALISMAN_SETUP_DIR}/talisman_hook_script 169 | echo_success "Talisman hook script updated successfully!" 170 | } 171 | 172 | function set_talisman_env_variables_properly() { 173 | FILE_PATH="$1" 174 | if [ -f $FILE_PATH ] && grep -q "TALISMAN_HOME" $FILE_PATH; then 175 | if ! grep -q ">>> talisman >>>" $FILE_PATH; then 176 | sed -i'-talisman.bak' '/TALISMAN_HOME/d' $FILE_PATH 177 | echo -e "\n" >>${ENV_FILE} 178 | echo "# >>> talisman >>>" >>$FILE_PATH 179 | echo "# Below environment variables should not be modified unless you know what you are doing" >>$FILE_PATH 180 | echo "export TALISMAN_HOME=${TALISMAN_SETUP_DIR}" >>$FILE_PATH 181 | echo "alias talisman=\$TALISMAN_HOME/${TALISMAN_BINARY_NAME}" >>$FILE_PATH 182 | echo "# <<< talisman <<<" >>$FILE_PATH 183 | fi 184 | fi 185 | } 186 | 187 | set_talisman_binary_name 188 | echo "Downloading latest talisman binary..." 189 | download_talisman_binary 190 | verify_binary_is_working 191 | setup_talisman 192 | if [ -z "$UPDATE_TYPE" ]; then 193 | echo "Downloading latest talisman hook script..." 194 | download_talisman_hook_script 195 | setup_talisman_hook_script 196 | fi 197 | # Correcting talisman env variables if they are not in proper format 198 | if [ -n "${TALISMAN_HOME:-}" ]; then 199 | set_talisman_env_variables_properly ~/.bashrc 200 | set_talisman_env_variables_properly ~/.bash_profile 201 | set_talisman_env_variables_properly ~/.profile 202 | fi 203 | 204 | } 205 | 206 | run $0 $@ 207 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module talisman 2 | 3 | require ( 4 | github.com/AlecAivazis/survey/v2 v2.3.7 5 | github.com/bmatcuk/doublestar v1.3.4 6 | github.com/cheggaaa/pb/v3 v3.1.7 7 | github.com/common-nighthawk/go-figure v0.0.0-20200609044655-c4b36f998cf2 8 | github.com/drhodes/golorem v0.0.0-20160418191928-ecccc744c2d9 9 | github.com/fatih/color v1.18.0 10 | github.com/golang/mock v1.6.0 11 | github.com/olekukonko/tablewriter v0.0.5 12 | github.com/sirupsen/logrus v1.9.3 13 | github.com/spf13/afero v1.14.0 14 | github.com/spf13/pflag v1.0.6 15 | github.com/stretchr/testify v1.10.0 16 | gopkg.in/yaml.v2 v2.4.0 17 | ) 18 | 19 | require ( 20 | github.com/VividCortex/ewma v1.2.0 // indirect 21 | github.com/davecgh/go-spew v1.1.1 // indirect 22 | github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect 23 | github.com/mattn/go-colorable v0.1.14 // indirect 24 | github.com/mattn/go-isatty v0.0.20 // indirect 25 | github.com/mattn/go-runewidth v0.0.16 // indirect 26 | github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect 27 | github.com/pmezard/go-difflib v1.0.0 // indirect 28 | github.com/rivo/uniseg v0.4.7 // indirect 29 | golang.org/x/sys v0.30.0 // indirect 30 | golang.org/x/term v0.29.0 // indirect 31 | golang.org/x/text v0.23.0 // indirect 32 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 33 | gopkg.in/yaml.v3 v3.0.1 // indirect 34 | ) 35 | 36 | go 1.23.0 37 | 38 | toolchain go1.24.2 39 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | declare BINARY_NAME 5 | 6 | E_UNSUPPORTED_ARCH=5 7 | CHECKSUM_FILE_NAME='checksums' 8 | 9 | DEBUG=${DEBUG:-''} 10 | VERSION=${VERSION:-'latest'} 11 | INSTALL_ORG_REPO=${INSTALL_ORG_REPO:-'thoughtworks/talisman'} 12 | INSTALL_LOCATION=${INSTALL_LOCATION:-'/usr/local/bin'} 13 | 14 | function echo_error() { 15 | echo -ne "$(tput setaf 1)" >&2 16 | echo "$1" >&2 17 | echo -ne "$(tput sgr0)" >&2 18 | } 19 | 20 | function echo_debug() { 21 | [[ -z "$DEBUG" ]] && return 22 | echo -ne "$(tput setaf 3)" >&2 23 | echo "$1" >&2 24 | echo -ne "$(tput sgr0)" >&2 25 | } 26 | 27 | function echo_success() { 28 | echo -ne "$(tput setaf 2)" 29 | echo "$1" >&2 30 | echo -ne "$(tput sgr0)" 31 | } 32 | 33 | function operating_system() { 34 | OS=$(uname -s) 35 | case $OS in 36 | "Linux") 37 | echo "linux" 38 | ;; 39 | "Darwin") 40 | echo "darwin" 41 | ;; 42 | MINGW32_NT-* | MINGW64_NT-* | MSYS_NT-*) 43 | echo "windows" 44 | ;; 45 | *) 46 | echo_error "Talisman currently only supports Windows, Linux, and MacOS (darwin) systems." 47 | echo_error "If this is a problem for you, please open an issue: https://github.com/$INSTALL_ORG_REPO/issues/new" 48 | exit $E_UNSUPPORTED_ARCH 49 | ;; 50 | esac 51 | } 52 | 53 | function architecture() { 54 | ARCH=$(uname -m) 55 | case $ARCH in 56 | "x86_64") 57 | echo "amd64" 58 | ;; 59 | "i686" | "i386") 60 | echo "386" 61 | ;; 62 | "arm64" | "aarch64") 63 | echo "arm64" 64 | ;; 65 | *) 66 | echo_error "Talisman currently only supports x86 and x86_64 and arm64 architectures." 67 | echo_error "If this is a problem for you, please open an issue: https://github.com/$INSTALL_ORG_REPO/issues/new" 68 | exit $E_UNSUPPORTED_ARCH 69 | ;; 70 | esac 71 | } 72 | 73 | function set_binary_name() { 74 | BINARY_NAME="talisman_$(operating_system)_$(architecture)" 75 | if [ "$(operating_system)" = "windows" ]; then 76 | BINARY_NAME="$BINARY_NAME.exe" 77 | fi 78 | echo_success "Selected $BINARY_NAME" 79 | } 80 | 81 | function download() { 82 | if [ "$VERSION" != "latest" ]; then 83 | VERSION="tags/$VERSION" 84 | fi 85 | 86 | ASSETS=$(curl -Ls https://api.github.com/repos/"$INSTALL_ORG_REPO"/releases/"$VERSION" | 87 | grep download_url | awk '{print $2}' | tr -d '"') 88 | BINARY_URL=$(echo "$ASSETS" | grep "$BINARY_NAME") 89 | CHECKSUM_URL=$(echo "$ASSETS" | grep $CHECKSUM_FILE_NAME) 90 | echo_debug "Downloading $BINARY_NAME and from $BINARY_URL" 91 | curl --location --silent "$BINARY_URL" >"$TEMP_DIR/$BINARY_NAME" 92 | echo_debug "Downloading $CHECKSUM_FILE_NAME and from $CHECKSUM_URL" 93 | curl --location --silent "$CHECKSUM_URL" >"$TEMP_DIR/$CHECKSUM_FILE_NAME" 94 | echo_success "Downloaded talisman binary and checksums" 95 | } 96 | 97 | function verify_checksum() { 98 | pushd "$TEMP_DIR" >/dev/null 2>&1 99 | 100 | if ! command -v shasum &>/dev/null; then 101 | sha256sum --ignore-missing -c $CHECKSUM_FILE_NAME 102 | else 103 | shasum -a 256 --ignore-missing -c $CHECKSUM_FILE_NAME 104 | fi 105 | 106 | popd >/dev/null 2>&1 107 | echo_success "Checksum OK" 108 | } 109 | 110 | function install() { 111 | if (touch "$INSTALL_LOCATION/$BINARY_NAME" &>/dev/null); then 112 | cp "$TEMP_DIR/$BINARY_NAME" "$INSTALL_LOCATION/$BINARY_NAME" 113 | chmod +x "$INSTALL_LOCATION/$BINARY_NAME" 114 | ln -s "$INSTALL_LOCATION/$BINARY_NAME" "$INSTALL_LOCATION/talisman" 115 | elif (which sudo &>/dev/null); then 116 | sudo cp "$TEMP_DIR/$BINARY_NAME" "$INSTALL_LOCATION/$BINARY_NAME" 117 | sudo chmod +x "$INSTALL_LOCATION/$BINARY_NAME" 118 | sudo ln -s "$INSTALL_LOCATION/$BINARY_NAME" "$INSTALL_LOCATION/talisman" 119 | else 120 | echo_error "Insufficient permission to install to $INSTALL_LOCATION" 121 | exit 126 122 | fi 123 | } 124 | 125 | function run() { 126 | TEMP_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'talisman_setup') 127 | # shellcheck disable=SC2064 128 | trap "rm -r $TEMP_DIR" EXIT 129 | chmod 0700 "$TEMP_DIR" 130 | 131 | if [ ! -d "$INSTALL_LOCATION" ]; then 132 | echo_error "$INSTALL_LOCATION is not a directory!" 133 | exit 1 134 | fi 135 | 136 | set_binary_name 137 | download 138 | verify_checksum 139 | install 140 | } 141 | 142 | run 143 | -------------------------------------------------------------------------------- /internal/mock/checksumcalculator/checksumcalculator.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: checksumcalculator/checksumcalculator.go 3 | 4 | // Package mock is a generated GoMock package. 5 | package mock 6 | 7 | import ( 8 | reflect "reflect" 9 | 10 | gomock "github.com/golang/mock/gomock" 11 | ) 12 | 13 | // MockChecksumCalculator is a mock of ChecksumCalculator interface. 14 | type MockChecksumCalculator struct { 15 | ctrl *gomock.Controller 16 | recorder *MockChecksumCalculatorMockRecorder 17 | } 18 | 19 | // MockChecksumCalculatorMockRecorder is the mock recorder for MockChecksumCalculator. 20 | type MockChecksumCalculatorMockRecorder struct { 21 | mock *MockChecksumCalculator 22 | } 23 | 24 | // NewMockChecksumCalculator creates a new mock instance. 25 | func NewMockChecksumCalculator(ctrl *gomock.Controller) *MockChecksumCalculator { 26 | mock := &MockChecksumCalculator{ctrl: ctrl} 27 | mock.recorder = &MockChecksumCalculatorMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use. 32 | func (m *MockChecksumCalculator) EXPECT() *MockChecksumCalculatorMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // CalculateCollectiveChecksumForPattern mocks base method. 37 | func (m *MockChecksumCalculator) CalculateCollectiveChecksumForPattern(fileNamePattern string) string { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "CalculateCollectiveChecksumForPattern", fileNamePattern) 40 | ret0, _ := ret[0].(string) 41 | return ret0 42 | } 43 | 44 | // CalculateCollectiveChecksumForPattern indicates an expected call of CalculateCollectiveChecksumForPattern. 45 | func (mr *MockChecksumCalculatorMockRecorder) CalculateCollectiveChecksumForPattern(fileNamePattern interface{}) *gomock.Call { 46 | mr.mock.ctrl.T.Helper() 47 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CalculateCollectiveChecksumForPattern", reflect.TypeOf((*MockChecksumCalculator)(nil).CalculateCollectiveChecksumForPattern), fileNamePattern) 48 | } 49 | 50 | // SuggestTalismanRC mocks base method. 51 | func (m *MockChecksumCalculator) SuggestTalismanRC(fileNamePatterns []string) string { 52 | m.ctrl.T.Helper() 53 | ret := m.ctrl.Call(m, "SuggestTalismanRC", fileNamePatterns) 54 | ret0, _ := ret[0].(string) 55 | return ret0 56 | } 57 | 58 | // SuggestTalismanRC indicates an expected call of SuggestTalismanRC. 59 | func (mr *MockChecksumCalculatorMockRecorder) SuggestTalismanRC(fileNamePatterns interface{}) *gomock.Call { 60 | mr.mock.ctrl.T.Helper() 61 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SuggestTalismanRC", reflect.TypeOf((*MockChecksumCalculator)(nil).SuggestTalismanRC), fileNamePatterns) 62 | } 63 | -------------------------------------------------------------------------------- /internal/mock/prompt/survey.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: prompt/survey.go 3 | 4 | // Package mock is a generated GoMock package. 5 | package mock 6 | 7 | import ( 8 | reflect "reflect" 9 | 10 | gomock "github.com/golang/mock/gomock" 11 | ) 12 | 13 | // MockPrompt is a mock of Prompt interface. 14 | type MockPrompt struct { 15 | ctrl *gomock.Controller 16 | recorder *MockPromptMockRecorder 17 | } 18 | 19 | // MockPromptMockRecorder is the mock recorder for MockPrompt. 20 | type MockPromptMockRecorder struct { 21 | mock *MockPrompt 22 | } 23 | 24 | // NewMockPrompt creates a new mock instance. 25 | func NewMockPrompt(ctrl *gomock.Controller) *MockPrompt { 26 | mock := &MockPrompt{ctrl: ctrl} 27 | mock.recorder = &MockPromptMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use. 32 | func (m *MockPrompt) EXPECT() *MockPromptMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // Confirm mocks base method. 37 | func (m *MockPrompt) Confirm(arg0 string) bool { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "Confirm", arg0) 40 | ret0, _ := ret[0].(bool) 41 | return ret0 42 | } 43 | 44 | // Confirm indicates an expected call of Confirm. 45 | func (mr *MockPromptMockRecorder) Confirm(arg0 interface{}) *gomock.Call { 46 | mr.mock.ctrl.T.Helper() 47 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Confirm", reflect.TypeOf((*MockPrompt)(nil).Confirm), arg0) 48 | } 49 | -------------------------------------------------------------------------------- /internal/mock/utility/sha_256_hasher.go: -------------------------------------------------------------------------------- 1 | // Code generated by MockGen. DO NOT EDIT. 2 | // Source: utility/sha_256_hasher.go 3 | 4 | // Package mock is a generated GoMock package. 5 | package mock 6 | 7 | import ( 8 | reflect "reflect" 9 | 10 | gomock "github.com/golang/mock/gomock" 11 | ) 12 | 13 | // MockSHA256Hasher is a mock of SHA256Hasher interface. 14 | type MockSHA256Hasher struct { 15 | ctrl *gomock.Controller 16 | recorder *MockSHA256HasherMockRecorder 17 | } 18 | 19 | // MockSHA256HasherMockRecorder is the mock recorder for MockSHA256Hasher. 20 | type MockSHA256HasherMockRecorder struct { 21 | mock *MockSHA256Hasher 22 | } 23 | 24 | // NewMockSHA256Hasher creates a new mock instance. 25 | func NewMockSHA256Hasher(ctrl *gomock.Controller) *MockSHA256Hasher { 26 | mock := &MockSHA256Hasher{ctrl: ctrl} 27 | mock.recorder = &MockSHA256HasherMockRecorder{mock} 28 | return mock 29 | } 30 | 31 | // EXPECT returns an object that allows the caller to indicate expected use. 32 | func (m *MockSHA256Hasher) EXPECT() *MockSHA256HasherMockRecorder { 33 | return m.recorder 34 | } 35 | 36 | // CollectiveSHA256Hash mocks base method. 37 | func (m *MockSHA256Hasher) CollectiveSHA256Hash(paths []string) string { 38 | m.ctrl.T.Helper() 39 | ret := m.ctrl.Call(m, "CollectiveSHA256Hash", paths) 40 | ret0, _ := ret[0].(string) 41 | return ret0 42 | } 43 | 44 | // CollectiveSHA256Hash indicates an expected call of CollectiveSHA256Hash. 45 | func (mr *MockSHA256HasherMockRecorder) CollectiveSHA256Hash(paths interface{}) *gomock.Call { 46 | mr.mock.ctrl.T.Helper() 47 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CollectiveSHA256Hash", reflect.TypeOf((*MockSHA256Hasher)(nil).CollectiveSHA256Hash), paths) 48 | } 49 | 50 | // Shutdown mocks base method. 51 | func (m *MockSHA256Hasher) Shutdown() error { 52 | m.ctrl.T.Helper() 53 | ret := m.ctrl.Call(m, "Shutdown") 54 | ret0, _ := ret[0].(error) 55 | return ret0 56 | } 57 | 58 | // Shutdown indicates an expected call of Shutdown. 59 | func (mr *MockSHA256HasherMockRecorder) Shutdown() *gomock.Call { 60 | mr.mock.ctrl.T.Helper() 61 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Shutdown", reflect.TypeOf((*MockSHA256Hasher)(nil).Shutdown)) 62 | } 63 | 64 | // Start mocks base method. 65 | func (m *MockSHA256Hasher) Start() error { 66 | m.ctrl.T.Helper() 67 | ret := m.ctrl.Call(m, "Start") 68 | ret0, _ := ret[0].(error) 69 | return ret0 70 | } 71 | 72 | // Start indicates an expected call of Start. 73 | func (mr *MockSHA256HasherMockRecorder) Start() *gomock.Call { 74 | mr.mock.ctrl.T.Helper() 75 | return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockSHA256Hasher)(nil).Start)) 76 | } 77 | -------------------------------------------------------------------------------- /mock_scripts: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | install_mockgen() { 4 | pushd .. 5 | go install github.com/golang/mock/mockgen@latest 6 | popd 7 | } 8 | 9 | generate_mock() { 10 | FILE=$1 11 | mockgen -destination internal/mock/$FILE -package mock -source $FILE 12 | } 13 | 14 | set -ex 15 | #Ensure mockgen exists 16 | which mockgen || install_mockgen 17 | mkdir -p internal/mock 18 | 19 | # Mock for survey package 20 | 21 | generate_mock prompt/survey.go 22 | generate_mock checksumcalculator/checksumcalculator.go 23 | generate_mock utility/sha_256_hasher.go -------------------------------------------------------------------------------- /prompt/survey.go: -------------------------------------------------------------------------------- 1 | package prompt 2 | 3 | //go:generate mockgen -destination ../internal/mock/prompt/survey.go -package mock -source survey.go 4 | 5 | import ( 6 | "github.com/AlecAivazis/survey/v2" 7 | "log" 8 | ) 9 | 10 | type Prompt interface { 11 | Confirm(string) bool 12 | } 13 | 14 | func NewPrompt() Prompt { 15 | return prompt{} 16 | } 17 | 18 | type prompt struct{} 19 | 20 | func NewPromptContext(interactive bool, prompt Prompt) PromptContext { 21 | return PromptContext{ 22 | Interactive: interactive, 23 | Prompt: prompt, 24 | } 25 | } 26 | 27 | type PromptContext struct { 28 | Interactive bool 29 | Prompt Prompt 30 | } 31 | 32 | func (p prompt) Confirm(message string) bool { 33 | if message == "" { 34 | message = "Confirm?" 35 | } 36 | 37 | confirmPrompt := &survey.Confirm{ 38 | Default: false, 39 | Message: message, 40 | } 41 | 42 | confirmation := false 43 | err := survey.AskOne(confirmPrompt, &confirmation) 44 | if err != nil { 45 | log.Printf("error occurred when getting input from user: %s", err) 46 | return false 47 | } 48 | 49 | return confirmation 50 | } 51 | -------------------------------------------------------------------------------- /report/report.go: -------------------------------------------------------------------------------- 1 | package report 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "os" 8 | "os/user" 9 | "path/filepath" 10 | "talisman/detector/helpers" 11 | "talisman/utility" 12 | 13 | "github.com/fatih/color" 14 | ) 15 | 16 | const jsonFileName string = "report.json" 17 | const htmlReportDir string = "talisman_html_report" 18 | const jsonReportDir string = "talisman_html_report" 19 | 20 | // GenerateReport generates a talisman scan report in html format 21 | func GenerateReport(r *helpers.DetectionResults, directory string) (string, error) { 22 | var jsonFilePath string 23 | var homeDir string 24 | var baseReportDirPath string 25 | 26 | usr, err := user.Current() 27 | if err != nil { 28 | return "", fmt.Errorf("error getting current user: %v", err.Error()) 29 | } 30 | homeDir = usr.HomeDir 31 | path := jsonReportDir 32 | if directory == htmlReportDir { 33 | path = directory 34 | baseReportDirPath = filepath.Join(homeDir, ".talisman", htmlReportDir) 35 | jsonFilePath = filepath.Join(path, "/data", jsonFileName) 36 | err = utility.Dir(baseReportDirPath, htmlReportDir) 37 | if err != nil { 38 | generateErrorMsg() 39 | return "", fmt.Errorf("error copying reports: %v", err) 40 | } 41 | } else { 42 | path = filepath.Join(directory, "talisman_reports") 43 | _ = os.RemoveAll(path) 44 | path = filepath.Join(path, "data") 45 | jsonFilePath = filepath.Join(path, jsonFileName) 46 | } 47 | 48 | err = os.MkdirAll(path, 0755) 49 | if err != nil { 50 | return "", fmt.Errorf("error creating path %s: %v", path, err) 51 | } 52 | 53 | _, err = generateAndWriteToFile(r, jsonFilePath) 54 | if err != nil { 55 | return "", err 56 | } 57 | return path, nil 58 | } 59 | 60 | func generateAndWriteToFile(r *helpers.DetectionResults, jsonFilePath string) (path string, err error) { 61 | jsonFile, err := os.Create(jsonFilePath) 62 | defer func() { 63 | if err = jsonFile.Close(); err != nil { 64 | err = fmt.Errorf("error closing file %s: %v %#v", jsonFilePath, err, err) 65 | } 66 | }() 67 | 68 | if err != nil { 69 | return "", fmt.Errorf("error creating file %s: %v", jsonFilePath, err) 70 | } 71 | 72 | jsonString, err := json.Marshal(r) 73 | if err != nil { 74 | return "", fmt.Errorf("error while rendering report json: %v : %#v", err, err) 75 | } 76 | _, err = jsonFile.Write(jsonString) 77 | if err != nil { 78 | return "", fmt.Errorf("error while writing report json to file: %v %#v", err, err) 79 | } 80 | return jsonFilePath, nil 81 | } 82 | 83 | func generateErrorMsg() { 84 | color.HiMagenta("\nLooks like you are using 'talisman --scanWithHtml' for scanning.") 85 | color.HiMagenta("But it appears that you have not installed Talisman Html Report") 86 | color.HiMagenta("Please go through Talisman Readme and make sure you install the same from:") 87 | color.Yellow("\nhttps://github.com/jaydeepc/talisman-html-report") 88 | color.Cyan("\nOR use 'talisman --scan' if you want the JSON report alone\n") 89 | fmt.Printf("\n") 90 | color.Red("Failed: Unable to perform Scan") 91 | fmt.Printf("\n") 92 | log.Fatalln("Run Status: Failed") 93 | } 94 | -------------------------------------------------------------------------------- /scanner/scanner.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "os/exec" 7 | "strings" 8 | "talisman/gitrepo" 9 | "talisman/utility" 10 | 11 | "github.com/sirupsen/logrus" 12 | ) 13 | 14 | type blobDetails struct { 15 | hash, filePath string 16 | } 17 | 18 | // BlobsInCommits is a map of blob and list of the commits the blobs is present in. 19 | type BlobsInCommits struct { 20 | commits map[blobDetails][]string 21 | } 22 | 23 | // GetAdditions will get all the additions for entire git history 24 | func GetAdditions(ignoreHistory bool, br gitrepo.BatchReader) []gitrepo.Addition { 25 | blobsInCommits := getBlobsInCommit(ignoreHistory) 26 | var additions []gitrepo.Addition 27 | err := br.Start() 28 | if err != nil { 29 | logrus.Errorf("error creating file reader %v", err) 30 | } 31 | defer func() { 32 | err = br.Shutdown() 33 | if err != nil { 34 | logrus.Errorf("error creating file reader %v", err) 35 | } 36 | }() 37 | 38 | for blob := range blobsInCommits.commits { 39 | contents, _ := br.Read(blob.hash) 40 | newAddition := gitrepo.NewScannerAddition(blob.filePath, blobsInCommits.commits[blob], contents) 41 | additions = append(additions, newAddition) 42 | } 43 | 44 | return additions 45 | } 46 | 47 | func getBlobsInCommit(ignoreHistory bool) BlobsInCommits { 48 | progressBar := utility.GetProgressBar(os.Stdout, "Talisman Fetch Blobs") 49 | commits := getAllCommits(ignoreHistory) 50 | progressBar.Start(len(commits) - 1) 51 | blobsInCommits := newBlobsInCommit() 52 | result := make(chan []string, len(commits)) 53 | for _, commit := range commits { 54 | go putBlobsInChannel(commit, result) 55 | } 56 | for i := 1; i < len(commits); i++ { 57 | progressBar.Increment() 58 | getBlobsFromChannel(blobsInCommits, result) 59 | } 60 | progressBar.Finish() 61 | return blobsInCommits 62 | } 63 | 64 | func putBlobsInChannel(commit string, result chan []string) { 65 | if commit != "" { 66 | blobDetailsBytes, _ := exec.Command("git", "ls-tree", "-r", commit).CombinedOutput() 67 | blobDetailsList := strings.Split(string(blobDetailsBytes), "\n") 68 | blobDetailsList = append(blobDetailsList, commit) 69 | result <- blobDetailsList 70 | } 71 | } 72 | 73 | func getBlobsFromChannel(blobsInCommits BlobsInCommits, result chan []string) { 74 | blobEntries := <-result 75 | commit := blobEntries[len(blobEntries)-1] 76 | for _, blobEntry := range blobEntries[:len(blobEntries)-1] { 77 | if blobEntry != "" { 78 | blobHashAndName := strings.Split(strings.Split(blobEntry, " ")[2], "\t") 79 | blob := blobDetails{hash: blobHashAndName[0], filePath: blobHashAndName[1]} 80 | blobsInCommits.commits[blob] = append(blobsInCommits.commits[blob], commit) 81 | } 82 | } 83 | } 84 | 85 | func getAllCommits(ignoreHistory bool) []string { 86 | commitRange := "--all" 87 | if ignoreHistory { 88 | commitRange = "--max-count=1" 89 | } 90 | out, err := exec.Command("git", "log", commitRange, "--pretty=%H").CombinedOutput() 91 | if err != nil { 92 | log.Fatal(err) 93 | } 94 | return strings.Split(string(out), "\n") 95 | } 96 | 97 | func newBlobsInCommit() BlobsInCommits { 98 | commits := make(map[blobDetails][]string) 99 | return BlobsInCommits{commits: commits} 100 | } 101 | -------------------------------------------------------------------------------- /scanner/scanner_test.go: -------------------------------------------------------------------------------- 1 | package scanner 2 | 3 | import ( 4 | "io/ioutil" 5 | "testing" 6 | 7 | logr "github.com/sirupsen/logrus" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func init() { 12 | logr.SetOutput(ioutil.Discard) 13 | } 14 | 15 | func Test_getBlobsFromChannel(t *testing.T) { 16 | ch := make(chan []string) 17 | go func() { 18 | ch <- []string{ 19 | "100644 blob 351324aa7b3c66043e484c2f2c7b7f1842152f35 .gitignore", 20 | "100644 blob 8715df9907604c8ee8fc5e377821817f84f014fa .pre-commit-hooks.yaml", 21 | "commitSha", 22 | } 23 | }() 24 | blobsInCommits := BlobsInCommits{commits: map[blobDetails][]string{}} 25 | getBlobsFromChannel(blobsInCommits, ch) 26 | 27 | commits := blobsInCommits.commits 28 | assert.Len(t, commits, 2) 29 | assert.Equal(t, []string{"commitSha"}, commits[blobDetails{"351324aa7b3c66043e484c2f2c7b7f1842152f35", ".gitignore"}]) 30 | assert.Equal(t, []string{"commitSha"}, commits[blobDetails{"8715df9907604c8ee8fc5e377821817f84f014fa", ".pre-commit-hooks.yaml"}]) 31 | } 32 | -------------------------------------------------------------------------------- /talisman.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thoughtworks/talisman/12fab7055d7e640d5dae43209bafc48bf5ef1fd7/talisman.png -------------------------------------------------------------------------------- /talismanrc/rc_file.go: -------------------------------------------------------------------------------- 1 | package talismanrc 2 | 3 | import ( 4 | "fmt" 5 | 6 | logr "github.com/sirupsen/logrus" 7 | 8 | "github.com/spf13/afero" 9 | "gopkg.in/yaml.v2" 10 | ) 11 | 12 | const ( 13 | // RCFileName represents the name of default file in which all the ignore patterns are configured in new version 14 | RCFileName = ".talismanrc" 15 | DefaultRCVersion = "1.0" 16 | ) 17 | 18 | var ( 19 | fs = afero.NewOsFs() 20 | ) 21 | 22 | // Load creates a TalismanRC struct based on a .talismanrc file, if present 23 | func Load() (*TalismanRC, error) { 24 | fileContents, err := afero.ReadFile(fs, RCFileName) 25 | if err != nil { 26 | // File does not exist or is not readable, proceed as if there is no .talismanrc 27 | fileContents = []byte{} 28 | } 29 | return talismanRCFromYaml(fileContents) 30 | } 31 | 32 | func talismanRCFromYaml(fileContents []byte) (*TalismanRC, error) { 33 | talismanRCFromFile := TalismanRC{} 34 | err := yaml.Unmarshal(fileContents, &talismanRCFromFile) 35 | if err != nil { 36 | logr.Errorf("Unable to parse .talismanrc : %v", err) 37 | fmt.Println(fmt.Errorf("\n\x1b[1m\x1b[31mUnable to parse .talismanrc %s. Please ensure it is following the right YAML structure\x1b[0m\x1b[0m", err)) 38 | return &TalismanRC{}, err 39 | } 40 | if talismanRCFromFile.Version == "" { 41 | talismanRCFromFile.Version = DefaultRCVersion 42 | } 43 | return &talismanRCFromFile, nil 44 | } 45 | 46 | func (tRC *TalismanRC) saveToFile() { 47 | ignoreEntries, _ := yaml.Marshal(&tRC) 48 | err := afero.WriteFile(fs, RCFileName, ignoreEntries, 0644) 49 | if err != nil { 50 | logr.Errorf("error writing to %s: %s", RCFileName, err) 51 | } 52 | } 53 | 54 | func SetFs__(_fs afero.Fs) { 55 | fs = _fs 56 | } 57 | -------------------------------------------------------------------------------- /talismanrc/rc_file_test.go: -------------------------------------------------------------------------------- 1 | package talismanrc 2 | 3 | import ( 4 | "regexp" 5 | "talisman/detector/severity" 6 | "testing" 7 | 8 | "github.com/spf13/afero" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestLoadingFromFile(t *testing.T) { 13 | fs := afero.NewMemMapFs() 14 | SetFs__(fs) 15 | 16 | t.Run("Creates an empty TalismanRC if .talismanrc file doesn't exist", func(t *testing.T) { 17 | talismanRCFileExists, _ := afero.Exists(fs, RCFileName) 18 | assert.False(t, talismanRCFileExists, ".talismanrc file should NOT exist for this test!") 19 | emptyRC, err := Load() 20 | assert.NoError(t, err, "Should not error if there is a problem reading the file") 21 | assert.Equal(t, &TalismanRC{Version: DefaultRCVersion}, emptyRC) 22 | }) 23 | 24 | t.Run("Loads all valid TalismanRC fields", func(t *testing.T) { 25 | err := afero.WriteFile(fs, RCFileName, []byte(fullyConfiguredTalismanRC), 0666) 26 | assert.NoError(t, err, "Problem setting up test .talismanrc?") 27 | 28 | talismanRCFromFile, _ := Load() 29 | expectedTalismanRC := &TalismanRC{ 30 | FileIgnoreConfig: []FileIgnoreConfig{ 31 | {FileName: "existing.pem", Checksum: "123444ddssa75333b25b6275f97680604add51b84eb8f4a3b9dcbbc652e6f27ac"}}, 32 | ScopeConfig: []ScopeConfig{{"go"}}, 33 | AllowedPatterns: []*Pattern{ 34 | {regexp.MustCompile("this-is-okay")}, 35 | {regexp.MustCompile("key={listOfThings.id}")}}, 36 | CustomPatterns: []PatternString{"this-isn't-okay"}, 37 | Threshold: severity.Medium, 38 | CustomSeverities: []CustomSeverityConfig{ 39 | {Detector: "HexContent", Severity: severity.Low}}, 40 | Experimental: ExperimentalConfig{Base64EntropyThreshold: 4.7}, 41 | Version: "1.0", 42 | } 43 | assert.Equal(t, expectedTalismanRC, talismanRCFromFile) 44 | }) 45 | } 46 | 47 | func TestWritingToFile(t *testing.T) { 48 | tRC := &TalismanRC{Version: DefaultRCVersion} 49 | fs := afero.NewMemMapFs() 50 | SetFs__(fs) 51 | 52 | t.Run("When there is no .talismanrc file", func(t *testing.T) { 53 | talismanRCFileExists, _ := afero.Exists(fs, RCFileName) 54 | assert.False(t, talismanRCFileExists, "Problem setting up tests") 55 | tRC.saveToFile() 56 | talismanRCFileExists, _ = afero.Exists(fs, RCFileName) 57 | assert.True(t, talismanRCFileExists, "Should have created a new .talismanrc file") 58 | fileContents, _ := afero.ReadFile(fs, RCFileName) 59 | assert.Equal(t, "version: \"1.0\"\n", string(fileContents)) 60 | }) 61 | 62 | t.Run("When there already is a .talismanrc file", func(t *testing.T) { 63 | err := afero.WriteFile(fs, RCFileName, []byte("Some existing content to overwrite"), 0666) 64 | assert.NoError(t, err, "Problem setting up tests") 65 | tRC.saveToFile() 66 | talismanRCFileExists, _ := afero.Exists(fs, RCFileName) 67 | assert.True(t, talismanRCFileExists, "Should have created a new .talismanrc file") 68 | fileContents, _ := afero.ReadFile(fs, RCFileName) 69 | assert.Equal(t, "version: \"1.0\"\n", string(fileContents)) 70 | }) 71 | } 72 | 73 | func TestUnmarshalsValidYaml(t *testing.T) { 74 | t.Run("Should not fail as long as the yaml structure is correct", func(t *testing.T) { 75 | fileContents := []byte(` 76 | --- 77 | fileignoreconfig: 78 | - filename: testfile_1.yml 79 | checksum: file1_checksum 80 | custom_patterns: 81 | - 'pwd_[a-z]{8,20}'`) 82 | 83 | rc, err := talismanRCFromYaml(fileContents) 84 | assert.Nil(t, err, "Should successfully unmarshal valid yaml") 85 | assert.Equal(t, 1, len(rc.FileIgnoreConfig)) 86 | assert.Equal(t, 1, len(rc.CustomPatterns)) 87 | }) 88 | 89 | t.Run("Should read multiple entries in rc file correctly", func(t *testing.T) { 90 | fileContent := []byte(` 91 | fileignoreconfig: 92 | - filename: testfile_1.yml 93 | checksum: file1_checksum 94 | - filename: testfile_2.yml 95 | checksum: file2_checksum 96 | - filename: testfile_3.yml 97 | checksum: file3_checksum`) 98 | 99 | rc, _ := talismanRCFromYaml(fileContent) 100 | assert.Equal(t, 3, len(rc.FileIgnoreConfig)) 101 | 102 | assert.Equal(t, rc.FileIgnoreConfig[0].GetFileName(), "testfile_1.yml") 103 | assert.True(t, rc.FileIgnoreConfig[0].ChecksumMatches("file1_checksum")) 104 | assert.Equal(t, rc.FileIgnoreConfig[1].GetFileName(), "testfile_2.yml") 105 | assert.True(t, rc.FileIgnoreConfig[1].ChecksumMatches("file2_checksum")) 106 | assert.Equal(t, rc.FileIgnoreConfig[2].GetFileName(), "testfile_3.yml") 107 | assert.True(t, rc.FileIgnoreConfig[2].ChecksumMatches("file3_checksum")) 108 | }) 109 | 110 | t.Run("Should read severity level", func(t *testing.T) { 111 | talismanRCContents := []byte("threshold: high") 112 | persistedTalismanrc, _ := talismanRCFromYaml(talismanRCContents) 113 | assert.Equal(t, persistedTalismanrc.Threshold, severity.High) 114 | }) 115 | 116 | t.Run("Should read custom severities", func(t *testing.T) { 117 | talismanRCContents := []byte(` 118 | custom_severities: 119 | - detector: Base64Content 120 | severity: low 121 | `) 122 | talismanRC, _ := talismanRCFromYaml(talismanRCContents) 123 | assert.Equal(t, talismanRC.CustomSeverities, []CustomSeverityConfig{{Detector: "Base64Content", Severity: severity.Low}}) 124 | }) 125 | } 126 | 127 | func TestShouldIgnoreUnformattedFiles(t *testing.T) { 128 | for _, s := range []string{"#", "#monkey", "# this monkey likes bananas "} { 129 | fileContents := []byte(s) 130 | talismanRC, err := talismanRCFromYaml(fileContents) 131 | assert.Nil(t, err, "Should successfully unmarshal commented yaml") 132 | assert.Equal(t, &TalismanRC{Version: "1.0"}, talismanRC, "Expected commented line '%s' to result in an empty TalismanRC") 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /talismanrc/scopes.go: -------------------------------------------------------------------------------- 1 | package talismanrc 2 | 3 | // Mapping of language scopes to names of files that should be ignored anywhere in a repository 4 | var knownScopes = map[string][]string{ 5 | "node": {"pnpm-lock.yaml", "yarn.lock", "package-lock.json"}, 6 | "go": {"makefile", "go.mod", "go.sum", "Gopkg.toml", "Gopkg.lock", "glide.yaml", "glide.lock"}, 7 | "images": {"*.jpeg", "*.jpg", "*.png", "*.tiff", "*.bmp"}, 8 | "bazel": {"*.bzl"}, 9 | "terraform": {".terraform.lock.hcl"}, 10 | "php": {"composer.lock"}, 11 | "python": {"poetry.lock", "Pipfile.lock", "requirements.txt", "uv.lock"}, 12 | } 13 | -------------------------------------------------------------------------------- /talismanrc/talismanrc.go: -------------------------------------------------------------------------------- 1 | package talismanrc 2 | 3 | import ( 4 | "sort" 5 | 6 | logr "github.com/sirupsen/logrus" 7 | "gopkg.in/yaml.v2" 8 | 9 | "talisman/detector/severity" 10 | 11 | "talisman/gitrepo" 12 | ) 13 | 14 | type TalismanRC struct { 15 | FileIgnoreConfig []FileIgnoreConfig `yaml:"fileignoreconfig,omitempty"` 16 | ScopeConfig []ScopeConfig `yaml:"scopeconfig,omitempty"` 17 | CustomPatterns []PatternString `yaml:"custom_patterns,omitempty"` 18 | CustomSeverities []CustomSeverityConfig `yaml:"custom_severities,omitempty"` 19 | AllowedPatterns []*Pattern `yaml:"allowed_patterns,omitempty"` 20 | Experimental ExperimentalConfig `yaml:"experimental,omitempty"` 21 | Threshold severity.Severity `yaml:"threshold,omitempty"` 22 | Version string `yaml:"version"` 23 | } 24 | 25 | // SuggestRCFor returns a string representation of a .talismanrc for the specified FileIgnoreConfigs 26 | func SuggestRCFor(configs []FileIgnoreConfig) string { 27 | tRC := TalismanRC{FileIgnoreConfig: configs, Version: DefaultRCVersion} 28 | result, _ := yaml.Marshal(tRC) 29 | 30 | return string(result) 31 | } 32 | 33 | // RemoveScopedFiles removes scope files from additions 34 | func (tRC *TalismanRC) RemoveScopedFiles(additions []gitrepo.Addition) []gitrepo.Addition { 35 | var applicableScopeFileNames []string 36 | if tRC.ScopeConfig != nil { 37 | for _, scope := range tRC.ScopeConfig { 38 | if len(knownScopes[scope.ScopeName]) > 0 { 39 | applicableScopeFileNames = append(applicableScopeFileNames, knownScopes[scope.ScopeName]...) 40 | } 41 | } 42 | } 43 | var result []gitrepo.Addition 44 | for _, addition := range additions { 45 | isFilePresentInScope := false 46 | for _, fileName := range applicableScopeFileNames { 47 | if addition.NameMatches(fileName) { 48 | isFilePresentInScope = true 49 | break 50 | } 51 | } 52 | if !isFilePresentInScope { 53 | result = append(result, addition) 54 | } 55 | } 56 | return result 57 | } 58 | 59 | // AddIgnores inserts the specified FileIgnoreConfigs to an existing .talismanrc file, or creates one if it doesn't exist. 60 | func (tRC *TalismanRC) AddIgnores(entriesToAdd []FileIgnoreConfig) { 61 | if len(entriesToAdd) > 0 { 62 | logr.Debugf("Adding entries: %v", entriesToAdd) 63 | tRC.FileIgnoreConfig = combineFileIgnores(tRC.FileIgnoreConfig, entriesToAdd) 64 | tRC.saveToFile() 65 | } 66 | } 67 | 68 | func combineFileIgnores(exsiting, incoming []FileIgnoreConfig) []FileIgnoreConfig { 69 | existingMap := make(map[string]FileIgnoreConfig) 70 | for _, fIC := range exsiting { 71 | existingMap[fIC.FileName] = fIC 72 | } 73 | for _, fIC := range incoming { 74 | existingMap[fIC.FileName] = fIC 75 | } 76 | result := make([]FileIgnoreConfig, len(existingMap)) 77 | resultKeys := make([]string, len(existingMap)) 78 | index := 0 79 | //sort keys in alphabetical order 80 | for k := range existingMap { 81 | resultKeys[index] = k 82 | index++ 83 | } 84 | sort.Strings(resultKeys) 85 | //add result entries based on sorted keys 86 | index = 0 87 | for _, k := range resultKeys { 88 | result[index] = existingMap[k] 89 | index++ 90 | } 91 | return result 92 | } 93 | 94 | // RemoveAllowedPatterns removes globally- and per-file allowed patterns from an Addition 95 | func (tRC *TalismanRC) RemoveAllowedPatterns(addition gitrepo.Addition) string { 96 | // Processing global allowed patterns 97 | for _, pattern := range tRC.AllowedPatterns { 98 | addition.Data = pattern.ReplaceAll(addition.Data, []byte("")) 99 | } 100 | 101 | // Processing allowed patterns based on file path 102 | for _, ignoreConfig := range tRC.FileIgnoreConfig { 103 | if addition.Matches(ignoreConfig.GetFileName()) { 104 | for _, pattern := range ignoreConfig.GetAllowedPatterns() { 105 | addition.Data = pattern.ReplaceAll(addition.Data, []byte("")) 106 | } 107 | } 108 | } 109 | return string(addition.Data) 110 | } 111 | 112 | // Deny answers true if the Addition should NOT be checked by the specified detector 113 | func (tRC *TalismanRC) Deny(addition gitrepo.Addition, detectorName string) bool { 114 | for _, pattern := range tRC.effectiveRules(detectorName) { 115 | if addition.Matches(pattern) { 116 | return true 117 | } 118 | } 119 | return false 120 | } 121 | 122 | // Accept answers true if the Addition should be checked by the specified detector 123 | func (tRC *TalismanRC) Accept(addition gitrepo.Addition, detectorName string) bool { 124 | return !tRC.Deny(addition, detectorName) 125 | } 126 | 127 | func (tRC *TalismanRC) effectiveRules(detectorName string) []string { 128 | var result []string 129 | for _, ignore := range tRC.FileIgnoreConfig { 130 | if ignore.isEffective(detectorName) { 131 | result = append(result, ignore.GetFileName()) 132 | } 133 | } 134 | return result 135 | } 136 | -------------------------------------------------------------------------------- /talismanrc/types.go: -------------------------------------------------------------------------------- 1 | package talismanrc 2 | 3 | import ( 4 | "regexp" 5 | "talisman/detector/severity" 6 | 7 | logr "github.com/sirupsen/logrus" 8 | ) 9 | 10 | type PatternString string 11 | 12 | type Pattern struct { 13 | *regexp.Regexp 14 | } 15 | 16 | func (p Pattern) MarshalYAML() (interface{}, error) { 17 | return p.String(), nil 18 | } 19 | 20 | func (p *Pattern) UnmarshalYAML(unmarshal func(interface{}) error) error { 21 | var s string 22 | err := unmarshal(&s) 23 | if err != nil { 24 | logr.Errorf("Pattern.UmarshalYAML error: %v", err) 25 | return err 26 | } 27 | *p = Pattern{regexp.MustCompile(s)} 28 | return nil 29 | } 30 | 31 | type CustomSeverityConfig struct { 32 | Detector string `yaml:"detector"` 33 | Severity severity.Severity `yaml:"severity"` 34 | } 35 | 36 | type FileIgnoreConfig struct { 37 | FileName string `yaml:"filename"` 38 | Checksum string `yaml:"checksum,omitempty"` 39 | IgnoreDetectors []string `yaml:"ignore_detectors,omitempty"` 40 | AllowedPatterns []string `yaml:"allowed_patterns,omitempty"` 41 | 42 | compiledPatterns []*regexp.Regexp 43 | } 44 | 45 | func (i *FileIgnoreConfig) isEffective(detectorName string) bool { 46 | return !isEmptyString(i.FileName) && 47 | contains(i.IgnoreDetectors, detectorName) 48 | } 49 | 50 | func (i *FileIgnoreConfig) GetFileName() string { 51 | return i.FileName 52 | } 53 | 54 | func (i *FileIgnoreConfig) ChecksumMatches(incomingChecksum string) bool { 55 | return i.Checksum == incomingChecksum 56 | } 57 | 58 | func (i *FileIgnoreConfig) GetAllowedPatterns() []*regexp.Regexp { 59 | if i.compiledPatterns == nil { 60 | i.compiledPatterns = make([]*regexp.Regexp, len(i.AllowedPatterns)) 61 | for idx, p := range i.AllowedPatterns { 62 | i.compiledPatterns[idx] = regexp.MustCompile(p) 63 | } 64 | } 65 | return i.compiledPatterns 66 | } 67 | 68 | func IgnoreFileWithChecksum(filename, checksum string) FileIgnoreConfig { 69 | return FileIgnoreConfig{FileName: filename, Checksum: checksum} 70 | } 71 | 72 | type ScopeConfig struct { 73 | ScopeName string `yaml:"scope"` 74 | } 75 | 76 | type ExperimentalConfig struct { 77 | Base64EntropyThreshold float64 `yaml:"base64EntropyThreshold,omitempty"` 78 | } 79 | -------------------------------------------------------------------------------- /talismanrc/types_test.go: -------------------------------------------------------------------------------- 1 | package talismanrc 2 | 3 | import ( 4 | "io" 5 | "regexp" 6 | "strings" 7 | "testing" 8 | 9 | logr "github.com/sirupsen/logrus" 10 | "github.com/stretchr/testify/assert" 11 | "gopkg.in/yaml.v2" 12 | ) 13 | 14 | func init() { 15 | logr.SetOutput(io.Discard) 16 | } 17 | 18 | func TestCustomMarshalling(t *testing.T) { 19 | t.Run("Can unmarshal yaml into a Pattern struct", func(t *testing.T) { 20 | savedPattern := []byte("text-pattern") 21 | fromText := Pattern{} 22 | err := yaml.Unmarshal(savedPattern, &fromText) 23 | assert.Nil(t, err, "Should have unmarshalled %s into a Pattern", savedPattern) 24 | assert.Equal(t, Pattern{regexp.MustCompile(string(savedPattern))}, fromText) 25 | }) 26 | 27 | t.Run("Can marshal a Pattern struct into yaml", func(t *testing.T) { 28 | pattern := Pattern{regexp.MustCompile("pattern")} 29 | str, err := yaml.Marshal(pattern) 30 | assert.Nil(t, err, "Should have marshalled %v into a string of yaml", pattern) 31 | assert.Equal(t, pattern.String(), strings.TrimSpace(string(str))) 32 | }) 33 | } 34 | 35 | func TestFileIgnoreConfig(t *testing.T) { 36 | t.Run("Checksum matching", func(t *testing.T) { 37 | fileIgnoreConfig := &FileIgnoreConfig{ 38 | FileName: "some_filename", 39 | Checksum: "some_checksum", 40 | IgnoreDetectors: nil, 41 | AllowedPatterns: nil, 42 | } 43 | 44 | assert.True(t, fileIgnoreConfig.ChecksumMatches("some_checksum")) 45 | assert.False(t, fileIgnoreConfig.ChecksumMatches("some_other_checksum")) 46 | }) 47 | 48 | t.Run("Compiles regexes for patterns as needed", func(t *testing.T) { 49 | fileIgnoreConfig := &FileIgnoreConfig{ 50 | FileName: "some_filename", 51 | Checksum: "some_checksum", 52 | IgnoreDetectors: nil, 53 | AllowedPatterns: nil, 54 | } 55 | 56 | //No allowed patterns specified 57 | allowedPatterns := fileIgnoreConfig.GetAllowedPatterns() 58 | assert.Equal(t, 0, len(allowedPatterns)) 59 | 60 | fileIgnoreConfig.compiledPatterns = nil 61 | fileIgnoreConfig.AllowedPatterns = []string{"[Ff]ile[nN]ame"} 62 | allowedPatterns = fileIgnoreConfig.GetAllowedPatterns() 63 | assert.Equal(t, 1, len(allowedPatterns)) 64 | assert.Regexp(t, allowedPatterns[0], "fileName") 65 | }) 66 | } 67 | -------------------------------------------------------------------------------- /talismanrc/util.go: -------------------------------------------------------------------------------- 1 | package talismanrc 2 | 3 | import "regexp" 4 | 5 | var emptyStringPattern = regexp.MustCompile(`^\s*$`) 6 | 7 | func isEmptyString(str string) bool { 8 | return emptyStringPattern.MatchString(str) 9 | } 10 | 11 | func contains(s []string, e string) bool { 12 | for _, a := range s { 13 | if a == e { 14 | return true 15 | } 16 | } 17 | return false 18 | } 19 | -------------------------------------------------------------------------------- /test-install.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | _linux_uname() { 4 | if [ "${FAKE_PARAMS[0]}" = "-m" ]; then 5 | echo "x86_64" 6 | else 7 | echo "Linux" 8 | fi 9 | } 10 | export -f _linux_uname 11 | 12 | _windows_uname() { 13 | if [ "${FAKE_PARAMS[0]}" = "-m" ]; then 14 | echo "i686" 15 | else 16 | echo "MINGW64_NT-10.0-19045" 17 | fi 18 | } 19 | export -f _windows_uname 20 | 21 | _mac_uname() { 22 | if [ "${FAKE_PARAMS[0]}" = "-m" ]; then 23 | echo "aarch64" 24 | else 25 | echo "Darwin" 26 | fi 27 | } 28 | export -f _mac_uname 29 | 30 | _curl_spy() { 31 | echo "${FAKE_PARAMS[@]}" >>"$1"/_curl_args 32 | echo 'download_url: talisman_linux_amd64checksums' 33 | } 34 | export -f _curl_spy 35 | 36 | setup() { 37 | temp=$(mktemp -d) 38 | fake uname _linux_uname 39 | fake curl "_curl_spy $temp" 40 | fake shasum true 41 | fake tput true 42 | } 43 | 44 | teardown() { 45 | rm -rf "$temp" 46 | } 47 | 48 | test_installs_without_sudo() { 49 | fake sudo 'echo "expected no sudo" && exit 1' 50 | INSTALL_LOCATION=$temp ./install.sh 51 | assert "test -x $temp/talisman_linux_amd64" "Should install file with executable mode" 52 | assert_matches "$temp/talisman_linux_amd64" "$(readlink "$temp/talisman")" "Should create a link" 53 | } 54 | 55 | test_installs_with_sudo_if_available() { 56 | fake touch 'echo "Permission denied" && exit 1' 57 | fake which 'echo "sudo installed" && exit 0' 58 | # shellcheck disable=SC2016 59 | fake sudo 'bash -c "${FAKE_PARAMS[*]}"' 60 | INSTALL_LOCATION=$temp ./install.sh 61 | assert "test -x $temp/talisman_linux_amd64" "Should install file with executable mode" 62 | assert_matches "$temp/talisman_linux_amd64" "$(readlink "$temp/talisman")" "Should create a link" 63 | } 64 | 65 | test_errors_if_unable_to_install() { 66 | fake touch 'echo "Permission denied" && exit 1' 67 | fake which 'echo "sudo not installed" && exit 1' 68 | assert_status_code 126 "INSTALL_LOCATION=$temp ./install.sh" 69 | } 70 | 71 | test_errors_if_no_install_location() { 72 | assert_status_code 1 "INSTALL_LOCATION=/does/not/exist ./install.sh" 73 | } 74 | 75 | test_defaults_to_installing_latest_release() { 76 | INSTALL_LOCATION=$temp ./install.sh 77 | requested_release=$(head -n 1 "$temp"/_curl_args | awk '{print $2}') 78 | assert_matches ".*releases/latest$" "$requested_release" "Should install latest release if no version specified" 79 | } 80 | 81 | test_installing_specific_version() { 82 | VERSION=v1.64.0 INSTALL_LOCATION=$temp ./install.sh 83 | requested_release=$(head -n 1 "$temp"/_curl_args | awk '{print $2}') 84 | assert_matches ".*releases/tags/v1.64.0$" "$requested_release" "Should install specified version" 85 | } 86 | 87 | test_mac_arm_binary_name() { 88 | fake uname _mac_uname 89 | fake curl 'echo "download_url: talisman_darwin_arm64checksums"' 90 | INSTALL_LOCATION=$temp ./install.sh 91 | assert "test -x $temp/talisman_darwin_arm64" "Should install file with executable mode" 92 | assert_matches "$temp/talisman_darwin_arm64" "$(readlink "$temp/talisman")" "Should create a link" 93 | } 94 | 95 | test_windows_binary_name() { 96 | fake uname _windows_uname 97 | fake curl 'echo "download_url: talisman_windows_386.exechecksums"' 98 | INSTALL_LOCATION=$temp ./install.sh 99 | assert "test -x $temp/talisman_windows_386.exe" "Should install file with executable mode" 100 | assert_matches "$temp/talisman_windows_386.exe" "$(readlink "$temp/talisman")" "Should create a link" 101 | } 102 | 103 | test_pre_commit_golang_version() { 104 | version_in_go_mod=$(grep '^go ' go.mod | awk '{print $2}') 105 | grep 'language_version:' .pre-commit-hooks.yaml | awk '{print $2}' | while read -r version_in_pre_commit; do 106 | assert_matches "$version_in_go_mod" "$version_in_pre_commit" "pre-commit-hooks should specify same golang version as go.mod" 107 | done 108 | } 109 | -------------------------------------------------------------------------------- /third-party/schema-store-talismanrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "$id": "https://github.com/thoughtworks/talisman/talismanrc", 4 | "title": "schema for .talismanrc", 5 | "type": "object", 6 | "additionalProperties": false, 7 | "fileMatch": [ 8 | ".talismanrc" 9 | ], 10 | "properties": { 11 | "fileignoreconfig": { 12 | "type": "array", 13 | "items": { 14 | "type": "object", 15 | "properties": { 16 | "filename": { 17 | "type": "string", 18 | "description": "Fully qualified filename" 19 | }, 20 | "checksum": { 21 | "type": "string", 22 | "description": "This field should always have the value specified by Talisman message" 23 | }, 24 | "ignore_detectors": { 25 | "type": "array", 26 | "description": "Disable specific detectors for a particular file", 27 | "items": { 28 | "type": "string", 29 | "enum": ["filecontent", "filename", "filesize"] 30 | } 31 | }, 32 | "allowed_patterns": { 33 | "type": "array", 34 | "description": "Keywords to ignore to reduce the number of false positives", 35 | "items": { 36 | "type": "string" 37 | } 38 | } 39 | }, 40 | "required": ["filename"] 41 | } 42 | }, 43 | "scopeconfig": { 44 | "type": "array", 45 | "description": "Talisman is configured to ignore certain files based on the specified scopes", 46 | "items": { 47 | "type": "object", 48 | "properties": { 49 | "scope": { 50 | "type": "string" 51 | } 52 | }, 53 | "required": ["scope"] 54 | } 55 | }, 56 | "allowed_patterns": { 57 | "type": "array", 58 | "description": "Keywords to ignore to reduce the number of false positives", 59 | "items": { 60 | "type": "string" 61 | } 62 | }, 63 | "custom_patterns": { 64 | "type": "array", 65 | "description": "You can specify custom regex patterns to look for in the current repository", 66 | "items": { 67 | "type": "string" 68 | } 69 | }, 70 | "custom_severities": { 71 | "type": "array", 72 | "description": "Custom detectors severities", 73 | "items": { 74 | "type": "object", 75 | "properties": { 76 | "detector": { 77 | "type": "string" 78 | }, 79 | "severity": { 80 | "type": "string", 81 | "enum": ["low", "medium", "high"] 82 | } 83 | }, 84 | "required": ["detector", "severity"] 85 | } 86 | }, 87 | "threshold": { 88 | "type": "string", 89 | "description": "Default minimal threshold", 90 | "enum": ["low", "medium", "high"] 91 | }, 92 | "version": { 93 | "type": "string", 94 | "description": ".talismanrc version" 95 | } 96 | }, 97 | "required": [] 98 | } 99 | -------------------------------------------------------------------------------- /utility/progress_bar.go: -------------------------------------------------------------------------------- 1 | package utility 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/cheggaaa/pb/v3" 8 | ) 9 | 10 | func GetProgressBar(out *os.File, title string) progressBar { 11 | if isTerminal(out) { 12 | return &defaultProgressBar{title: title} 13 | } else { 14 | return &noOpProgressBar{} 15 | } 16 | } 17 | 18 | func isTerminal(out *os.File) bool { 19 | fileInfo, _ := out.Stat() 20 | return (fileInfo.Mode() & os.ModeCharDevice) != 0 21 | } 22 | 23 | type progressBar interface { 24 | Start(int) 25 | Increment() 26 | Finish() 27 | } 28 | 29 | type noOpProgressBar struct { 30 | } 31 | 32 | func (d *noOpProgressBar) Start(int) {} 33 | 34 | func (d *noOpProgressBar) Increment() {} 35 | 36 | func (d *noOpProgressBar) Finish() {} 37 | 38 | type defaultProgressBar struct { 39 | bar *pb.ProgressBar 40 | title string 41 | } 42 | 43 | func (d *defaultProgressBar) Start(total int) { 44 | template := fmt.Sprintf(`{{ red "%s:" }} {{counters .}} {{ bar . "<" "-" (cycle . "↖" "↗" "↘" "↙" ) "." ">"}} {{percent . | rndcolor }} {{green}} {{blue}}`, d.title) 45 | bar := pb.ProgressBarTemplate(template).New(total) 46 | bar.Set(pb.Terminal, true) 47 | d.bar = bar.Start() 48 | } 49 | 50 | func (d *defaultProgressBar) Increment() { 51 | d.bar.Increment() 52 | } 53 | 54 | func (d *defaultProgressBar) Finish() { 55 | d.bar.Finish() 56 | } 57 | -------------------------------------------------------------------------------- /utility/progress_bar_test.go: -------------------------------------------------------------------------------- 1 | package utility 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestDefaultProgressBar(t *testing.T) { 10 | t.Run("Start should start inner progress bar", func(t *testing.T) { 11 | progressBar := &defaultProgressBar{} 12 | 13 | progressBar.Start(10) 14 | 15 | assert.True(t, progressBar.bar.IsStarted()) 16 | }) 17 | 18 | t.Run("Increment should update inner progress bar", func(t *testing.T) { 19 | progressBar := &defaultProgressBar{} 20 | 21 | progressBar.Start(10) 22 | progressBar.Increment() 23 | 24 | assert.Equal(t, int64(1), progressBar.bar.Current()) 25 | }) 26 | 27 | t.Run("Finish should finish progress bar", func(t *testing.T) { 28 | progressBar := &defaultProgressBar{} 29 | 30 | progressBar.Start(2) 31 | progressBar.Increment() 32 | progressBar.Increment() 33 | progressBar.Finish() 34 | 35 | assert.False(t, progressBar.bar.IsStarted()) 36 | }) 37 | } 38 | -------------------------------------------------------------------------------- /utility/sha_256_hasher.go: -------------------------------------------------------------------------------- 1 | package utility 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/hex" 6 | "talisman/gitrepo" 7 | 8 | "github.com/sirupsen/logrus" 9 | ) 10 | 11 | type SHA256Hasher interface { 12 | CollectiveSHA256Hash(paths []string) string 13 | Start() error 14 | Shutdown() error 15 | } 16 | 17 | type DefaultSHA256Hasher struct{} 18 | 19 | // CollectiveSHA256Hash return collective sha256 hash of the passed paths 20 | func (*DefaultSHA256Hasher) CollectiveSHA256Hash(paths []string) string { 21 | return collectiveSHA256Hash(paths, SafeReadFile) 22 | } 23 | 24 | func (*DefaultSHA256Hasher) Start() error { return nil } 25 | func (*DefaultSHA256Hasher) Shutdown() error { return nil } 26 | 27 | type gitBatchSHA256Hasher struct { 28 | br gitrepo.BatchReader 29 | } 30 | 31 | func (g *gitBatchSHA256Hasher) CollectiveSHA256Hash(paths []string) string { 32 | return collectiveSHA256Hash(paths, g.br.Read) 33 | } 34 | 35 | func (g *gitBatchSHA256Hasher) Start() error { 36 | return g.br.Start() 37 | } 38 | 39 | func (g *gitBatchSHA256Hasher) Shutdown() error { 40 | return g.br.Shutdown() 41 | } 42 | 43 | func hashByte(contentPtr *[]byte) string { 44 | contents := *contentPtr 45 | hasher := sha256.New() 46 | hasher.Write(contents) 47 | return hex.EncodeToString(hasher.Sum(nil)) 48 | } 49 | 50 | func collectiveSHA256Hash(paths []string, FileReader func(string) ([]byte, error)) string { 51 | var finHash = "" 52 | for _, path := range paths { 53 | sbyte := []byte(finHash) 54 | concatBytes := hashByte(&sbyte) 55 | nameByte := []byte(path) 56 | nameHash := hashByte(&nameByte) 57 | fileBytes, _ := FileReader(path) 58 | fileHash := hashByte(&fileBytes) 59 | finHash = concatBytes + fileHash + nameHash 60 | } 61 | c := []byte(finHash) 62 | m := hashByte(&c) 63 | return m 64 | } 65 | 66 | var hashers = make(map[string]SHA256Hasher) 67 | 68 | // MakeHasher returns a SHA256 file/object hasher based on mode and a repo root 69 | func MakeHasher(mode string, root string) SHA256Hasher { 70 | if hashers[mode] != nil { 71 | return hashers[mode] 72 | } 73 | switch mode { 74 | case "pre-push": 75 | hashers[mode] = &gitBatchSHA256Hasher{gitrepo.NewBatchGitHeadPathReader(root)} 76 | case "pre-commit": 77 | hashers[mode] = &gitBatchSHA256Hasher{gitrepo.NewBatchGitStagedPathReader(root)} 78 | case "scan": 79 | hashers[mode] = &gitBatchSHA256Hasher{gitrepo.NewBatchGitObjectHashReader(root)} 80 | case "pattern": 81 | hashers[mode] = &DefaultSHA256Hasher{} 82 | case "checksum": 83 | hashers[mode] = &gitBatchSHA256Hasher{gitrepo.NewBatchGitStagedPathReader(root)} 84 | case "default": 85 | hashers[mode] = &DefaultSHA256Hasher{} 86 | } 87 | err := hashers[mode].Start() 88 | if err != nil { 89 | logrus.Errorf("unable to start hasher: %v", err) 90 | return nil 91 | } 92 | return hashers[mode] 93 | } 94 | 95 | func DestroyHashers() { 96 | for _, hasher := range hashers { 97 | hasher.Shutdown() 98 | } 99 | hashers = make(map[string]SHA256Hasher) 100 | } 101 | -------------------------------------------------------------------------------- /utility/sha_256_hasher_test.go: -------------------------------------------------------------------------------- 1 | package utility 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestShouldReturnCorrectFileHash(t *testing.T) { 9 | hasher := DefaultSHA256Hasher{} 10 | checksumSomeFile := hasher.CollectiveSHA256Hash([]string{"some_file.pem"}) 11 | checksumTestSomeFile := hasher.CollectiveSHA256Hash([]string{"test/some_file.pem"}) 12 | assert.Equal(t, checksumSomeFile, "87139cc4d975333b25b6275f97680604add51b84eb8f4a3b9dcbbc652e6f27ac", "Should be equal to some_file.pem hash value") 13 | assert.Equal(t, checksumTestSomeFile, "25bd31a28bf9d4e06327f1c4a5cab2260574ae508803f66adcc393350e994866", "Should be equal to test/some_file.pem hash value") 14 | } 15 | 16 | func TestShouldReturnEmptyFileHashWhenNoPathsPassed(t *testing.T) { 17 | hasher := DefaultSHA256Hasher{} 18 | checksum := hasher.CollectiveSHA256Hash([]string{}) 19 | assert.Equal(t, checksum, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "Should be equal to empty hash value when no paths passed") 20 | } 21 | -------------------------------------------------------------------------------- /utility/utility.go: -------------------------------------------------------------------------------- 1 | package utility 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "os" 8 | "path" 9 | 10 | log "github.com/sirupsen/logrus" 11 | 12 | figure "github.com/common-nighthawk/go-figure" 13 | ) 14 | 15 | // UniqueItems returns the array of strings containing unique items 16 | func UniqueItems(stringSlice []string) []string { 17 | keys := make(map[string]bool) 18 | list := []string{} 19 | for _, entry := range stringSlice { 20 | if _, value := keys[entry]; !value { 21 | keys[entry] = true 22 | list = append(list, entry) 23 | } 24 | } 25 | return list 26 | } 27 | 28 | // Creates art for console output 29 | func CreateArt(msg string) { 30 | myFigure := figure.NewFigure(msg, "basic", true) 31 | myFigure.Print() 32 | } 33 | 34 | // Copies Files and Directories from source to destination 35 | func File(src, dst string) error { 36 | var err error 37 | var srcfd *os.File 38 | var dstfd *os.File 39 | var srcinfo os.FileInfo 40 | 41 | if srcfd, err = os.Open(src); err != nil { 42 | return err 43 | } 44 | defer srcfd.Close() 45 | 46 | if dstfd, err = os.Create(dst); err != nil { 47 | return err 48 | } 49 | defer dstfd.Close() 50 | 51 | if _, err = io.Copy(dstfd, srcfd); err != nil { 52 | return err 53 | } 54 | if srcinfo, err = os.Stat(src); err != nil { 55 | return err 56 | } 57 | return os.Chmod(dst, srcinfo.Mode()) 58 | } 59 | 60 | func Dir(src string, dst string) error { 61 | var err error 62 | var fds []os.FileInfo 63 | var srcinfo os.FileInfo 64 | 65 | if srcinfo, err = os.Stat(src); err != nil { 66 | return err 67 | 68 | } 69 | 70 | if err = os.MkdirAll(dst, srcinfo.Mode()); err != nil { 71 | return err 72 | } 73 | 74 | if fds, err = ioutil.ReadDir(src); err != nil { 75 | return err 76 | } 77 | for _, fd := range fds { 78 | srcfp := path.Join(src, fd.Name()) 79 | dstfp := path.Join(dst, fd.Name()) 80 | 81 | if fd.IsDir() { 82 | if err = Dir(srcfp, dstfp); err != nil { 83 | fmt.Println(err) 84 | } 85 | } else { 86 | if err = File(srcfp, dstfp); err != nil { 87 | fmt.Println(err) 88 | } 89 | } 90 | } 91 | return nil 92 | } 93 | 94 | func IsFileSymlink(path string) bool { 95 | fileMetadata, err := os.Lstat(path) 96 | if err != nil { 97 | return false 98 | } 99 | return fileMetadata.Mode()&os.ModeSymlink != 0 100 | } 101 | 102 | func SafeReadFile(path string) ([]byte, error) { 103 | if IsFileSymlink(path) { 104 | log.Debug("Symlink was detected! Not following symlink ", path) 105 | return []byte{}, nil 106 | } 107 | return ioutil.ReadFile(path) 108 | } 109 | -------------------------------------------------------------------------------- /utility/utility_test.go: -------------------------------------------------------------------------------- 1 | package utility 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "io/ioutil" 6 | "os" 7 | "testing" 8 | ) 9 | 10 | func TestShouldReadNormalFileCorrectly(t *testing.T) { 11 | tempFile, _ := ioutil.TempFile(os.TempDir(), "somefile") 12 | dataToBeWrittenInFile := []byte{0, 1, 2, 3} 13 | tempFile.Write(dataToBeWrittenInFile) 14 | tempFile.Close() 15 | 16 | readDataFromFileUsingIoutilDotReadFile, _ := ioutil.ReadFile(tempFile.Name()) 17 | readDataFromFileUsingSafeFileRead, _ := SafeReadFile(tempFile.Name()) 18 | os.Remove(tempFile.Name()) 19 | 20 | assert.Equal(t, readDataFromFileUsingIoutilDotReadFile, dataToBeWrittenInFile) 21 | assert.Equal(t, readDataFromFileUsingSafeFileRead, dataToBeWrittenInFile) 22 | } 23 | 24 | func TestShouldNotReadSymbolicLinkTargetFile(t *testing.T) { 25 | tempFile, _ := ioutil.TempFile(os.TempDir(), "somefile") 26 | dataToBeWrittenInFile := []byte{0, 1, 2, 3} 27 | tempFile.Write(dataToBeWrittenInFile) 28 | tempFile.Close() 29 | symlinkFileName := tempFile.Name() + "symlink" 30 | os.Symlink(tempFile.Name(), symlinkFileName) 31 | 32 | readDataFromSymlinkUsingIoutilDotReadFile, _ := ioutil.ReadFile(symlinkFileName) 33 | readDataFromSymlinkUsingSafeFileRead, _ := SafeReadFile(symlinkFileName) 34 | os.Remove(symlinkFileName) 35 | os.Remove(tempFile.Name()) 36 | 37 | assert.Equal(t, readDataFromSymlinkUsingIoutilDotReadFile, dataToBeWrittenInFile) 38 | assert.Equal(t, readDataFromSymlinkUsingSafeFileRead, []byte{}) 39 | } 40 | --------------------------------------------------------------------------------